From 762d52bca821b4a81ed703988ab2d2b43d97b142 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 20:39:35 +0100 Subject: [PATCH 01/14] Fix failing tests --- app/cli.php | 15 ++++++----- app/controllers/api/health.php | 9 ++++--- app/init.php | 8 ++++-- app/worker.php | 14 +++++++++- app/workers/deletes.php | 25 +++++++++--------- composer.json | 2 +- composer.lock | 27 +++++++++++++------- src/Appwrite/Event/Event.php | 8 +++--- src/Appwrite/Event/Func.php | 3 --- src/Appwrite/Resque/Worker.php | 12 +++++++++ tests/e2e/Services/Teams/TeamsBaseServer.php | 1 - 11 files changed, 81 insertions(+), 43 deletions(-) diff --git a/app/cli.php b/app/cli.php index edab558c99..1d0a41128b 100644 --- a/app/cli.php +++ b/app/cli.php @@ -67,7 +67,9 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $databaseName = $project->getAttribute('database'); if (isset($databases[$databaseName])) { - return $databases[$databaseName]; + $database = $databases[$databaseName]; + $database->setNamespace('_' . $project->getInternalId()); + return $database; } $dbAdapter = $pools @@ -76,10 +78,11 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, ->getResource(); $database = new Database($dbAdapter, $cache); - $database->setNamespace('_' . $project->getInternalId()); $databases[$databaseName] = $database; + $database->setNamespace('_' . $project->getInternalId()); + return $database; }; @@ -158,7 +161,7 @@ $cli = $platform->getCli(); $cli ->error() ->inject('error') - ->action(function(Throwable $error) { + ->action(function (Throwable $error) { Console::error($error->getMessage()); }); @@ -166,7 +169,7 @@ $cli ->init() ->inject('pools') ->inject('cache') - ->action(function(Group $pools, Cache $cache) { + ->action(function (Group $pools, Cache $cache) { $maxAttempts = 5; $sleep = 3; @@ -189,7 +192,7 @@ $cli $collections = Config::getParam('collections', []); $last = \array_key_last($collections); - if($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last)) { + if ($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last)) { $ready = true; break; } @@ -197,7 +200,7 @@ $cli sleep($sleep); } while ($attempts < $maxAttempts); - if(!$ready) { + if (!$ready) { throw new Exception("Console is not ready yet. Please try again later."); } }); diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index f65e65ba23..4ffa6771f8 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -8,6 +8,8 @@ use Utopia\App; use Utopia\Config\Config; use Utopia\Database\Document; use Utopia\Pools\Group; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; @@ -396,10 +398,11 @@ App::get('/v1/health/queue/functions') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queueConnection') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::FUNCTIONS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queueConnection, Response $response) { + $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queueConnection); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/storage/local') diff --git a/app/init.php b/app/init.php index 31f82740b6..172fd845c2 100644 --- a/app/init.php +++ b/app/init.php @@ -76,6 +76,7 @@ use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; use Utopia\Queue; +use Utopia\Queue\Connection; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -852,9 +853,12 @@ App::setResource('mails', fn() => new Mail()); App::setResource('deletes', fn() => new Delete()); App::setResource('database', fn() => new EventDatabase()); App::setResource('messaging', fn() => new Phone()); -App::setResource('queueForFunctions', function (Group $pools) { - return new Func($pools->get('queue')->pop()->getResource()); +App::setResource('queueConnection', function (Group $pools) { + return $pools->get('queue')->pop()->getResource(); }, ['pools']); +App::setResource('queueForFunctions', function (Connection $queueConnection) { + return new Func($queueConnection); +}, ['queueConnection']); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); diff --git a/app/worker.php b/app/worker.php index 42a5f92439..8ebd1b5427 100644 --- a/app/worker.php +++ b/app/worker.php @@ -17,6 +17,7 @@ use Utopia\Queue\Server; use Utopia\Registry\Registry; use Utopia\Logger\Log; use Utopia\Logger\Logger; +use Utopia\Pools\Group; Runtime::enableCoroutine(SWOOLE_HOOK_ALL); @@ -55,7 +56,6 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, $adapter = new Database($database, $cache); $adapter->setNamespace('_' . $project->getInternalId()); - return $adapter; }, ['cache', 'register', 'message', 'dbForConsole']); @@ -93,9 +93,14 @@ Server::setResource('statsd', function ($register) { return $register->get('statsd'); }, ['register']); +Server::setResource('pools', function ($register) { + return $register->get('pools'); +}, ['register']); + $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); +$workerNumber = 1; if (empty(App::getEnv('QUEUE'))) { throw new Exception('Please configure "QUEUE" environemnt variable.'); @@ -104,6 +109,13 @@ if (empty(App::getEnv('QUEUE'))) { $adapter = new Swoole($connection, $workerNumber, App::getEnv('QUEUE')); $server = new Server($adapter); +$server + ->shutdown() + ->inject('pools') + ->action(function (Group $pools) { + $pools->reclaim(); + }); + $server ->error() ->inject('error') diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 3e0df8ce38..bfbb55008f 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -276,30 +276,29 @@ class DeletesV1 extends Worker { $userId = $document->getId(); + $dbForProject = $this->getProjectDB($project); + // Delete all sessions of this user from the sessions table and update the sessions field of the user record $this->deleteByGroup('sessions', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($project)); + ], $dbForProject); - $this->getProjectDB($project)->deleteCachedDocument('users', $userId); + $dbForProject->deleteCachedDocument('users', $userId); // Delete Memberships and decrement team membership counts $this->deleteByGroup('memberships', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($project), function (Document $document) use ($project) { - + ], $dbForProject, function (Document $document) use ($dbForProject) { if ($document->getAttribute('confirm')) { // Count only confirmed members $teamId = $document->getAttribute('teamId'); - $team = $this->getProjectDB($project)->getDocument('teams', $teamId); + $team = $dbForProject->getDocument('teams', $teamId); if (!$team->isEmpty()) { - $team = $this - ->getProjectDB($project) - ->updateDocument( - 'teams', - $teamId, - // Ensure that total >= 0 + $team = $dbForProject->updateDocument( + 'teams', + $teamId, + // Ensure that total >= 0 $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0)) - ); + ); } } }); @@ -307,7 +306,7 @@ class DeletesV1 extends Worker // Delete tokens $this->deleteByGroup('tokens', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($project)); + ], $dbForProject); } /** diff --git a/composer.json b/composer.json index ffd9c6b6b9..7195967194 100644 --- a/composer.json +++ b/composer.json @@ -53,7 +53,7 @@ "utopia-php/domains": "1.1.*", "utopia-php/framework": "0.25.*", "utopia-php/image": "0.5.*", - "utopia-php/queue": "0.4.*", + "utopia-php/queue": "0.5.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.3.*", "utopia-php/orchestration": "0.9.*", diff --git a/composer.lock b/composer.lock index 517065af56..04dc731953 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a673091aa6bd8ef01380b63245427c93", + "content-hash": "56cd96524e05cf30210256539cd7cb6a", "packages": [ { "name": "adhocore/jwt", @@ -2359,16 +2359,16 @@ }, { "name": "utopia-php/queue", - "version": "0.4.1", + "version": "dev-feat-connection-ping", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e" + "reference": "42e47dc8b40b2040ea9b70c644799d7d7a097520" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/0b69ede484a04c567cbb202f592d8e5e3cd2433e", - "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/42e47dc8b40b2040ea9b70c644799d7d7a097520", + "reference": "42e47dc8b40b2040ea9b70c644799d7d7a097520", "shasum": "" }, "require": { @@ -2414,9 +2414,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.4.1" + "source": "https://github.com/utopia-php/queue/tree/feat-connection-ping" }, - "time": "2022-11-15T16:56:37+00:00" + "time": "2022-11-16T17:12:06+00:00" }, { "name": "utopia-php/registry", @@ -5276,9 +5276,18 @@ "time": "2022-09-28T08:42:51+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/queue", + "version": "dev-feat-connection-ping", + "alias": "0.4.0", + "alias_normalized": "0.4.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/queue": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 6e3401e11b..adb5772f97 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -116,9 +116,9 @@ class Event /** * Get project for this event. * - * @return Document + * @return ?Document */ - public function getProject(): Document + public function getProject(): ?Document { return $this->project; } @@ -139,9 +139,9 @@ class Event /** * Get project for this event. * - * @return Document + * @return ?Document */ - public function getUser(): Document + public function getUser(): ?Document { return $this->user; } diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 22940ad08e..5f8b4c80c6 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -2,9 +2,6 @@ namespace Appwrite\Event; -use DateTime; -use Resque; -use ResqueScheduler; use Utopia\Database\Document; use Utopia\Queue\Client; use Utopia\Queue\Connection; diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 5d05d77576..bdb1ccb8df 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -169,6 +169,7 @@ abstract class Worker * @param Document $project * @return Database */ + protected static $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools protected function getProjectDB(Document $project): Database { global $register; @@ -179,6 +180,14 @@ abstract class Worker return $this->getConsoleDB(); } + $databaseName = $project->getAttribute('database'); + + if (isset(self::$databases[$databaseName])) { + $database = self::$databases[$databaseName]; + $database->setNamespace('_' . $project->getInternalId()); + return $database; + } + $dbAdapter = $pools ->get($project->getAttribute('database')) ->pop() @@ -186,6 +195,9 @@ abstract class Worker ; $database = new Database($dbAdapter, $this->getCache()); + + self::$databases[$databaseName] = $database; + $database->setNamespace('_' . $project->getInternalId()); return $database; diff --git a/tests/e2e/Services/Teams/TeamsBaseServer.php b/tests/e2e/Services/Teams/TeamsBaseServer.php index df508e5e37..c4d92ff0d9 100644 --- a/tests/e2e/Services/Teams/TeamsBaseServer.php +++ b/tests/e2e/Services/Teams/TeamsBaseServer.php @@ -246,7 +246,6 @@ trait TeamsBaseServer 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); - $this->assertEquals(200, $response['headers']['status-code']); $this->assertNotEmpty($response['body']['$id']); $this->assertEquals('Arsenal', $response['body']['name']); From 48f143547abdd94f53d4424d76690d05e186f8c6 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 20:49:32 +0100 Subject: [PATCH 02/14] Upgrade libs --- composer.lock | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/composer.lock b/composer.lock index eb02c929ae..20824e73a3 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "56cd96524e05cf30210256539cd7cb6a", + "content-hash": "fee282106d2be39403dadaa9843adb67", "packages": [ { "name": "adhocore/jwt", @@ -2359,16 +2359,16 @@ }, { "name": "utopia-php/queue", - "version": "dev-feat-connection-ping", + "version": "0.5.1", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "42e47dc8b40b2040ea9b70c644799d7d7a097520" + "reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/42e47dc8b40b2040ea9b70c644799d7d7a097520", - "reference": "42e47dc8b40b2040ea9b70c644799d7d7a097520", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/4b39e1f232b2e619b0d7fb4004f2356df334ddc1", + "reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1", "shasum": "" }, "require": { @@ -2414,9 +2414,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/feat-connection-ping" + "source": "https://github.com/utopia-php/queue/tree/0.5.1" }, - "time": "2022-11-16T17:12:06+00:00" + "time": "2022-11-16T19:47:26+00:00" }, { "name": "utopia-php/registry", @@ -5276,18 +5276,9 @@ "time": "2022-09-28T08:42:51+00:00" } ], - "aliases": [ - { - "package": "utopia-php/queue", - "version": "dev-feat-connection-ping", - "alias": "0.4.0", - "alias_normalized": "0.4.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/queue": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -5311,5 +5302,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.1.0" } From 5dd31bcdfbe019e27b7d5b80ffcb980e1809ace4 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 16 Nov 2022 20:53:59 +0100 Subject: [PATCH 03/14] Fix init startup --- app/cli.php | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/app/cli.php b/app/cli.php index 1d0a41128b..9dec6118f6 100644 --- a/app/cli.php +++ b/app/cli.php @@ -179,25 +179,29 @@ $cli do { $attempts++; - // Prepare database connection - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource(); + try { + // Prepare database connection + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource(); - $dbForConsole = new Database($dbAdapter, $cache); - $dbForConsole->setNamespace('console'); + $dbForConsole = new Database($dbAdapter, $cache); + $dbForConsole->setNamespace('console'); - // Ensure tables exist - $collections = Config::getParam('collections', []); - $last = \array_key_last($collections); + // Ensure tables exist + $collections = Config::getParam('collections', []); + $last = \array_key_last($collections); + + if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) { + throw new Exception('Tables not ready yet.'); + } - if ($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last)) { $ready = true; - break; + } catch (\Exception $err) { + Console::warning($err->getMessage()); + sleep($sleep); } - - sleep($sleep); } while ($attempts < $maxAttempts); if (!$ready) { From d0c253e45de0ed6283fd06ac4c3013e7b73796d7 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Fri, 18 Nov 2022 08:10:31 +0100 Subject: [PATCH 04/14] Update lockfile after merge --- composer.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/composer.lock b/composer.lock index d450e93cb9..03dfcd1da7 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2b8002030d115fd67f57b89a9972f104", + "content-hash": "4893e1c13630239fe6a20d1c652eb484", "packages": [ { "name": "adhocore/jwt", @@ -2460,16 +2460,16 @@ }, { "name": "utopia-php/queue", - "version": "0.4.1", + "version": "0.5.1", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e" + "reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/0b69ede484a04c567cbb202f592d8e5e3cd2433e", - "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/4b39e1f232b2e619b0d7fb4004f2356df334ddc1", + "reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1", "shasum": "" }, "require": { @@ -2515,9 +2515,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.4.1" + "source": "https://github.com/utopia-php/queue/tree/0.5.1" }, - "time": "2022-11-15T16:56:37+00:00" + "time": "2022-11-16T19:47:26+00:00" }, { "name": "utopia-php/registry", @@ -5403,5 +5403,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.1.0" } From 26ea9d675ace292ac0b16150013b5643f2c03fc7 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Fri, 18 Nov 2022 08:13:22 +0100 Subject: [PATCH 05/14] Linter fix --- app/controllers/api/project.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 049a957a4a..49b493f00a 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -106,7 +106,6 @@ App::get('/v1/project/usage') 'executions' => $stats[$metrics[6]] ?? [], 'buckets' => $stats[$metrics[7]] ?? [], ]); - } $response->dynamic($usage, Response::MODEL_USAGE_PROJECT); From 715273798e0e7a9491c35ade7531f1366eec31c4 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Fri, 18 Nov 2022 09:22:04 +0100 Subject: [PATCH 06/14] More bug fixing around failing tests --- app/cli.php | 2 ++ docker-compose.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/app/cli.php b/app/cli.php index 9dec6118f6..4d64867ed3 100644 --- a/app/cli.php +++ b/app/cli.php @@ -180,6 +180,8 @@ $cli $attempts++; try { + $pools->get('console')->reclaim(); + // Prepare database connection $dbAdapter = $pools ->get('console') diff --git a/docker-compose.yml b/docker-compose.yml index 3011e63af6..854e9cd167 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -390,6 +390,7 @@ services: - _APP_CONNECTIONS_DB_PROJECT - _APP_CONNECTIONS_CACHE - _APP_CONNECTIONS_QUEUE + - _APP_CONNECTIONS_STORAGE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG From 0075956c6cf13c9913c0e786a8150cba563b47eb Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Fri, 18 Nov 2022 09:51:28 +0100 Subject: [PATCH 07/14] Introduce ENV to disable console invites, fix outdated code after merge --- .env | 1 + app/cli.php | 2 +- app/config/variables.php | 11 ++++++++++- app/init.php | 2 +- app/views/install/compose.phtml | 1 + app/workers/messaging.php | 26 +++++++++++++++----------- docker-compose.yml | 1 + 7 files changed, 30 insertions(+), 14 deletions(-) diff --git a/.env b/.env index 5359d6b9a7..14f064bc6e 100644 --- a/.env +++ b/.env @@ -4,6 +4,7 @@ _APP_WORKER_PER_CORE=6 _APP_CONSOLE_WHITELIST_ROOT=disabled _APP_CONSOLE_WHITELIST_EMAILS= _APP_CONSOLE_WHITELIST_IPS= +_APP_CONSOLE_INVITES=enabled _APP_SYSTEM_EMAIL_NAME=Appwrite _APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io _APP_SYSTEM_SECURITY_EMAIL_ADDRESS=security@appwrite.io diff --git a/app/cli.php b/app/cli.php index 4d64867ed3..9857261b47 100644 --- a/app/cli.php +++ b/app/cli.php @@ -181,7 +181,7 @@ $cli try { $pools->get('console')->reclaim(); - + // Prepare database connection $dbAdapter = $pools ->get('console') diff --git a/app/config/variables.php b/app/config/variables.php index 5fe1bc35c4..ba39e01a2f 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -203,7 +203,16 @@ return [ 'required' => false, 'question' => '', 'filter' => '' - ] + ], + [ + 'name' => '_APP_CONSOLE_INVITES', + 'description' => 'This option allows you to disable the invitation of new users on the Appwrite console. When enabled, console users are allowed to invite new users to a project. By default this option is enabled.', + 'introduction' => '1.2.0', + 'default' => 'enabled', + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], ], [ diff --git a/app/init.php b/app/init.php index f77d82697e..5e66cd5872 100644 --- a/app/init.php +++ b/app/init.php @@ -1028,7 +1028,7 @@ App::setResource('console', function () { 'legalAddress' => '', 'legalTaxId' => '', 'auths' => [ - 'invites' => false, + 'invites' => App::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', 'limit' => (App::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds ], diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index bac07e8415..dc7549aa3f 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -88,6 +88,7 @@ services: - _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_EMAILS - _APP_CONSOLE_WHITELIST_IPS + - _APP_CONSOLE_INVITES - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS diff --git a/app/workers/messaging.php b/app/workers/messaging.php index 4c82a1abff..5732c8c00b 100644 --- a/app/workers/messaging.php +++ b/app/workers/messaging.php @@ -1,16 +1,17 @@ args['recipient']; - $message = $this->args['message']; + $message = new SMS( + to: [$this->args['recipient']], + content: $this->args['message'], + from: $this->from, + ); try { - $this->sms->send($this->from, $recipient, $message); + $this->sms->send($message); } catch (\Exception $error) { throw new Exception('Error sending message: ' . $error->getMessage(), 500); } diff --git a/docker-compose.yml b/docker-compose.yml index 854e9cd167..1834330f8c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -109,6 +109,7 @@ services: - _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_EMAILS - _APP_CONSOLE_WHITELIST_IPS + - _APP_CONSOLE_INVITES - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS From eba34a294dedab9c02ff920a57779164514c6c5e Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Fri, 18 Nov 2022 09:52:22 +0100 Subject: [PATCH 08/14] Update composer.lock --- composer.lock | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/composer.lock b/composer.lock index b8fc10b161..f47d853d6b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fc65b26befaa50feffc51053f5064e1a", + "content-hash": "4893e1c13630239fe6a20d1c652eb484", "packages": [ { "name": "adhocore/jwt", @@ -2460,16 +2460,16 @@ }, { "name": "utopia-php/queue", - "version": "0.4.1", + "version": "0.5.1", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e" + "reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/0b69ede484a04c567cbb202f592d8e5e3cd2433e", - "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/4b39e1f232b2e619b0d7fb4004f2356df334ddc1", + "reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1", "shasum": "" }, "require": { @@ -2515,9 +2515,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.4.1" + "source": "https://github.com/utopia-php/queue/tree/0.5.1" }, - "time": "2022-11-15T16:56:37+00:00" + "time": "2022-11-16T19:47:26+00:00" }, { "name": "utopia-php/registry", @@ -3555,16 +3555,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.18", + "version": "9.2.19", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a" + "reference": "c77b56b63e3d2031bd8997fcec43c1925ae46559" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/12fddc491826940cf9b7e88ad9664cf51f0f6d0a", - "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/c77b56b63e3d2031bd8997fcec43c1925ae46559", + "reference": "c77b56b63e3d2031bd8997fcec43c1925ae46559", "shasum": "" }, "require": { @@ -3620,7 +3620,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.18" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.19" }, "funding": [ { @@ -3628,7 +3628,7 @@ "type": "github" } ], - "time": "2022-10-27T13:35:33+00:00" + "time": "2022-11-18T07:47:47+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5403,5 +5403,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.1.0" } From b39aa060c80c25fe94e1913239a3e79ba29548e5 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Fri, 18 Nov 2022 10:15:11 +0100 Subject: [PATCH 09/14] Fix usage endpoint in test --- .env | 2 +- tests/e2e/General/UsageTest.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 14f064bc6e..405d72cd5f 100644 --- a/.env +++ b/.env @@ -63,7 +63,7 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 -_APP_USAGE_TIMESERIES_INTERVAL=60 +_APP_USAGE_TIMESERIES_INTERVAL=15 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_USAGE_STATS=enabled _APP_LOGGING_PROVIDER= diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index cbe90c91fb..4d71edea5a 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -93,13 +93,13 @@ class UsageTest extends Scope $requestsCount = $data['requestsCount']; // console request - $cheaders = [ + $headers = [ 'origin' => 'http://localhost', 'x-appwrite-project' => 'console', 'cookie' => 'a_session_console=' . $this->getRoot()['session'], ]; - $res = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId . '/usage?range=30d', $cheaders); + $res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers); $res = $res['body']; $this->assertEquals(9, count($res)); @@ -110,7 +110,7 @@ class UsageTest extends Scope $this->assertEquals($requestsCount, $res['requests'][array_key_last($res['requests'])]['value']); $this->validateDates($res['requests']); - $res = $this->client->call(Client::METHOD_GET, '/users/usage?range=30d', array_merge($cheaders, [ + $res = $this->client->call(Client::METHOD_GET, '/users/usage?range=30d', array_merge($headers, [ 'x-appwrite-project' => $projectId, 'x-appwrite-mode' => 'admin' ])); From 5ec548cc06725e651fcc6cdef79a58fcd4b80292 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Fri, 18 Nov 2022 14:13:33 +0100 Subject: [PATCH 10/14] Fix more failing tests --- .env | 2 +- app/controllers/api/account.php | 1 + src/Appwrite/Usage/Calculators/TimeSeries.php | 1 + tests/e2e/General/UsageTest.php | 24 +++++++++++++++---- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 405d72cd5f..15a25aa933 100644 --- a/.env +++ b/.env @@ -63,7 +63,7 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 -_APP_USAGE_TIMESERIES_INTERVAL=15 +_APP_USAGE_TIMESERIES_INTERVAL=20 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_USAGE_STATS=enabled _APP_LOGGING_PROVIDER= diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 115948f710..d973ffd14c 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -279,6 +279,7 @@ App::get('/v1/account/sessions/oauth2/:provider') ->action(function (string $provider, string $success, string $failure, array $scopes, Request $request, Response $response, Document $project) use ($oauthDefaultSuccess, $oauthDefaultFailure) { $protocol = $request->getProtocol(); + $callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId(); $appId = $project->getAttribute('authProviders', [])[$provider . 'Appid'] ?? ''; $appSecret = $project->getAttribute('authProviders', [])[$provider . 'Secret'] ?? '{}'; diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 2ef3d77cac..6f4fb94a7d 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -494,6 +494,7 @@ class TimeSeries extends Calculator if (empty($point['projectInternalId'] ?? null)) { continue; } + $this->createOrUpdateMetric( $point['projectInternalId'], $point['time'], diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 4d71edea5a..a38fa133cd 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -97,6 +97,8 @@ class UsageTest extends Scope 'origin' => 'http://localhost', 'x-appwrite-project' => 'console', 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + 'x-appwrite-project' => $projectId, + 'x-appwrite-mode' => 'admin', ]; $res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers); @@ -146,11 +148,20 @@ class UsageTest extends Scope for ($i = 0; $i < 10; $i++) { $name = uniqid() . ' bucket'; - $res = $this->client->call(Client::METHOD_POST, '/storage/buckets', $headers, [ + $res = $this->client->call(Client::METHOD_POST, '/storage/buckets', \array_merge($headers, [ + 'content-type' => 'application/json' + ]), [ 'bucketId' => 'unique()', 'name' => $name, - 'permission' => 'bucket' + 'fileSecurity' => false, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], ]); + \var_dump($res['body']); $this->assertEquals($name, $res['body']['name']); $this->assertNotEmpty($res['body']['$id']); $bucketId = $res['body']['$id']; @@ -263,9 +274,11 @@ class UsageTest extends Scope 'origin' => 'http://localhost', 'x-appwrite-project' => 'console', 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + 'x-appwrite-project' => $projectId, + 'x-appwrite-mode' => 'admin', ]; - $res = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId . '/usage?range=30d', $headers); + $res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers); $res = $res['body']; $this->assertEquals(9, count($res)); @@ -503,8 +516,11 @@ class UsageTest extends Scope 'origin' => 'http://localhost', 'x-appwrite-project' => 'console', 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + 'x-appwrite-project' => $projectId, + 'x-appwrite-mode' => 'admin', ]; - $res = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId . '/usage?range=30d', $headers); + + $res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers); $res = $res['body']; $this->assertEquals(9, count($res)); From 9163aa360ef6d1a3b82347f0f717ddcf5e354049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 18 Nov 2022 15:17:14 +0100 Subject: [PATCH 11/14] Update tests/e2e/General/UsageTest.php Co-authored-by: Christy Jacob --- tests/e2e/General/UsageTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index a38fa133cd..4dddaf1535 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -161,7 +161,6 @@ class UsageTest extends Scope Permission::delete(Role::any()), ], ]); - \var_dump($res['body']); $this->assertEquals($name, $res['body']['name']); $this->assertNotEmpty($res['body']['$id']); $bucketId = $res['body']['$id']; From 421cebced8d6a375dc772d1821378237c4e432c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 18 Nov 2022 15:17:51 +0100 Subject: [PATCH 12/14] Update app/config/variables.php Co-authored-by: Christy Jacob --- app/config/variables.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/variables.php b/app/config/variables.php index ba39e01a2f..1eb2688c62 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -206,7 +206,7 @@ return [ ], [ 'name' => '_APP_CONSOLE_INVITES', - 'description' => 'This option allows you to disable the invitation of new users on the Appwrite console. When enabled, console users are allowed to invite new users to a project. By default this option is enabled.', + 'description' => 'This option allows you to disable the invitation of new users to the Appwrite console. When enabled, console users are allowed to invite new users to a project. By default this option is enabled.', 'introduction' => '1.2.0', 'default' => 'enabled', 'required' => false, From a13c671a2baa1c28eb02f9f3ada2948e11373103 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Fri, 18 Nov 2022 15:20:05 +0100 Subject: [PATCH 13/14] PR review changes --- app/init.php | 8 ++++---- app/worker.php | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/app/init.php b/app/init.php index c22ce76151..233e6ba9cc 100644 --- a/app/init.php +++ b/app/init.php @@ -848,12 +848,12 @@ App::setResource('mails', fn() => new Mail()); App::setResource('deletes', fn() => new Delete()); App::setResource('database', fn() => new EventDatabase()); App::setResource('messaging', fn() => new Phone()); -App::setResource('queueConnection', function (Group $pools) { +App::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); -App::setResource('queueForFunctions', function (Connection $queueConnection) { - return new Func($queueConnection); -}, ['queueConnection']); +App::setResource('queueForFunctions', function (Connection $queue) { + return new Func($queue); +}, ['queue']); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); diff --git a/app/worker.php b/app/worker.php index 8ebd1b5427..8151381d4a 100644 --- a/app/worker.php +++ b/app/worker.php @@ -100,7 +100,6 @@ Server::setResource('pools', function ($register) { $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); -$workerNumber = 1; if (empty(App::getEnv('QUEUE'))) { throw new Exception('Please configure "QUEUE" environemnt variable.'); From 91de1cbcf35ab40cd3c7e9b1c7ed06578405e362 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Fri, 18 Nov 2022 15:22:46 +0100 Subject: [PATCH 14/14] Fix bug --- app/controllers/api/health.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 4ffa6771f8..d6a2612f32 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -398,10 +398,10 @@ App::get('/v1/health/queue/functions') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->inject('queueConnection') + ->inject('queue') ->inject('response') - ->action(function (Connection $queueConnection, Response $response) { - $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queueConnection); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue); $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']);