diff --git a/.env b/.env index 5359d6b9a7..15a25aa933 100644 --- a/.env +++ b/.env @@ -4,6 +4,7 @@ _APP_WORKER_PER_CORE=6 _APP_CONSOLE_WHITELIST_ROOT=disabled _APP_CONSOLE_WHITELIST_EMAILS= _APP_CONSOLE_WHITELIST_IPS= +_APP_CONSOLE_INVITES=enabled _APP_SYSTEM_EMAIL_NAME=Appwrite _APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io _APP_SYSTEM_SECURITY_EMAIL_ADDRESS=security@appwrite.io @@ -62,7 +63,7 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 -_APP_USAGE_TIMESERIES_INTERVAL=60 +_APP_USAGE_TIMESERIES_INTERVAL=20 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_USAGE_STATS=enabled _APP_LOGGING_PROVIDER= diff --git a/app/cli.php b/app/cli.php index c7e5b98157..9857261b47 100644 --- a/app/cli.php +++ b/app/cli.php @@ -67,7 +67,9 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $databaseName = $project->getAttribute('database'); if (isset($databases[$databaseName])) { - return $databases[$databaseName]; + $database = $databases[$databaseName]; + $database->setNamespace('_' . $project->getInternalId()); + return $database; } $dbAdapter = $pools @@ -76,10 +78,11 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, ->getResource(); $database = new Database($dbAdapter, $cache); - $database->setNamespace('_' . $project->getInternalId()); $databases[$databaseName] = $database; + $database->setNamespace('_' . $project->getInternalId()); + return $database; }; @@ -176,25 +179,31 @@ $cli do { $attempts++; - // Prepare database connection - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource(); + try { + $pools->get('console')->reclaim(); - $dbForConsole = new Database($dbAdapter, $cache); - $dbForConsole->setNamespace('console'); + // Prepare database connection + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource(); - // Ensure tables exist - $collections = Config::getParam('collections', []); - $last = \array_key_last($collections); + $dbForConsole = new Database($dbAdapter, $cache); + $dbForConsole->setNamespace('console'); + + // Ensure tables exist + $collections = Config::getParam('collections', []); + $last = \array_key_last($collections); + + if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) { + throw new Exception('Tables not ready yet.'); + } - if ($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last)) { $ready = true; - break; + } catch (\Exception $err) { + Console::warning($err->getMessage()); + sleep($sleep); } - - sleep($sleep); } while ($attempts < $maxAttempts); if (!$ready) { diff --git a/app/config/variables.php b/app/config/variables.php index 5fe1bc35c4..1eb2688c62 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -203,7 +203,16 @@ return [ 'required' => false, 'question' => '', 'filter' => '' - ] + ], + [ + 'name' => '_APP_CONSOLE_INVITES', + 'description' => 'This option allows you to disable the invitation of new users to the Appwrite console. When enabled, console users are allowed to invite new users to a project. By default this option is enabled.', + 'introduction' => '1.2.0', + 'default' => 'enabled', + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], ], [ diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 115948f710..d973ffd14c 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -279,6 +279,7 @@ App::get('/v1/account/sessions/oauth2/:provider') ->action(function (string $provider, string $success, string $failure, array $scopes, Request $request, Response $response, Document $project) use ($oauthDefaultSuccess, $oauthDefaultFailure) { $protocol = $request->getProtocol(); + $callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId(); $appId = $project->getAttribute('authProviders', [])[$provider . 'Appid'] ?? ''; $appSecret = $project->getAttribute('authProviders', [])[$provider . 'Secret'] ?? '{}'; diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index f65e65ba23..d6a2612f32 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -8,6 +8,8 @@ use Utopia\App; use Utopia\Config\Config; use Utopia\Database\Document; use Utopia\Pools\Group; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; @@ -396,10 +398,11 @@ App::get('/v1/health/queue/functions') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::FUNCTIONS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/storage/local') diff --git a/app/init.php b/app/init.php index f9442b8a86..5016a4246c 100644 --- a/app/init.php +++ b/app/init.php @@ -75,6 +75,7 @@ use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; use Utopia\CLI\Console; use Utopia\Queue; +use Utopia\Queue\Connection; use Utopia\Storage\Storage; const APP_NAME = 'Appwrite'; @@ -847,9 +848,12 @@ App::setResource('mails', fn() => new Mail()); App::setResource('deletes', fn() => new Delete()); App::setResource('database', fn() => new EventDatabase()); App::setResource('messaging', fn() => new Phone()); -App::setResource('queueForFunctions', function (Group $pools) { - return new Func($pools->get('queue')->pop()->getResource()); +App::setResource('queue', function (Group $pools) { + return $pools->get('queue')->pop()->getResource(); }, ['pools']); +App::setResource('queueForFunctions', function (Connection $queue) { + return new Func($queue); +}, ['queue']); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); @@ -1024,7 +1028,7 @@ App::setResource('console', function () { 'legalAddress' => '', 'legalTaxId' => '', 'auths' => [ - 'invites' => false, + 'invites' => App::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', 'limit' => (App::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds ], diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index bac07e8415..dc7549aa3f 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -88,6 +88,7 @@ services: - _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_EMAILS - _APP_CONSOLE_WHITELIST_IPS + - _APP_CONSOLE_INVITES - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS diff --git a/app/worker.php b/app/worker.php index 42a5f92439..8151381d4a 100644 --- a/app/worker.php +++ b/app/worker.php @@ -17,6 +17,7 @@ use Utopia\Queue\Server; use Utopia\Registry\Registry; use Utopia\Logger\Log; use Utopia\Logger\Logger; +use Utopia\Pools\Group; Runtime::enableCoroutine(SWOOLE_HOOK_ALL); @@ -55,7 +56,6 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, $adapter = new Database($database, $cache); $adapter->setNamespace('_' . $project->getInternalId()); - return $adapter; }, ['cache', 'register', 'message', 'dbForConsole']); @@ -93,6 +93,10 @@ Server::setResource('statsd', function ($register) { return $register->get('statsd'); }, ['register']); +Server::setResource('pools', function ($register) { + return $register->get('pools'); +}, ['register']); + $pools = $register->get('pools'); $connection = $pools->get('queue')->pop()->getResource(); $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); @@ -104,6 +108,13 @@ if (empty(App::getEnv('QUEUE'))) { $adapter = new Swoole($connection, $workerNumber, App::getEnv('QUEUE')); $server = new Server($adapter); +$server + ->shutdown() + ->inject('pools') + ->action(function (Group $pools) { + $pools->reclaim(); + }); + $server ->error() ->inject('error') diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 1c05a6ca1e..cd33ed0b79 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -307,30 +307,29 @@ class DeletesV1 extends Worker { $userId = $document->getId(); + $dbForProject = $this->getProjectDB($project); + // Delete all sessions of this user from the sessions table and update the sessions field of the user record $this->deleteByGroup('sessions', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($project)); + ], $dbForProject); - $this->getProjectDB($project)->deleteCachedDocument('users', $userId); + $dbForProject->deleteCachedDocument('users', $userId); // Delete Memberships and decrement team membership counts $this->deleteByGroup('memberships', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($project), function (Document $document) use ($project) { - + ], $dbForProject, function (Document $document) use ($dbForProject) { if ($document->getAttribute('confirm')) { // Count only confirmed members $teamId = $document->getAttribute('teamId'); - $team = $this->getProjectDB($project)->getDocument('teams', $teamId); + $team = $dbForProject->getDocument('teams', $teamId); if (!$team->isEmpty()) { - $team = $this - ->getProjectDB($project) - ->updateDocument( - 'teams', - $teamId, - // Ensure that total >= 0 + $team = $dbForProject->updateDocument( + 'teams', + $teamId, + // Ensure that total >= 0 $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0)) - ); + ); } } }); @@ -338,7 +337,7 @@ class DeletesV1 extends Worker // Delete tokens $this->deleteByGroup('tokens', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($project)); + ], $dbForProject); } /** diff --git a/app/workers/messaging.php b/app/workers/messaging.php index 4c82a1abff..5732c8c00b 100644 --- a/app/workers/messaging.php +++ b/app/workers/messaging.php @@ -1,16 +1,17 @@ args['recipient']; - $message = $this->args['message']; + $message = new SMS( + to: [$this->args['recipient']], + content: $this->args['message'], + from: $this->from, + ); try { - $this->sms->send($this->from, $recipient, $message); + $this->sms->send($message); } catch (\Exception $error) { throw new Exception('Error sending message: ' . $error->getMessage(), 500); } diff --git a/composer.json b/composer.json index 2d61a9dcca..1d6d1be25b 100644 --- a/composer.json +++ b/composer.json @@ -50,7 +50,7 @@ "utopia-php/cli": "0.14.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.28.*", - "utopia-php/queue": "0.4.*", + "utopia-php/queue": "0.5.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.3.*", "utopia-php/pools": "0.4.*", diff --git a/composer.lock b/composer.lock index b8fc10b161..f47d853d6b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fc65b26befaa50feffc51053f5064e1a", + "content-hash": "4893e1c13630239fe6a20d1c652eb484", "packages": [ { "name": "adhocore/jwt", @@ -2460,16 +2460,16 @@ }, { "name": "utopia-php/queue", - "version": "0.4.1", + "version": "0.5.1", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e" + "reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/0b69ede484a04c567cbb202f592d8e5e3cd2433e", - "reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/4b39e1f232b2e619b0d7fb4004f2356df334ddc1", + "reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1", "shasum": "" }, "require": { @@ -2515,9 +2515,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.4.1" + "source": "https://github.com/utopia-php/queue/tree/0.5.1" }, - "time": "2022-11-15T16:56:37+00:00" + "time": "2022-11-16T19:47:26+00:00" }, { "name": "utopia-php/registry", @@ -3555,16 +3555,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.18", + "version": "9.2.19", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a" + "reference": "c77b56b63e3d2031bd8997fcec43c1925ae46559" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/12fddc491826940cf9b7e88ad9664cf51f0f6d0a", - "reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/c77b56b63e3d2031bd8997fcec43c1925ae46559", + "reference": "c77b56b63e3d2031bd8997fcec43c1925ae46559", "shasum": "" }, "require": { @@ -3620,7 +3620,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.18" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.19" }, "funding": [ { @@ -3628,7 +3628,7 @@ "type": "github" } ], - "time": "2022-10-27T13:35:33+00:00" + "time": "2022-11-18T07:47:47+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5403,5 +5403,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.1.0" } diff --git a/docker-compose.yml b/docker-compose.yml index 76813e76e0..4ca8876f9b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -109,6 +109,7 @@ services: - _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_EMAILS - _APP_CONSOLE_WHITELIST_IPS + - _APP_CONSOLE_INVITES - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS @@ -390,6 +391,7 @@ services: - _APP_CONNECTIONS_DB_PROJECT - _APP_CONNECTIONS_CACHE - _APP_CONNECTIONS_QUEUE + - _APP_CONNECTIONS_STORAGE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 22940ad08e..5f8b4c80c6 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -2,9 +2,6 @@ namespace Appwrite\Event; -use DateTime; -use Resque; -use ResqueScheduler; use Utopia\Database\Document; use Utopia\Queue\Client; use Utopia\Queue\Connection; diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 479855504d..e96c24d755 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -171,6 +171,7 @@ abstract class Worker * @param Document $project * @return Database */ + protected static $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools protected function getProjectDB(Document $project): Database { global $register; @@ -181,6 +182,14 @@ abstract class Worker return $this->getConsoleDB(); } + $databaseName = $project->getAttribute('database'); + + if (isset(self::$databases[$databaseName])) { + $database = self::$databases[$databaseName]; + $database->setNamespace('_' . $project->getInternalId()); + return $database; + } + $dbAdapter = $pools ->get($project->getAttribute('database')) ->pop() @@ -188,6 +197,9 @@ abstract class Worker ; $database = new Database($dbAdapter, $this->getCache()); + + self::$databases[$databaseName] = $database; + $database->setNamespace('_' . $project->getInternalId()); return $database; diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 2ef3d77cac..6f4fb94a7d 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -494,6 +494,7 @@ class TimeSeries extends Calculator if (empty($point['projectInternalId'] ?? null)) { continue; } + $this->createOrUpdateMetric( $point['projectInternalId'], $point['time'], diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index cbe90c91fb..4dddaf1535 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -93,13 +93,15 @@ class UsageTest extends Scope $requestsCount = $data['requestsCount']; // console request - $cheaders = [ + $headers = [ 'origin' => 'http://localhost', 'x-appwrite-project' => 'console', 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + 'x-appwrite-project' => $projectId, + 'x-appwrite-mode' => 'admin', ]; - $res = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId . '/usage?range=30d', $cheaders); + $res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers); $res = $res['body']; $this->assertEquals(9, count($res)); @@ -110,7 +112,7 @@ class UsageTest extends Scope $this->assertEquals($requestsCount, $res['requests'][array_key_last($res['requests'])]['value']); $this->validateDates($res['requests']); - $res = $this->client->call(Client::METHOD_GET, '/users/usage?range=30d', array_merge($cheaders, [ + $res = $this->client->call(Client::METHOD_GET, '/users/usage?range=30d', array_merge($headers, [ 'x-appwrite-project' => $projectId, 'x-appwrite-mode' => 'admin' ])); @@ -146,10 +148,18 @@ class UsageTest extends Scope for ($i = 0; $i < 10; $i++) { $name = uniqid() . ' bucket'; - $res = $this->client->call(Client::METHOD_POST, '/storage/buckets', $headers, [ + $res = $this->client->call(Client::METHOD_POST, '/storage/buckets', \array_merge($headers, [ + 'content-type' => 'application/json' + ]), [ 'bucketId' => 'unique()', 'name' => $name, - 'permission' => 'bucket' + 'fileSecurity' => false, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], ]); $this->assertEquals($name, $res['body']['name']); $this->assertNotEmpty($res['body']['$id']); @@ -263,9 +273,11 @@ class UsageTest extends Scope 'origin' => 'http://localhost', 'x-appwrite-project' => 'console', 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + 'x-appwrite-project' => $projectId, + 'x-appwrite-mode' => 'admin', ]; - $res = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId . '/usage?range=30d', $headers); + $res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers); $res = $res['body']; $this->assertEquals(9, count($res)); @@ -503,8 +515,11 @@ class UsageTest extends Scope 'origin' => 'http://localhost', 'x-appwrite-project' => 'console', 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + 'x-appwrite-project' => $projectId, + 'x-appwrite-mode' => 'admin', ]; - $res = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId . '/usage?range=30d', $headers); + + $res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers); $res = $res['body']; $this->assertEquals(9, count($res)); diff --git a/tests/e2e/Services/Teams/TeamsBaseServer.php b/tests/e2e/Services/Teams/TeamsBaseServer.php index df508e5e37..c4d92ff0d9 100644 --- a/tests/e2e/Services/Teams/TeamsBaseServer.php +++ b/tests/e2e/Services/Teams/TeamsBaseServer.php @@ -246,7 +246,6 @@ trait TeamsBaseServer 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); - $this->assertEquals(200, $response['headers']['status-code']); $this->assertNotEmpty($response['body']['$id']); $this->assertEquals('Arsenal', $response['body']['name']);