Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4697 from appwrite/fix-init-retry

Browse Source 
Fix: Failing errors
This commit is contained in:
Christy Jacob
2022-11-18 19:53:43 +05:30
committed by GitHub
parent 1057ab4e42 91de1cbcf3
commit 948936dcff
18 changed files with 143 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+2 -1
View File
@@ -4,6 +4,7 @@ _APP_WORKER_PER_CORE=6
_APP_CONSOLE_WHITELIST_ROOT=disabled
_APP_CONSOLE_WHITELIST_EMAILS=
_APP_CONSOLE_WHITELIST_IPS=
_APP_CONSOLE_INVITES=enabled
_APP_SYSTEM_EMAIL_NAME=Appwrite
_APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io
_APP_SYSTEM_SECURITY_EMAIL_ADDRESS=security@appwrite.io
@@ -62,7 +63,7 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600
_APP_MAINTENANCE_RETENTION_ABUSE=86400
_APP_MAINTENANCE_RETENTION_AUDIT=1209600
_APP_MAINTENANCE_RETENTION_SCHEDULES=86400
_APP_USAGE_TIMESERIES_INTERVAL=60
_APP_USAGE_TIMESERIES_INTERVAL=20
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
_APP_USAGE_STATS=enabled
_APP_LOGGING_PROVIDER=
app/cli.php
+25 -16
View File
@@ -67,7 +67,9 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
$databaseName = $project->getAttribute('database');
if (isset($databases[$databaseName])) {
return $databases[$databaseName];
$database = $databases[$databaseName];
$database->setNamespace('_' . $project->getInternalId());
return $database;
}
$dbAdapter = $pools
@@ -76,10 +78,11 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
->getResource();
$database = new Database($dbAdapter, $cache);
$database->setNamespace('_' . $project->getInternalId());
$databases[$databaseName] = $database;
$database->setNamespace('_' . $project->getInternalId());
return $database;
};
@@ -176,25 +179,31 @@ $cli
do {
$attempts++;
// Prepare database connection
$dbAdapter = $pools
->get('console')
->pop()
->getResource();
try {
$pools->get('console')->reclaim();
$dbForConsole = new Database($dbAdapter, $cache);
$dbForConsole->setNamespace('console');
// Prepare database connection
$dbAdapter = $pools
->get('console')
->pop()
->getResource();
// Ensure tables exist
$collections = Config::getParam('collections', []);
$last = \array_key_last($collections);
$dbForConsole = new Database($dbAdapter, $cache);
$dbForConsole->setNamespace('console');
// Ensure tables exist
$collections = Config::getParam('collections', []);
$last = \array_key_last($collections);
if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) {
throw new Exception('Tables not ready yet.');
}
if ($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last)) {
$ready = true;
break;
} catch (\Exception $err) {
Console::warning($err->getMessage());
sleep($sleep);
}
sleep($sleep);
} while ($attempts < $maxAttempts);
if (!$ready) {
app/config/variables.php
+10 -1
View File
@@ -203,7 +203,16 @@ return [
'required' => false,
'question' => '',
'filter' => ''
]
],
[
'name' => '_APP_CONSOLE_INVITES',
'description' => 'This option allows you to disable the invitation of new users to the Appwrite console. When enabled, console users are allowed to invite new users to a project. By default this option is enabled.',
'introduction' => '1.2.0',
'default' => 'enabled',
'required' => false,
'question' => '',
'filter' => ''
],
],
],
[
app/controllers/api/account.php
+1
View File
@@ -279,6 +279,7 @@ App::get('/v1/account/sessions/oauth2/:provider')
->action(function (string $provider, string $success, string $failure, array $scopes, Request $request, Response $response, Document $project) use ($oauthDefaultSuccess, $oauthDefaultFailure) {
$protocol = $request->getProtocol();
$callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId();
$appId = $project->getAttribute('authProviders', [])[$provider . 'Appid'] ?? '';
$appSecret = $project->getAttribute('authProviders', [])[$provider . 'Secret'] ?? '{}';
app/controllers/api/health.php
+6 -3
View File
@@ -8,6 +8,8 @@ use Utopia\App;
use Utopia\Config\Config;
use Utopia\Database\Document;
use Utopia\Pools\Group;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
use Utopia\Registry\Registry;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
@@ -396,10 +398,11 @@ App::get('/v1/health/queue/functions')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->inject('queue')
->inject('response')
->action(function (Response $response) {
$response->dynamic(new Document([ 'size' => Resque::size(Event::FUNCTIONS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE);
->action(function (Connection $queue, Response $response) {
$client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue);
$response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/storage/local')
app/init.php
+7 -3
View File
@@ -75,6 +75,7 @@ use PHPMailer\PHPMailer\PHPMailer;
use Swoole\Database\PDOProxy;
use Utopia\CLI\Console;
use Utopia\Queue;
use Utopia\Queue\Connection;
use Utopia\Storage\Storage;
const APP_NAME = 'Appwrite';
@@ -847,9 +848,12 @@ App::setResource('mails', fn() => new Mail());
App::setResource('deletes', fn() => new Delete());
App::setResource('database', fn() => new EventDatabase());
App::setResource('messaging', fn() => new Phone());
App::setResource('queueForFunctions', function (Group $pools) {
return new Func($pools->get('queue')->pop()->getResource());
App::setResource('queue', function (Group $pools) {
return $pools->get('queue')->pop()->getResource();
}, ['pools']);
App::setResource('queueForFunctions', function (Connection $queue) {
return new Func($queue);
}, ['queue']);
App::setResource('usage', function ($register) {
return new Stats($register->get('statsd'));
}, ['register']);
@@ -1024,7 +1028,7 @@ App::setResource('console', function () {
'legalAddress' => '',
'legalTaxId' => '',
'auths' => [
'invites' => false,
'invites' => App::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled',
'limit' => (App::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user
'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds
],
app/views/install/compose.phtml
+1
View File
@@ -88,6 +88,7 @@ services:
- _APP_CONSOLE_WHITELIST_ROOT
- _APP_CONSOLE_WHITELIST_EMAILS
- _APP_CONSOLE_WHITELIST_IPS
- _APP_CONSOLE_INVITES
- _APP_SYSTEM_EMAIL_NAME
- _APP_SYSTEM_EMAIL_ADDRESS
- _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
app/worker.php
+12 -1
View File
@@ -17,6 +17,7 @@ use Utopia\Queue\Server;
use Utopia\Registry\Registry;
use Utopia\Logger\Log;
use Utopia\Logger\Logger;
use Utopia\Pools\Group;
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
@@ -55,7 +56,6 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
$adapter = new Database($database, $cache);
$adapter->setNamespace('_' . $project->getInternalId());
return $adapter;
}, ['cache', 'register', 'message', 'dbForConsole']);
@@ -93,6 +93,10 @@ Server::setResource('statsd', function ($register) {
return $register->get('statsd');
}, ['register']);
Server::setResource('pools', function ($register) {
return $register->get('pools');
}, ['register']);
$pools = $register->get('pools');
$connection = $pools->get('queue')->pop()->getResource();
$workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
@@ -104,6 +108,13 @@ if (empty(App::getEnv('QUEUE'))) {
$adapter = new Swoole($connection, $workerNumber, App::getEnv('QUEUE'));
$server = new Server($adapter);
$server
->shutdown()
->inject('pools')
->action(function (Group $pools) {
$pools->reclaim();
});
$server
->error()
->inject('error')
app/workers/deletes.php
+12 -13
View File
@@ -307,30 +307,29 @@ class DeletesV1 extends Worker
{
$userId = $document->getId();
$dbForProject = $this->getProjectDB($project);
// Delete all sessions of this user from the sessions table and update the sessions field of the user record
$this->deleteByGroup('sessions', [
Query::equal('userId', [$userId])
], $this->getProjectDB($project));
], $dbForProject);
$this->getProjectDB($project)->deleteCachedDocument('users', $userId);
$dbForProject->deleteCachedDocument('users', $userId);
// Delete Memberships and decrement team membership counts
$this->deleteByGroup('memberships', [
Query::equal('userId', [$userId])
], $this->getProjectDB($project), function (Document $document) use ($project) {
], $dbForProject, function (Document $document) use ($dbForProject) {
if ($document->getAttribute('confirm')) { // Count only confirmed members
$teamId = $document->getAttribute('teamId');
$team = $this->getProjectDB($project)->getDocument('teams', $teamId);
$team = $dbForProject->getDocument('teams', $teamId);
if (!$team->isEmpty()) {
$team = $this
->getProjectDB($project)
->updateDocument(
'teams',
$teamId,
// Ensure that total >= 0
$team = $dbForProject->updateDocument(
'teams',
$teamId,
// Ensure that total >= 0
$team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
);
);
}
}
});
@@ -338,7 +337,7 @@ class DeletesV1 extends Worker
// Delete tokens
$this->deleteByGroup('tokens', [
Query::equal('userId', [$userId])
], $this->getProjectDB($project));
], $dbForProject);
}
/**
app/workers/messaging.php
+15 -11
View File
@@ -1,16 +1,17 @@
<?php
use Appwrite\SMS\Adapter\Mock;
use Appwrite\SMS\Adapter\Telesign;
use Appwrite\SMS\Adapter\TextMagic;
use Appwrite\SMS\Adapter\Twilio;
use Appwrite\SMS\Adapter\Msg91;
use Appwrite\SMS\Adapter\Vonage;
use Utopia\DSN\DSN;
use Appwrite\Resque\Worker;
use Appwrite\SMS\Adapter;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\DSN\DSN;
use Utopia\Messaging\Adapter;
use Utopia\Messaging\Adapters\SMS\Mock;
use Utopia\Messaging\Adapters\SMS\Msg91;
use Utopia\Messaging\Adapters\SMS\Telesign;
use Utopia\Messaging\Adapters\SMS\TextMagic;
use Utopia\Messaging\Adapters\SMS\Twilio;
use Utopia\Messaging\Adapters\SMS\Vonage;
use Utopia\Messaging\Messages\SMS;
require_once __DIR__ . '/../init.php';
@@ -58,11 +59,14 @@ class MessagingV1 extends Worker
return;
}
$recipient = $this->args['recipient'];
$message = $this->args['message'];
$message = new SMS(
to: [$this->args['recipient']],
content: $this->args['message'],
from: $this->from,
);
try {
$this->sms->send($this->from, $recipient, $message);
$this->sms->send($message);
} catch (\Exception $error) {
throw new Exception('Error sending message: ' . $error->getMessage(), 500);
}
composer.json
+1 -1
View File
@@ -50,7 +50,7 @@
"utopia-php/cli": "0.14.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.28.*",
"utopia-php/queue": "0.4.*",
"utopia-php/queue": "0.5.*",
"utopia-php/orchestration": "0.9.*",
"utopia-php/platform": "0.3.*",
"utopia-php/pools": "0.4.*",
composer.lock
Generated
+14 -14
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "fc65b26befaa50feffc51053f5064e1a",
"content-hash": "4893e1c13630239fe6a20d1c652eb484",
"packages": [
{
"name": "adhocore/jwt",
@@ -2460,16 +2460,16 @@
},
{
"name": "utopia-php/queue",
"version": "0.4.1",
"version": "0.5.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/queue.git",
"reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e"
"reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/0b69ede484a04c567cbb202f592d8e5e3cd2433e",
"reference": "0b69ede484a04c567cbb202f592d8e5e3cd2433e",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/4b39e1f232b2e619b0d7fb4004f2356df334ddc1",
"reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1",
"shasum": ""
},
"require": {
@@ -2515,9 +2515,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/queue/issues",
"source": "https://github.com/utopia-php/queue/tree/0.4.1"
"source": "https://github.com/utopia-php/queue/tree/0.5.1"
},
"time": "2022-11-15T16:56:37+00:00"
"time": "2022-11-16T19:47:26+00:00"
},
{
"name": "utopia-php/registry",
@@ -3555,16 +3555,16 @@
},
{
"name": "phpunit/php-code-coverage",
"version": "9.2.18",
"version": "9.2.19",
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/php-code-coverage.git",
"reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a"
"reference": "c77b56b63e3d2031bd8997fcec43c1925ae46559"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/12fddc491826940cf9b7e88ad9664cf51f0f6d0a",
"reference": "12fddc491826940cf9b7e88ad9664cf51f0f6d0a",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/c77b56b63e3d2031bd8997fcec43c1925ae46559",
"reference": "c77b56b63e3d2031bd8997fcec43c1925ae46559",
"shasum": ""
},
"require": {
@@ -3620,7 +3620,7 @@
],
"support": {
"issues": "https://github.com/sebastianbergmann/php-code-coverage/issues",
"source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.18"
"source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.19"
},
"funding": [
{
@@ -3628,7 +3628,7 @@
"type": "github"
}
],
"time": "2022-10-27T13:35:33+00:00"
"time": "2022-11-18T07:47:47+00:00"
},
{
"name": "phpunit/php-file-iterator",
@@ -5403,5 +5403,5 @@
"platform-overrides": {
"php": "8.0"
},
"plugin-api-version": "2.3.0"
"plugin-api-version": "2.1.0"
}
docker-compose.yml
+2
View File
@@ -109,6 +109,7 @@ services:
- _APP_CONSOLE_WHITELIST_ROOT
- _APP_CONSOLE_WHITELIST_EMAILS
- _APP_CONSOLE_WHITELIST_IPS
- _APP_CONSOLE_INVITES
- _APP_SYSTEM_EMAIL_NAME
- _APP_SYSTEM_EMAIL_ADDRESS
- _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
@@ -390,6 +391,7 @@ services:
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_CONNECTIONS_STORAGE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
src/Appwrite/Event/Func.php
-3
View File
@@ -2,9 +2,6 @@
namespace Appwrite\Event;
use DateTime;
use Resque;
use ResqueScheduler;
use Utopia\Database\Document;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
src/Appwrite/Resque/Worker.php
+12
View File
@@ -171,6 +171,7 @@ abstract class Worker
* @param Document $project
* @return Database
*/
protected static $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
protected function getProjectDB(Document $project): Database
{
global $register;
@@ -181,6 +182,14 @@ abstract class Worker
return $this->getConsoleDB();
}
$databaseName = $project->getAttribute('database');
if (isset(self::$databases[$databaseName])) {
$database = self::$databases[$databaseName];
$database->setNamespace('_' . $project->getInternalId());
return $database;
}
$dbAdapter = $pools
->get($project->getAttribute('database'))
->pop()
@@ -188,6 +197,9 @@ abstract class Worker
;
$database = new Database($dbAdapter, $this->getCache());
self::$databases[$databaseName] = $database;
$database->setNamespace('_' . $project->getInternalId());
return $database;
src/Appwrite/Usage/Calculators/TimeSeries.php
+1
View File
@@ -494,6 +494,7 @@ class TimeSeries extends Calculator
if (empty($point['projectInternalId'] ?? null)) {
continue;
}
$this->createOrUpdateMetric(
$point['projectInternalId'],
$point['time'],
tests/e2e/General/UsageTest.php
+22 -7
View File
@@ -93,13 +93,15 @@ class UsageTest extends Scope
$requestsCount = $data['requestsCount'];
// console request
$cheaders = [
$headers = [
'origin' => 'http://localhost',
'x-appwrite-project' => 'console',
'cookie' => 'a_session_console=' . $this->getRoot()['session'],
'x-appwrite-project' => $projectId,
'x-appwrite-mode' => 'admin',
];
$res = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId . '/usage?range=30d', $cheaders);
$res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers);
$res = $res['body'];
$this->assertEquals(9, count($res));
@@ -110,7 +112,7 @@ class UsageTest extends Scope
$this->assertEquals($requestsCount, $res['requests'][array_key_last($res['requests'])]['value']);
$this->validateDates($res['requests']);
$res = $this->client->call(Client::METHOD_GET, '/users/usage?range=30d', array_merge($cheaders, [
$res = $this->client->call(Client::METHOD_GET, '/users/usage?range=30d', array_merge($headers, [
'x-appwrite-project' => $projectId,
'x-appwrite-mode' => 'admin'
]));
@@ -146,10 +148,18 @@ class UsageTest extends Scope
for ($i = 0; $i < 10; $i++) {
$name = uniqid() . ' bucket';
$res = $this->client->call(Client::METHOD_POST, '/storage/buckets', $headers, [
$res = $this->client->call(Client::METHOD_POST, '/storage/buckets', \array_merge($headers, [
'content-type' => 'application/json'
]), [
'bucketId' => 'unique()',
'name' => $name,
'permission' => 'bucket'
'fileSecurity' => false,
'permissions' => [
Permission::read(Role::any()),
Permission::create(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
]);
$this->assertEquals($name, $res['body']['name']);
$this->assertNotEmpty($res['body']['$id']);
@@ -263,9 +273,11 @@ class UsageTest extends Scope
'origin' => 'http://localhost',
'x-appwrite-project' => 'console',
'cookie' => 'a_session_console=' . $this->getRoot()['session'],
'x-appwrite-project' => $projectId,
'x-appwrite-mode' => 'admin',
];
$res = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId . '/usage?range=30d', $headers);
$res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers);
$res = $res['body'];
$this->assertEquals(9, count($res));
@@ -503,8 +515,11 @@ class UsageTest extends Scope
'origin' => 'http://localhost',
'x-appwrite-project' => 'console',
'cookie' => 'a_session_console=' . $this->getRoot()['session'],
'x-appwrite-project' => $projectId,
'x-appwrite-mode' => 'admin',
];
$res = $this->client->call(Client::METHOD_GET, '/projects/' . $projectId . '/usage?range=30d', $headers);
$res = $this->client->call(Client::METHOD_GET, '/project/usage?range=30d', $headers);
$res = $res['body'];
$this->assertEquals(9, count($res));
tests/e2e/Services/Teams/TeamsBaseServer.php
-1
View File
@@ -246,7 +246,6 @@ trait TeamsBaseServer
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals('Arsenal', $response['body']['name']);