From 9fb65250632897746b60aaaa331390c5402fc396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 22 Nov 2023 14:50:57 +0100 Subject: [PATCH 01/34] Add new tags from worker, fix queue name --- app/worker.php | 16 +++++------ src/Appwrite/Platform/Workers/Builds.php | 25 ++++++++++++----- .../Platform/Workers/Certificates.php | 9 +++++-- src/Appwrite/Platform/Workers/Databases.php | 12 +++++++-- src/Appwrite/Platform/Workers/Deletes.php | 10 +++++-- src/Appwrite/Platform/Workers/Functions.php | 27 ++++++++++++------- src/Appwrite/Platform/Workers/Mails.php | 9 +++++-- src/Appwrite/Platform/Workers/Messaging.php | 9 +++++-- src/Appwrite/Platform/Workers/Migrations.php | 16 ++++++++--- src/Appwrite/Platform/Workers/Webhooks.php | 9 +++++-- 10 files changed, 102 insertions(+), 40 deletions(-) diff --git a/app/worker.php b/app/worker.php index 32a8b9804e..aa51bc9ab9 100644 --- a/app/worker.php +++ b/app/worker.php @@ -199,6 +199,12 @@ if (!empty($workerIndex)) { $workerName .= '_' . $workerIndex; } +if (\str_starts_with($workerName, 'databases')) { + $queueName = App::getEnv('_APP_QUEUE_NAME', 'database_db_main'); +} else { + $queueName = App::getEnv('_APP_QUEUE_NAME', 'v1-' . strtolower($workerName)); +} + try { /** * Any worker can be configured with the following env vars: @@ -206,12 +212,6 @@ try { * - _APP_WORKER_PER_CORE The number of worker processes per core (ignored if _APP_WORKERS_NUM is set) * - _APP_QUEUE_NAME The name of the queue to read for database events */ - if ($workerName === 'databases') { - $queueName = App::getEnv('_APP_QUEUE_NAME', 'database_db_main'); - } else { - $queueName = App::getEnv('_APP_QUEUE_NAME', 'v1-' . strtolower($workerName)); - } - $platform->init(Service::TYPE_WORKER, [ 'workersNum' => App::getEnv('_APP_WORKERS_NUM', 1), 'connection' => $pools->get('queue')->pop()->getResource(), @@ -237,7 +237,7 @@ $worker ->inject('error') ->inject('logger') ->inject('log') - ->action(function (Throwable $error, ?Logger $logger, Log $log) { + ->action(function (Throwable $error, ?Logger $logger, Log $log) use ($queueName) { $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); if ($error instanceof PDOException) { @@ -250,7 +250,7 @@ $worker $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); - $log->setAction('appwrite-queue-' . App::getEnv('QUEUE')); + $log->setAction('appwrite-queue-' . $queueName); $log->addTag('verboseType', get_class($error)); $log->addTag('code', $error->getCode()); $log->addExtra('file', $error->getFile()); diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index f1ef0df602..d2b5b9f48b 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -24,6 +24,7 @@ use Utopia\Database\Exception\Restricted; use Utopia\Database\Validator\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; +use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Storage\Device\Local; @@ -52,7 +53,8 @@ class Builds extends Action ->inject('cache') ->inject('dbForProject') ->inject('getFunctionsDevice') - ->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $getFunctionsDevice)); + ->inject('log') + ->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice, Log $log) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $getFunctionsDevice, $log)); } /** @@ -64,10 +66,11 @@ class Builds extends Action * @param Cache $cache * @param Database $dbForProject * @param callable $getFunctionsDevice + * @param Log $log * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice): void + public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice, Log $log): void { $payload = $message->getPayload() ?? []; @@ -81,12 +84,15 @@ class Builds extends Action $deployment = new Document($payload['deployment'] ?? []); $template = new Document($payload['template'] ?? []); + $log->addTag('projectId', $project->getId()); + $log->addTag('type', $type); + switch ($type) { case BUILD_TYPE_DEPLOYMENT: case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $usage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template); + $this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $usage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template, $log); break; default: @@ -106,20 +112,27 @@ class Builds extends Action * @param Document $function * @param Document $deployment * @param Document $template + * @param Log $log * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template): void + protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void { $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); - $function = $dbForProject->getDocument('functions', $function->getId()); + $functionId = $function->getId(); + $log->addTag('functionId', $function->getId()); + + $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { throw new Exception('Function not found', 404); } - $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); + $deploymentId = $deployment->getId(); + $log->addTag('deploymentId', $deploymentId); + + $deployment = $dbForProject->getDocument('deployments', $deploymentId); if ($deployment->isEmpty()) { throw new Exception('Deployment not found', 404); } diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index ade2125e4a..0721654995 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -23,6 +23,7 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Domains\Domain; use Utopia\Locale\Locale; +use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; @@ -45,7 +46,8 @@ class Certificates extends Action ->inject('queueForMails') ->inject('queueForEvents') ->inject('queueForFunctions') - ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions)); + ->inject('log') + ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log)); } /** @@ -54,11 +56,12 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions + * @param Log $log * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions): void + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void { $payload = $message->getPayload() ?? []; @@ -70,6 +73,8 @@ class Certificates extends Action $domain = new Domain($document->getAttribute('domain', '')); $skipRenewCheck = $payload['skipRenewCheck'] ?? false; + $log->addTag('domain', $domain->get()); + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $skipRenewCheck); } diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index e0ec75e1d4..c397d744d3 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -16,6 +16,7 @@ use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Query; +use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; @@ -36,17 +37,19 @@ class Databases extends Action ->inject('message') ->inject('dbForConsole') ->inject('dbForProject') - ->callback(fn($message, $dbForConsole, $dbForProject) => $this->action($message, $dbForConsole, $dbForProject)); + ->inject('log') + ->callback(fn(Message $message, Database $dbForConsole, Database $dbForProject, Log $log) => $this->action($message, $dbForConsole, $dbForProject, $log)); } /** * @param Message $message * @param Database $dbForConsole * @param Database $dbForProject + * @param Log $log * @return void * @throws \Exception */ - public function action(Message $message, Database $dbForConsole, Database $dbForProject): void + public function action(Message $message, Database $dbForConsole, Database $dbForProject, Log $log): void { $payload = $message->getPayload() ?? []; @@ -60,10 +63,15 @@ class Databases extends Action $document = new Document($payload['document'] ?? []); $database = new Document($payload['database'] ?? []); + $log->addTag('projectId', $project->getId()); + $log->addTag('type', $type); + if ($database->isEmpty()) { throw new Exception('Missing database'); } + $log->addTag('databaseId', $database->getId()); + match (strval($type)) { DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject), DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject), diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index b95a13a12e..c4793c2497 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Workers; +use Appwrite\Auth\Auth; use Executor\Executor; use Throwable; use Utopia\Abuse\Abuse; @@ -20,6 +21,7 @@ use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Query; +use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Storage\Device; @@ -45,14 +47,15 @@ class Deletes extends Action ->inject('getFunctionsDevice') ->inject('getBuildsDevice') ->inject('getCacheDevice') - ->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice)); + ->inject('log') + ->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $log)); } /** * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Log $log): void { $payload = $message->getPayload() ?? []; @@ -67,6 +70,9 @@ class Deletes extends Action $document = new Document($payload['document'] ?? []); $project = new Document($payload['project'] ?? []); + $log->addTag('projectId', $project->getId()); + $log->addTag('type', $type); + switch (strval($type)) { case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index 7a6cfe115f..131834aadf 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -90,6 +90,10 @@ class Functions extends Action return; } + $log->addTag('functionId', $function->getId()); + $log->addTag('projectId', $project->getId()); + $log->addTag('type', $type); + if (!empty($events)) { $limit = 30; $sum = 30; @@ -236,15 +240,14 @@ class Functions extends Action string $eventData = null, string $executionId = null, ): void { - $user ??= new Document(); - $functionId = $function->getId(); - $deploymentId = $function->getAttribute('deployment', ''); + $user ??= new Document(); + $functionId = $function->getId(); + $deploymentId = $function->getAttribute('deployment', ''); - $log->addTag('functionId', $functionId); - $log->addTag('projectId', $project->getId()); + $log->addTag('deploymentId', $deploymentId); - /** Check if deployment exists */ - $deployment = $dbForProject->getDocument('deployments', $deploymentId); + /** Check if deployment exists */ + $deployment = $dbForProject->getDocument('deployments', $deploymentId); if ($deployment->getAttribute('resourceId') !== $functionId) { throw new Exception('Deployment not found. Create deployment before trying to execute a function'); @@ -254,8 +257,12 @@ class Functions extends Action throw new Exception('Deployment not found. Create deployment before trying to execute a function'); } - /** Check if build has exists */ - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + $buildId = $deployment->getAttribute('buildId', ''); + + $log->addTag('buildId', $buildId); + + /** Check if build has exists */ + $build = $dbForProject->getDocument('builds', $buildId); if ($build->isEmpty()) { throw new Exception('Build not found'); } @@ -264,7 +271,7 @@ class Functions extends Action throw new Exception('Build not ready'); } - /** Check if runtime is supported */ + /** Check if runtime is supported */ $version = $function->getAttribute('version', 'v2'); $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); diff --git a/src/Appwrite/Platform/Workers/Mails.php b/src/Appwrite/Platform/Workers/Mails.php index 94a2a46087..ea9b77a8c4 100644 --- a/src/Appwrite/Platform/Workers/Mails.php +++ b/src/Appwrite/Platform/Workers/Mails.php @@ -8,6 +8,7 @@ use PHPMailer\PHPMailer\PHPMailer; use Swoole\Runtime; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Registry\Registry; @@ -28,17 +29,19 @@ class Mails extends Action ->desc('Mails worker') ->inject('message') ->inject('register') - ->callback(fn($message, $register) => $this->action($message, $register)); + ->inject('log') + ->callback(fn(Message $message, Registry $register, Log $log) => $this->action($message, $register, $log)); } /** * @param Message $message * @param Registry $register + * @param Log $log * @throws \PHPMailer\PHPMailer\Exception * @return void * @throws Exception */ - public function action(Message $message, Registry $register): void + public function action(Message $message, Registry $register, Log $log): void { Runtime::setHookFlags(SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_TCP); $payload = $message->getPayload() ?? []; @@ -54,6 +57,8 @@ class Mails extends Action return; } + $log->addTag('type', empty($smtp) ? 'cloud' : 'smtp'); + $recipient = $payload['recipient']; $subject = $payload['subject']; $variables = $payload['variables']; diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 76b86e4f0c..874de84137 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -6,6 +6,7 @@ use Exception; use Utopia\App; use Utopia\CLI\Console; use Utopia\DSN\DSN; +use Utopia\Logger\Log; use Utopia\Messaging\Messages\Sms; use Utopia\Messaging\Adapters\SMS\Mock; use Utopia\Messaging\Adapters\SMS\Msg91; @@ -43,15 +44,17 @@ class Messaging extends Action $this ->desc('Messaging worker') ->inject('message') - ->callback(fn($message) => $this->action($message)); + ->inject('log') + ->callback(fn(Message $message, Log $log) => $this->action($message, $log)); } /** * @param Message $message + * @param Log $log * @return void * @throws Exception */ - public function action(Message $message): void + public function action(Message $message, Log $log): void { $payload = $message->getPayload() ?? []; @@ -70,6 +73,8 @@ class Messaging extends Action return; } + $log->addTag('type', $this->dsn->getHost()); + $sms = match ($this->dsn->getHost()) { 'mock' => new Mock($this->user, $this->secret), // used for tests 'twilio' => new Twilio($this->user, $this->secret), diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 31b0df59a3..95c31d07ad 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -17,6 +17,7 @@ use Appwrite\Role; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Helpers\ID; +use Utopia\Logger\Log; use Utopia\Migration\Destinations\Appwrite as DestinationsAppwrite; use Utopia\Migration\Resource; use Utopia\Migration\Source; @@ -46,17 +47,19 @@ class Migrations extends Action ->inject('message') ->inject('dbForProject') ->inject('dbForConsole') - ->callback(fn(Message $message, Database $dbForProject, Database $dbForConsole) => $this->action($message, $dbForProject, $dbForConsole)); + ->inject('log') + ->callback(fn(Message $message, Database $dbForProject, Database $dbForConsole, Log $log) => $this->action($message, $dbForProject, $dbForConsole, $log)); } /** * @param Message $message * @param Database $dbForProject * @param Database $dbForConsole + * @param Log $log * @return void * @throws Exception */ - public function action(Message $message, Database $dbForProject, Database $dbForConsole): void + public function action(Message $message, Database $dbForProject, Database $dbForConsole, Log $log): void { $payload = $message->getPayload() ?? []; @@ -82,7 +85,9 @@ class Migrations extends Action return; } - $this->processMigration($project, $migration); + $log->addTag('projectId', $project->getId()); + + $this->processMigration($project, $migration, $log); } /** @@ -228,6 +233,7 @@ class Migrations extends Action /** * @param Document $project * @param Document $migration + * @param Log $log * @return void * @throws Authorization * @throws Conflict @@ -235,7 +241,7 @@ class Migrations extends Action * @throws Structure * @throws \Utopia\Database\Exception */ - protected function processMigration(Document $project, Document $migration): void + protected function processMigration(Document $project, Document $migration, Log $log): void { /** * @var Document $migrationDocument @@ -252,6 +258,8 @@ class Migrations extends Action $migrationDocument->setAttribute('status', 'processing'); $this->updateMigrationDocument($migrationDocument, $projectDocument); + $log->addTag('type', $migrationDocument->getAttribute('source')); + $source = $this->processSource($migrationDocument->getAttribute('source'), $migrationDocument->getAttribute('credentials')); $source->report(); diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index fa25145a13..28bc2a7317 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -5,6 +5,7 @@ namespace Appwrite\Platform\Workers; use Exception; use Utopia\App; use Utopia\Database\Document; +use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; @@ -25,15 +26,17 @@ class Webhooks extends Action $this ->desc('Webhooks worker') ->inject('message') - ->callback(fn ($message) => $this->action($message)); + ->inject('log') + ->callback(fn (Message $message, Log $log) => $this->action($message, $log)); } /** * @param Message $message + * @param Log $log * @return void * @throws Exception */ - public function action(Message $message): void + public function action(Message $message, Log $log): void { $payload = $message->getPayload() ?? []; @@ -46,6 +49,8 @@ class Webhooks extends Action $project = new Document($payload['project']); $user = new Document($payload['user'] ?? []); + $log->addTag('projectId', $project->getId()); + foreach ($project->getAttribute('webhooks', []) as $webhook) { if (array_intersect($webhook->getAttribute('events', []), $events)) { $this->execute($events, $webhookPayload, $webhook, $user, $project); From 9b2fbc5c52476f453fd62ba977254dacb1b9dcb4 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 29 Dec 2023 21:52:50 +0100 Subject: [PATCH 02/34] feat: console hostname env variable --- .env | 3 ++- app/config/variables.php | 17 +++++++++-------- app/init.php | 12 ++++++++++++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.env b/.env index ad551e705a..81145b563d 100644 --- a/.env +++ b/.env @@ -4,12 +4,13 @@ _APP_WORKER_PER_CORE=6 _APP_CONSOLE_WHITELIST_ROOT=disabled _APP_CONSOLE_WHITELIST_EMAILS= _APP_CONSOLE_WHITELIST_IPS= +_APP_CONSOLE_HOSTNAMES= _APP_SYSTEM_EMAIL_NAME=Appwrite _APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io _APP_SYSTEM_SECURITY_EMAIL_ADDRESS=security@appwrite.io _APP_SYSTEM_RESPONSE_FORMAT= _APP_OPTIONS_ABUSE=disabled -_APP_OPTIONS_ROUTER_PROTECTION=disbled +_APP_OPTIONS_ROUTER_PROTECTION=disabled _APP_OPTIONS_FORCE_HTTPS=disabled _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled _APP_OPENSSL_KEY_V1=your-secret-key diff --git a/app/config/variables.php b/app/config/variables.php index 9d555bf013..a88145601d 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -115,14 +115,6 @@ return [ 'question' => '', 'filter' => '' ], - // [ - // 'name' => '_APP_CONSOLE_WHITELIST_DOMAINS', - // 'description' => 'This option allows you to limit creation of users to Appwrite console for users sharing the same email domains. This option is very useful for team working with company emails domain.\n\nTo enable this option, pass a list of allowed email domains separated by a comma.', - // 'introduction' => '', - // 'default' => '', - // 'required' => false, - // 'question' => '', - // ], [ 'name' => '_APP_CONSOLE_WHITELIST_IPS', 'description' => "This last option allows you to limit creation of users in Appwrite console for users sharing the same set of IP addresses. This option is very useful for team working with a VPN service or a company IP.\n\nTo enable/activate this option, pass a list of allowed IP addresses separated by a comma.", @@ -132,6 +124,15 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_CONSOLE_HOSTNAMES', + 'description' => 'This option allows you to add additional hostnames to your Appwrite console. This option is very useful for allowing access to the console project from additional domains. To enable it, pass a list of allowed hostnames separated by a comma.', + 'introduction' => '', + 'default' => '', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_SYSTEM_EMAIL_NAME', 'description' => 'This is the sender name value that will appear on email messages sent to developers from the Appwrite console. The default value is: \'Appwrite\'. You can use url encoded strings for spaces and special chars.', diff --git a/app/init.php b/app/init.php index 72bf75d41b..ccd7b96573 100644 --- a/app/init.php +++ b/app/init.php @@ -946,6 +946,18 @@ App::setResource('clients', function ($request, $console, $project) { 'hostname' => $request->getHostname(), ], Document::SET_TYPE_APPEND); + $hostnames = explode(',', App::getEnv('_APP_CONSOLE_HOSTNAMES', '')); + if (is_array($hostnames)) { + foreach ($hostnames as $hostname) { + $console->setAttribute('platforms', [ + '$collection' => ID::custom('platforms'), + 'type' => Origin::CLIENT_TYPE_WEB, + 'name' => $hostname, + 'hostname' => $hostname, + ], Document::SET_TYPE_APPEND); + } + } + /** * Get All verified client URLs for both console and current projects * + Filter for duplicated entries From cc1272f80bb2e73c10d89b52e0142ba4c384b237 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 29 Dec 2023 22:09:59 +0100 Subject: [PATCH 03/34] feat: add new env variable to compsoe files --- app/views/install/compose.phtml | 5 +++-- docker-compose.yml | 17 +++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 898b46b3a5..2321dbf290 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -79,6 +79,7 @@ services: - _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_EMAILS - _APP_CONSOLE_WHITELIST_IPS + - _APP_CONSOLE_HOSTNAMES - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS @@ -273,7 +274,7 @@ services: depends_on: - redis - mariadb - volumes: + volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw - appwrite-functions:/storage/functions:rw @@ -420,7 +421,7 @@ services: depends_on: - redis - mariadb - volumes: + volumes: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw environment: diff --git a/docker-compose.yml b/docker-compose.yml index 97cf7e5136..fcec1b7732 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,7 +53,7 @@ services: DEBUG: false TESTING: true VERSION: dev - ports: + ports: - 9501:80 networks: - appwrite @@ -88,7 +88,7 @@ services: - mariadb - redis # - clamav - entrypoint: + entrypoint: - php - -e - app/http.php @@ -100,6 +100,7 @@ services: - _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_EMAILS - _APP_CONSOLE_WHITELIST_IPS + - _APP_CONSOLE_HOSTNAMES - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS @@ -195,7 +196,7 @@ services: container_name: appwrite-realtime image: appwrite-dev restart: unless-stopped - ports: + ports: - 9505:80 labels: - "traefik.enable=true" @@ -303,7 +304,7 @@ services: depends_on: - redis - mariadb - volumes: + volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw - appwrite-functions:/storage/functions:rw @@ -357,7 +358,7 @@ services: image: appwrite-dev networks: - appwrite - volumes: + volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src depends_on: @@ -456,7 +457,7 @@ services: depends_on: - redis - mariadb - volumes: + volumes: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw - ./app:/usr/src/code/app @@ -744,7 +745,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_MIXPANEL_TOKEN - + appwrite-hamster-scheduler: entrypoint: hamster <<: *x-logging @@ -995,7 +996,7 @@ services: # - './debug:/tmp' # ports: # - '3001:80' - + graphql-explorer: container_name: appwrite-graphql-explorer image: appwrite/altair:0.3.0 From 92c4c52a81f1d041d6dfd40599dd8308e6e1129b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 2 Jan 2024 13:02:11 +0000 Subject: [PATCH 04/34] QA changes --- app/worker.php | 2 +- src/Appwrite/Platform/Workers/Builds.php | 7 +------ src/Appwrite/Platform/Workers/Databases.php | 12 ++++++++---- src/Appwrite/Platform/Workers/Deletes.php | 4 ++-- src/Appwrite/Platform/Workers/Mails.php | 4 +--- 5 files changed, 13 insertions(+), 16 deletions(-) diff --git a/app/worker.php b/app/worker.php index 4de14c396a..7713f6c55a 100644 --- a/app/worker.php +++ b/app/worker.php @@ -247,7 +247,7 @@ $worker throw $error; } - if ($logger && ($error->getCode() >= 500 || $error->getCode() === 0)) { + if ($logger) { $log->setNamespace("appwrite-worker"); $log->setServer(\gethostname()); $log->setVersion($version); diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index d2b5b9f48b..bf4c75e696 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -4,7 +4,6 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Event; use Appwrite\Event\Func; -use Appwrite\Event\Usage; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Usage\Stats; use Appwrite\Utopia\Response\Model\Deployment; @@ -28,7 +27,6 @@ use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Storage\Device\Local; -use Utopia\Storage\Storage; use Utopia\VCS\Adapter\Git\GitHub; class Builds extends Action @@ -514,10 +512,7 @@ class Builds extends Action $build->setAttribute('endTime', $endTime); $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); $build->setAttribute('status', 'failed'); - $build->setAttribute('logs', $th->getMessage()); - Console::error($th->getMessage()); - Console::error($th->getFile() . ':' . $th->getLine()); - Console::error($th->getTraceAsString()); + $build->setAttribute('logs', $th->getMessage() . "\n" . $th->getFile() . ':' . $th->getLine() . "\n" . $th->getTraceAsString()); if ($isVcsEnabled) { $this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole); diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index c397d744d3..4ef939d900 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -4,7 +4,6 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Event; use Appwrite\Messaging\Adapter\Realtime; -use Appwrite\Utopia\Response\Model\Platform; use Exception; use Utopia\Audit\Audit; use Utopia\CLI\Console; @@ -72,14 +71,14 @@ class Databases extends Action $log->addTag('databaseId', $database->getId()); - match (strval($type)) { + match (\strval($type)) { DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject), DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject), DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject), DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject), DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject), DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject), - default => Console::error('No database operation for type: ' . $type), + default => throw new \Exception('No database operation for type: ' . \strval($type)), }; } @@ -167,6 +166,7 @@ class Databases extends Action $dbForProject->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'available')); } catch (\Exception $e) { + // TODO: Send non DatabaseExceptions to Sentry Console::error($e->getMessage()); if ($e instanceof DatabaseException) { @@ -176,6 +176,7 @@ class Databases extends Action } } + $dbForProject->updateDocument( 'attributes', $attribute->getId(), @@ -269,6 +270,7 @@ class Databases extends Action $dbForProject->deleteDocument('attributes', $relatedAttribute->getId()); } } catch (\Exception $e) { + // TODO: Send non DatabaseExceptions to Sentry Console::error($e->getMessage()); if ($e instanceof DatabaseException) { @@ -396,6 +398,7 @@ class Databases extends Action } $dbForProject->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'available')); } catch (\Exception $e) { + // TODO: Send non DatabaseExceptions to Sentry Console::error($e->getMessage()); if ($e instanceof DatabaseException) { @@ -453,6 +456,7 @@ class Databases extends Action $dbForProject->deleteDocument('indexes', $index->getId()); $index->setAttribute('status', 'deleted'); } catch (\Exception $e) { + // TODO: Send non DatabaseExceptions to Sentry Console::error($e->getMessage()); if ($e instanceof DatabaseException) { @@ -589,7 +593,7 @@ class Databases extends Action $callback($document); } } else { - Console::error('Failed to delete document: ' . $document->getId()); + Console::warning('Failed to delete document: ' . $document->getId()); } $count++; } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 2affc1f77d..c718a5a3bf 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -73,7 +73,7 @@ class Deletes extends Action $log->addTag('projectId', $project->getId()); $log->addTag('type', $type); - switch (strval($type)) { + switch (\strval($type)) { case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { case DELETE_TYPE_DATABASES: @@ -159,7 +159,7 @@ class Deletes extends Action $this->deleteTopic($project, $getProjectDB, $document); break; default: - Console::error('No delete operation for type: ' . $type); + throw new \Exception('No delete operation for type: ' . \strval($type)); break; } } diff --git a/src/Appwrite/Platform/Workers/Mails.php b/src/Appwrite/Platform/Workers/Mails.php index ea9b77a8c4..7b4bbbebcb 100644 --- a/src/Appwrite/Platform/Workers/Mails.php +++ b/src/Appwrite/Platform/Workers/Mails.php @@ -7,7 +7,6 @@ use Exception; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Runtime; use Utopia\App; -use Utopia\CLI\Console; use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; @@ -53,8 +52,7 @@ class Mails extends Action $smtp = $payload['smtp']; if (empty($smtp) && empty(App::getEnv('_APP_SMTP_HOST'))) { - Console::info('Skipped mail processing. No SMTP configuration has been set.'); - return; + throw new Exception('Skipped mail processing. No SMTP configuration has been set.'); } $log->addTag('type', empty($smtp) ? 'cloud' : 'smtp'); From 06e385b346e09d5fbe386070e0bbd60ab7ecf6b8 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 3 Jan 2024 19:12:21 +0000 Subject: [PATCH 05/34] Add changes from previous console platforms variable PR See https://github.com/appwrite/appwrite/pull/4581 --- .env | 2 +- app/config/variables.php | 2 +- tests/e2e/General/HTTPTest.php | 46 ++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 81145b563d..9df5864bd0 100644 --- a/.env +++ b/.env @@ -4,7 +4,7 @@ _APP_WORKER_PER_CORE=6 _APP_CONSOLE_WHITELIST_ROOT=disabled _APP_CONSOLE_WHITELIST_EMAILS= _APP_CONSOLE_WHITELIST_IPS= -_APP_CONSOLE_HOSTNAMES= +_APP_CONSOLE_HOSTNAMES=localhost,appwrite.io,*.appwrite.io _APP_SYSTEM_EMAIL_NAME=Appwrite _APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io _APP_SYSTEM_SECURITY_EMAIL_ADDRESS=security@appwrite.io diff --git a/app/config/variables.php b/app/config/variables.php index a88145601d..69eca57533 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -127,7 +127,7 @@ return [ [ 'name' => '_APP_CONSOLE_HOSTNAMES', 'description' => 'This option allows you to add additional hostnames to your Appwrite console. This option is very useful for allowing access to the console project from additional domains. To enable it, pass a list of allowed hostnames separated by a comma.', - 'introduction' => '', + 'introduction' => '1.5.0', 'default' => '', 'required' => false, 'question' => '', diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index f83f28c26d..bf8f6de279 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -171,4 +171,50 @@ class HTTPTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); } + + public function testCors() + { + /** + * Test for SUCCESS + */ + + $endpoint = '/v1/projects'; // Can be any non-404 route + + $response = $this->client->call(Client::METHOD_GET, $endpoint); + + $this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']); + + $response = $this->client->call(Client::METHOD_GET, $endpoint, [ + 'origin' => 'http://localhost', + ]); + + $this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']); + + $response = $this->client->call(Client::METHOD_GET, $endpoint, [ + 'origin' => 'http://appwrite.io', + ]); + + $this->assertEquals('http://appwrite.io', $response['headers']['access-control-allow-origin']); + + $response = $this->client->call(Client::METHOD_GET, $endpoint, [ + 'origin' => 'https://appwrite.io', + ]); + + $this->assertEquals('https://appwrite.io', $response['headers']['access-control-allow-origin']); + + $response = $this->client->call(Client::METHOD_GET, $endpoint, [ + 'origin' => 'http://cloud.appwrite.io', + ]); + + $this->assertEquals('http://cloud.appwrite.io', $response['headers']['access-control-allow-origin']); + + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, $endpoint, [ + 'origin' => 'http://google.com', + ]); + + $this->assertEquals('http://localhost', $response['headers']['access-control-allow-origin']); + } } From faf39fcc81b824e81d6fd68029721117f878f2ce Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 3 Jan 2024 19:25:54 +0000 Subject: [PATCH 06/34] Fix _APP_CONSOLE_HOSTNAMES check Ensure invalid hostnames such as empty strings are not added as a hostname. --- app/init.php | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/app/init.php b/app/init.php index ccd7b96573..1bc362fdc2 100644 --- a/app/init.php +++ b/app/init.php @@ -80,6 +80,7 @@ use Utopia\Queue\Connection; use Utopia\Storage\Storage; use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub; use Utopia\Validator\Range; +use Utopia\Validator\Hostname; use Utopia\Validator\IP; use Utopia\Validator\URL; use Utopia\Validator\WhiteList; @@ -947,15 +948,18 @@ App::setResource('clients', function ($request, $console, $project) { ], Document::SET_TYPE_APPEND); $hostnames = explode(',', App::getEnv('_APP_CONSOLE_HOSTNAMES', '')); - if (is_array($hostnames)) { - foreach ($hostnames as $hostname) { - $console->setAttribute('platforms', [ - '$collection' => ID::custom('platforms'), - 'type' => Origin::CLIENT_TYPE_WEB, - 'name' => $hostname, - 'hostname' => $hostname, - ], Document::SET_TYPE_APPEND); + $validator = new Hostname(); + foreach ($hostnames as $hostname) { + $hostname = trim($hostname); + if (!$validator->isValid($hostname)) { + continue; } + $console->setAttribute('platforms', [ + '$collection' => ID::custom('platforms'), + 'type' => Origin::CLIENT_TYPE_WEB, + 'name' => $hostname, + 'hostname' => $hostname, + ], Document::SET_TYPE_APPEND); } /** From 86158838a14d77952dc4fb8ddfdf44181432b543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 9 Jan 2024 12:23:13 +0000 Subject: [PATCH 07/34] Magic URL improvements --- app/config/locale/templates/email-base.tpl | 6 +++--- .../locale/templates/email-magic-url.tpl | 20 +++++++++++++++++++ app/config/locale/translations/en.json | 12 ++++++----- app/controllers/api/account.php | 19 ++++++++++++++---- src/Appwrite/Auth/Auth.php | 2 +- 5 files changed, 46 insertions(+), 13 deletions(-) create mode 100644 app/config/locale/templates/email-magic-url.tpl diff --git a/app/config/locale/templates/email-base.tpl b/app/config/locale/templates/email-base.tpl index f41a9530e1..346f2f1589 100644 --- a/app/config/locale/templates/email-base.tpl +++ b/app/config/locale/templates/email-base.tpl @@ -8,13 +8,13 @@ href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=Poppins:wght@500;600&display=swap" rel="stylesheet">