From bae6e465fa564e1be53b4adfca59cc13b2a4e3ba Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 11 Nov 2024 18:46:47 +0200 Subject: [PATCH 01/17] updateAttributes --- app/controllers/api/databases.php | 38 ++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 0114fd343c..6af4f592d6 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -396,20 +396,37 @@ function updateAttribute( } } - if (!empty($newKey) && $key !== $newKey) { - // Delete attribute and recreate since we can't modify IDs - $original = clone $attribute; + $purge = false; - $dbForProject->deleteDocument('attributes', $attribute->getId()); + if (!empty($newKey) && $key !== $newKey) { + $originalUid = $attribute->getId(); $attribute ->setAttribute('$id', ID::custom($db->getInternalId() . '_' . $collection->getInternalId() . '_' . $newKey)) ->setAttribute('key', $newKey); - try { - $attribute = $dbForProject->createDocument('attributes', $attribute); - } catch (DatabaseException|PDOException) { - $attribute = $dbForProject->createDocument('attributes', $original); + $dbForProject->updateDocument('attributes', $originalUid, $attribute); + + /** + * @var Document $index + */ + foreach ($collection->getAttribute('indexes') as $index) { + /** + * @var string[] $attributes + */ + $attributes = $index->getAttribute('attributes', []); + $found = \array_search($key, $attributes); + + var_dump($attributes); + var_dump('found==='); + var_dump($found); + + if ($found !== false) { + $attributes[$found] = $newKey; + $index->setAttribute('attributes', $attributes); + $dbForProject->updateDocument('indexes', $index->getId(), $index); + $purge = true; + } } } else { $attribute = $dbForProject->updateDocument('attributes', $db->getInternalId() . '_' . $collection->getInternalId() . '_' . $key, $attribute); @@ -417,6 +434,11 @@ function updateAttribute( $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $collection->getId()); + if($purge){ + // I think we are missing this? + $dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $collection->getInternalId()); + } + $queueForEvents ->setContext('collection', $collection) ->setContext('database', $db) From b62640489dde923fbe2cecf92a20bd59a062c44a Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 12 Nov 2024 11:29:19 +0900 Subject: [PATCH 02/17] Fire logs for each individual migration error --- src/Appwrite/Platform/Workers/Migrations.php | 54 +++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 4b1926ed26..91056d5a49 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -17,6 +17,7 @@ use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; use Utopia\Logger\Log; +use Utopia\Logger\Logger; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; @@ -28,6 +29,7 @@ use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\System\System; class Migrations extends Action { @@ -37,6 +39,8 @@ class Migrations extends Action protected Document $project; + protected Logger $logger; + public static function getName(): string { return 'migrations'; @@ -53,13 +57,14 @@ class Migrations extends Action ->inject('dbForProject') ->inject('dbForConsole') ->inject('log') - ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, Log $log) => $this->action($message, $dbForProject, $dbForConsole, $log)); + ->inject('logger') + ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, Log $log, Logger $logger) => $this->action($message, $dbForProject, $dbForConsole, $log, $logger)); } /** * @throws Exception */ - public function action(Message $message, Database $dbForProject, Database $dbForConsole, Log $log): void + public function action(Message $message, Database $dbForProject, Database $dbForConsole, Log $log, Logger $logger): void { $payload = $message->getPayload() ?? []; @@ -78,6 +83,7 @@ class Migrations extends Action $this->dbForProject = $dbForProject; $this->dbForConsole = $dbForConsole; $this->project = $project; + $this->logger = $logger; /** * Handle Event execution. @@ -324,7 +330,6 @@ class Migrations extends Action $errorMessages = []; foreach ($sourceErrors as $error) { - /** @var $sourceErrors $error */ $message = "Error occurred while fetching '{$error->getResourceName()}:{$error->getResourceId()}' from source with message: '{$error->getMessage()}'"; if ($error->getPrevious()) { $message .= " Message: ".$error->getPrevious()->getMessage() . " File: ".$error->getPrevious()->getFile() . " Line: ".$error->getPrevious()->getLine(); @@ -359,7 +364,6 @@ class Migrations extends Action if (! $migration->isEmpty()) { $migration->setAttribute('status', 'failed'); $migration->setAttribute('stage', 'finished'); - $migration->setAttribute('errors', [$th->getMessage()]); return; } @@ -379,7 +383,6 @@ class Migrations extends Action } $migration->setAttribute('errors', $errorMessages); - $log->addTag('migrationErrors', json_encode($errorMessages)); } } finally { if (! $tempAPIKey->isEmpty()) { @@ -394,7 +397,13 @@ class Migrations extends Action $destination->error(); $source->error(); - throw new Exception('Migration failed'); + foreach ($source->getErrors() as $error) { + $this->triggerExceptionLog($migration, $error); + } + + foreach ($destination->getErrors() as $error) { + $this->triggerExceptionLog($migration, $error); + } } if ($migration->getAttribute('status', '') === 'completed') { @@ -403,4 +412,37 @@ class Migrations extends Action } } } + + public function triggerExceptionLog(Document $migration, MigrationException $error) { + if (empty($this->logger)) { + return; + } + + $log = new Log(); + + $log->setNamespace("appwrite-worker"); + $log->setServer(\gethostname()); + $log->setVersion(System::getEnv('_APP_VERSION', 'UNKNOWN')); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + $log->setAction('appwrite-queue-' . Self::getName()); + $log->addTag('verboseType', get_class($error)); + $log->addTag('projectId', $this->project->getId() ?? 'n/a'); + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + $log->addExtra('migrationId', $migration->getId() ?? 'n/a'); + $log->addExtra('source', $migration->getAttribute('source') ?? 'n/a'); + $log->addExtra('resourceName', $error->getResourceName()); + $log->addExtra('resourceGroup', $error->getResourceGroup()); + $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + try { + $responseCode = $this->logger->addLog($log); + Console::info('Error log pushed with status code: ' . $responseCode); + } catch (\Throwable $th) { + Console::error('Error pushing log: ' . $th->getMessage()); + } + } } From a460a41fe56ca2a5ed893bfae62e9580ae10d264 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 12 Nov 2024 11:33:13 +0900 Subject: [PATCH 03/17] Run Linter --- src/Appwrite/Platform/Workers/Migrations.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 91056d5a49..770caa2e69 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -413,7 +413,8 @@ class Migrations extends Action } } - public function triggerExceptionLog(Document $migration, MigrationException $error) { + public function triggerExceptionLog(Document $migration, MigrationException $error) + { if (empty($this->logger)) { return; } @@ -425,7 +426,7 @@ class Migrations extends Action $log->setVersion(System::getEnv('_APP_VERSION', 'UNKNOWN')); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); - $log->setAction('appwrite-queue-' . Self::getName()); + $log->setAction('appwrite-queue-' . self::getName()); $log->addTag('verboseType', get_class($error)); $log->addTag('projectId', $this->project->getId() ?? 'n/a'); $log->addExtra('file', $error->getFile()); @@ -444,5 +445,5 @@ class Migrations extends Action } catch (\Throwable $th) { Console::error('Error pushing log: ' . $th->getMessage()); } - } + } } From 1337ea77b1cb764e8e4cc81a45b16129d306e910 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 12 Nov 2024 16:18:09 +0900 Subject: [PATCH 04/17] Fix Optionals --- src/Appwrite/Platform/Workers/Migrations.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 770caa2e69..921d6dc69a 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -39,7 +39,7 @@ class Migrations extends Action protected Document $project; - protected Logger $logger; + protected ?Logger $logger; public static function getName(): string { @@ -58,13 +58,13 @@ class Migrations extends Action ->inject('dbForConsole') ->inject('log') ->inject('logger') - ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, Log $log, Logger $logger) => $this->action($message, $dbForProject, $dbForConsole, $log, $logger)); + ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, Log $log, ?Logger $logger) => $this->action($message, $dbForProject, $dbForConsole, $log, $logger)); } /** * @throws Exception */ - public function action(Message $message, Database $dbForProject, Database $dbForConsole, Log $log, Logger $logger): void + public function action(Message $message, Database $dbForProject, Database $dbForConsole, Log $log, ?Logger $logger): void { $payload = $message->getPayload() ?? []; From 29e7d1f7d07719c580be7a7c2d71912e492b61fe Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 13 Nov 2024 09:13:00 +0900 Subject: [PATCH 05/17] Address Comments --- src/Appwrite/Platform/Workers/Migrations.php | 21 +++++++------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 921d6dc69a..2514c73d6a 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -56,15 +56,14 @@ class Migrations extends Action ->inject('message') ->inject('dbForProject') ->inject('dbForConsole') - ->inject('log') ->inject('logger') - ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, Log $log, ?Logger $logger) => $this->action($message, $dbForProject, $dbForConsole, $log, $logger)); + ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, ?Logger $logger) => $this->action($message, $dbForProject, $dbForConsole, $logger)); } /** * @throws Exception */ - public function action(Message $message, Database $dbForProject, Database $dbForConsole, Log $log, ?Logger $logger): void + public function action(Message $message, Database $dbForProject, Database $dbForConsole, ?Logger $logger): void { $payload = $message->getPayload() ?? []; @@ -92,10 +91,7 @@ class Migrations extends Action return; } - $log->addTag('migrationId', $migration->getId()); - $log->addTag('projectId', $project->getId()); - - $this->processMigration($migration, $log); + $this->processMigration($migration); } /** @@ -265,7 +261,7 @@ class Migrations extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function processMigration(Document $migration, Log $log): void + protected function processMigration(Document $migration): void { $project = $this->project; $projectDocument = $this->dbForConsole->getDocument('projects', $project->getId()); @@ -291,8 +287,6 @@ class Migrations extends Action $migration->setAttribute('status', 'processing'); $this->updateMigrationDocument($migration, $projectDocument); - $log->addTag('type', $migration->getAttribute('source')); - $source = $this->processSource($migration); $destination = $this->processDestination($migration, $tempAPIKey->getAttribute('secret')); @@ -349,7 +343,6 @@ class Migrations extends Action } $migration->setAttribute('errors', $errorMessages); - $log->addExtra('migrationErrors', json_encode($errorMessages)); $this->updateMigrationDocument($migration, $projectDocument); return; @@ -428,12 +421,12 @@ class Migrations extends Action $log->setMessage($error->getMessage()); $log->setAction('appwrite-queue-' . self::getName()); $log->addTag('verboseType', get_class($error)); - $log->addTag('projectId', $this->project->getId() ?? 'n/a'); + $log->addTag('projectId', $this->project->getId() ?? ''); $log->addExtra('file', $error->getFile()); $log->addExtra('line', $error->getLine()); $log->addExtra('trace', $error->getTraceAsString()); - $log->addExtra('migrationId', $migration->getId() ?? 'n/a'); - $log->addExtra('source', $migration->getAttribute('source') ?? 'n/a'); + $log->addExtra('migrationId', $migration->getId() ?? ''); + $log->addExtra('source', $migration->getAttribute('source') ?? ''); $log->addExtra('resourceName', $error->getResourceName()); $log->addExtra('resourceGroup', $error->getResourceGroup()); $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; From 91cc0e627aca1372a3ca1493cbe97a9ec4a489b2 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 13 Nov 2024 10:29:43 +0900 Subject: [PATCH 06/17] Address Comments --- app/worker.php | 44 +++++++++++++++ src/Appwrite/Platform/Workers/Migrations.php | 58 ++++++-------------- 2 files changed, 61 insertions(+), 41 deletions(-) diff --git a/app/worker.php b/app/worker.php index 4741afe7ea..6d55bdb2d3 100644 --- a/app/worker.php +++ b/app/worker.php @@ -277,6 +277,50 @@ Server::setResource( fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false ); +Server::setResource('logError', function (Registry $register, Document $project) { + return function (Throwable $error, string $namespace, string $action, ?array $extras) use ($register, $project) { + $logger = $register->get('logger'); + + if ($logger) { + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); + + $log = new Log(); + $log->setNamespace($namespace); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + + $log->addTag('code', $error->getCode()); + $log->addTag('verboseType', get_class($error)); + $log->addTag('projectId', $project->getId() ?? ''); + + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + + + foreach ($extras as $key => $value) { + $log->addExtra($key, $value); + } + + $log->setAction($action); + + $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + try { + $responseCode = $logger->addLog($log); + Console::info('Error log pushed with status code: ' . $responseCode); + } catch (Throwable $th) { + Console::error('Error pushing log: ' . $th->getMessage()); + } + } + + Console::warning("Failed: {$error->getMessage()}"); + Console::warning($error->getTraceAsString()); + }; +}, ['register', 'project']); $pools = $register->get('pools'); $platform = new Appwrite(); diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 2514c73d6a..6aa3b08211 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -39,7 +39,7 @@ class Migrations extends Action protected Document $project; - protected ?Logger $logger; + protected $logError; public static function getName(): string { @@ -56,14 +56,14 @@ class Migrations extends Action ->inject('message') ->inject('dbForProject') ->inject('dbForConsole') - ->inject('logger') - ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, ?Logger $logger) => $this->action($message, $dbForProject, $dbForConsole, $logger)); + ->inject('logError') + ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, callable $logError) => $this->action($message, $dbForProject, $dbForConsole, $logError)); } /** * @throws Exception */ - public function action(Message $message, Database $dbForProject, Database $dbForConsole, ?Logger $logger): void + public function action(Message $message, Database $dbForProject, Database $dbForConsole, callable $logError): void { $payload = $message->getPayload() ?? []; @@ -82,7 +82,7 @@ class Migrations extends Action $this->dbForProject = $dbForProject; $this->dbForConsole = $dbForConsole; $this->project = $project; - $this->logger = $logger; + $this->logError = $logError; /** * Handle Event execution. @@ -391,11 +391,21 @@ class Migrations extends Action $source->error(); foreach ($source->getErrors() as $error) { - $this->triggerExceptionLog($migration, $error); + call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ + 'migrationId' => $migration->getId() ?? '', + 'source' => $migration->getAttribute('source') ?? '', + 'resourceName' => $error->getResourceName(), + 'resourceGroup' => $error->getResourceGroup() + ]); } foreach ($destination->getErrors() as $error) { - $this->triggerExceptionLog($migration, $error); + call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ + 'migrationId' => $migration->getId() ?? '', + 'source' => $migration->getAttribute('source') ?? '', + 'resourceName' => $error->getResourceName(), + 'resourceGroup' => $error->getResourceGroup() + ]); } } @@ -405,38 +415,4 @@ class Migrations extends Action } } } - - public function triggerExceptionLog(Document $migration, MigrationException $error) - { - if (empty($this->logger)) { - return; - } - - $log = new Log(); - - $log->setNamespace("appwrite-worker"); - $log->setServer(\gethostname()); - $log->setVersion(System::getEnv('_APP_VERSION', 'UNKNOWN')); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($error->getMessage()); - $log->setAction('appwrite-queue-' . self::getName()); - $log->addTag('verboseType', get_class($error)); - $log->addTag('projectId', $this->project->getId() ?? ''); - $log->addExtra('file', $error->getFile()); - $log->addExtra('line', $error->getLine()); - $log->addExtra('trace', $error->getTraceAsString()); - $log->addExtra('migrationId', $migration->getId() ?? ''); - $log->addExtra('source', $migration->getAttribute('source') ?? ''); - $log->addExtra('resourceName', $error->getResourceName()); - $log->addExtra('resourceGroup', $error->getResourceGroup()); - $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; - $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - - try { - $responseCode = $this->logger->addLog($log); - Console::info('Error log pushed with status code: ' . $responseCode); - } catch (\Throwable $th) { - Console::error('Error pushing log: ' . $th->getMessage()); - } - } } From 54db5655a15b362a51266cc6f595dcad930d6986 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 13 Nov 2024 13:42:49 +0900 Subject: [PATCH 07/17] Run Linter --- app/worker.php | 2 +- src/Appwrite/Platform/Workers/Migrations.php | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/app/worker.php b/app/worker.php index 6d55bdb2d3..6b7aba5c1b 100644 --- a/app/worker.php +++ b/app/worker.php @@ -298,7 +298,7 @@ Server::setResource('logError', function (Registry $register, Document $project) $log->addExtra('file', $error->getFile()); $log->addExtra('line', $error->getLine()); $log->addExtra('trace', $error->getTraceAsString()); - + foreach ($extras as $key => $value) { $log->addExtra($key, $value); diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 6aa3b08211..fdd885effa 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -16,8 +16,6 @@ use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; -use Utopia\Logger\Log; -use Utopia\Logger\Logger; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; @@ -29,7 +27,6 @@ use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; use Utopia\Platform\Action; use Utopia\Queue\Message; -use Utopia\System\System; class Migrations extends Action { From 71f2c6c1316eb041269409bc591ff5bc2201eaeb Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 13 Nov 2024 14:55:56 +0200 Subject: [PATCH 08/17] Remove purgeCachedCollection --- app/controllers/api/databases.php | 12 ------------ src/Appwrite/Platform/Workers/Databases.php | 2 -- 2 files changed, 14 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 6af4f592d6..523529698d 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -396,8 +396,6 @@ function updateAttribute( } } - $purge = false; - if (!empty($newKey) && $key !== $newKey) { $originalUid = $attribute->getId(); @@ -417,15 +415,10 @@ function updateAttribute( $attributes = $index->getAttribute('attributes', []); $found = \array_search($key, $attributes); - var_dump($attributes); - var_dump('found==='); - var_dump($found); - if ($found !== false) { $attributes[$found] = $newKey; $index->setAttribute('attributes', $attributes); $dbForProject->updateDocument('indexes', $index->getId(), $index); - $purge = true; } } } else { @@ -434,11 +427,6 @@ function updateAttribute( $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $collection->getId()); - if($purge){ - // I think we are missing this? - $dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $collection->getInternalId()); - } - $queueForEvents ->setContext('collection', $collection) ->setContext('database', $db) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index f697e7be13..1cbc3a3f8b 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -350,11 +350,9 @@ class Databases extends Action } $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); - $dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); - $dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); } } From 68e96e9851a9500a2b087e3bf48b97be1f898e0d Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 13 Nov 2024 18:32:20 +0200 Subject: [PATCH 09/17] comment --- app/controllers/api/databases.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 523529698d..f07a1e039d 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2609,6 +2609,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') throw new Exception(Exception::ATTRIBUTE_NOT_AVAILABLE, 'Attribute not available: ' . $oldAttributes[$attributeIndex]['key']); } + // todo: Think of a better solution $lengths[$i] = null; if ($attributeType === Database::VAR_STRING) { From 45f85d6b4d143a0c287c6f9b1c70eb40921a1d96 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:30:48 -0800 Subject: [PATCH 10/17] fix: remove duplicate dart-2.16 runtime template --- app/config/function-templates.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/function-templates.php b/app/config/function-templates.php index 762c33dd9a..fec9d2227f 100644 --- a/app/config/function-templates.php +++ b/app/config/function-templates.php @@ -11,7 +11,7 @@ const TEMPLATE_RUNTIMES = [ ], 'DART' => [ 'name' => 'dart', - 'versions' => ['3.5', '3.3', '3.1', '3.0', '2.19', '2.18', '2.17', '2.16', '2.16'] + 'versions' => ['3.5', '3.3', '3.1', '3.0', '2.19', '2.18', '2.17', '2.16'] ], 'GO' => [ 'name' => 'go', From 72e491ac69648c1cd5d3d6e53597f6b91dc2d283 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 14 Nov 2024 14:11:36 +0200 Subject: [PATCH 11/17] Remove unsetting index length for strings --- app/controllers/api/databases.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 0114fd343c..33a47dfff1 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2587,7 +2587,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') $attributeStatus = $oldAttributes[$attributeIndex]['status']; $attributeType = $oldAttributes[$attributeIndex]['type']; - $attributeSize = $oldAttributes[$attributeIndex]['size']; $attributeArray = $oldAttributes[$attributeIndex]['array'] ?? false; if ($attributeType === Database::VAR_RELATIONSHIP) { @@ -2601,10 +2600,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') $lengths[$i] = null; - if ($attributeType === Database::VAR_STRING) { - $lengths[$i] = $attributeSize; // set attribute size as index length only for strings - } - if ($attributeArray === true) { $lengths[$i] = Database::ARRAY_INDEX_LENGTH; $orders[$i] = null; From d8799e5a9c67a2d91e9bd2846e251690239f2bea Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 14 Nov 2024 14:13:28 +0200 Subject: [PATCH 12/17] Remove comment --- app/controllers/api/databases.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 81aee28ad7..13bb8c53b2 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2608,7 +2608,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') throw new Exception(Exception::ATTRIBUTE_NOT_AVAILABLE, 'Attribute not available: ' . $oldAttributes[$attributeIndex]['key']); } - // todo: Think of a better solution $lengths[$i] = null; if ($attributeArray === true) { From abf97e391e92121a7965f2db1ffdc329f51f8853 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 14 Nov 2024 14:19:33 +0200 Subject: [PATCH 13/17] formatting --- app/controllers/api/databases.php | 1 - composer.lock | 62 +++++++++++++++---------------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 13bb8c53b2..4dfaf0cd5c 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -20,7 +20,6 @@ use Utopia\Audit\Audit; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Conflict as ConflictException; use Utopia\Database\Exception\Duplicate as DuplicateException; diff --git a/composer.lock b/composer.lock index 508de82034..26da13625d 100644 --- a/composer.lock +++ b/composer.lock @@ -2453,16 +2453,16 @@ }, { "name": "symfony/http-client", - "version": "v7.1.7", + "version": "v7.1.8", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "90ab2a4992dcf5d1f19a9b8737eba36a7c305fd0" + "reference": "c30d91a1deac0dc3ed5e604683cf2e1dfc635b8a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/90ab2a4992dcf5d1f19a9b8737eba36a7c305fd0", - "reference": "90ab2a4992dcf5d1f19a9b8737eba36a7c305fd0", + "url": "https://api.github.com/repos/symfony/http-client/zipball/c30d91a1deac0dc3ed5e604683cf2e1dfc635b8a", + "reference": "c30d91a1deac0dc3ed5e604683cf2e1dfc635b8a", "shasum": "" }, "require": { @@ -2527,7 +2527,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.1.7" + "source": "https://github.com/symfony/http-client/tree/v7.1.8" }, "funding": [ { @@ -2543,7 +2543,7 @@ "type": "tidelift" } ], - "time": "2024-11-05T16:45:54+00:00" + "time": "2024-11-13T13:40:27+00:00" }, { "name": "symfony/http-client-contracts", @@ -4806,16 +4806,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.39.24", + "version": "0.39.25", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "412451c87f6ef17e24e9a5cf41721043d74c60c8" + "reference": "5b5323636a8d75a1c4faaae9728098dd6a6a47d1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/412451c87f6ef17e24e9a5cf41721043d74c60c8", - "reference": "412451c87f6ef17e24e9a5cf41721043d74c60c8", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/5b5323636a8d75a1c4faaae9728098dd6a6a47d1", + "reference": "5b5323636a8d75a1c4faaae9728098dd6a6a47d1", "shasum": "" }, "require": { @@ -4823,7 +4823,7 @@ "ext-json": "*", "ext-mbstring": "*", "matthiasmullie/minify": "1.3.*", - "php": ">=8.0", + "php": ">=8.3", "twig/twig": "3.14.*" }, "require-dev": { @@ -4851,9 +4851,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.39.24" + "source": "https://github.com/appwrite/sdk-generator/tree/0.39.25" }, - "time": "2024-10-09T19:13:27+00:00" + "time": "2024-11-08T10:16:34+00:00" }, { "name": "doctrine/annotations", @@ -7576,16 +7576,16 @@ }, { "name": "symfony/console", - "version": "v7.1.7", + "version": "v7.1.8", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "3284aafcac338b6e86fd955ee4d794cbe434151a" + "reference": "ff04e5b5ba043d2badfb308197b9e6b42883fcd5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/3284aafcac338b6e86fd955ee4d794cbe434151a", - "reference": "3284aafcac338b6e86fd955ee4d794cbe434151a", + "url": "https://api.github.com/repos/symfony/console/zipball/ff04e5b5ba043d2badfb308197b9e6b42883fcd5", + "reference": "ff04e5b5ba043d2badfb308197b9e6b42883fcd5", "shasum": "" }, "require": { @@ -7649,7 +7649,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.1.7" + "source": "https://github.com/symfony/console/tree/v7.1.8" }, "funding": [ { @@ -7665,7 +7665,7 @@ "type": "tidelift" } ], - "time": "2024-11-05T15:34:55+00:00" + "time": "2024-11-06T14:23:19+00:00" }, { "name": "symfony/filesystem", @@ -8180,16 +8180,16 @@ }, { "name": "symfony/process", - "version": "v7.1.7", + "version": "v7.1.8", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "9b8a40b7289767aa7117e957573c2a535efe6585" + "reference": "42783370fda6e538771f7c7a36e9fa2ee3a84892" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/9b8a40b7289767aa7117e957573c2a535efe6585", - "reference": "9b8a40b7289767aa7117e957573c2a535efe6585", + "url": "https://api.github.com/repos/symfony/process/zipball/42783370fda6e538771f7c7a36e9fa2ee3a84892", + "reference": "42783370fda6e538771f7c7a36e9fa2ee3a84892", "shasum": "" }, "require": { @@ -8221,7 +8221,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.1.7" + "source": "https://github.com/symfony/process/tree/v7.1.8" }, "funding": [ { @@ -8237,20 +8237,20 @@ "type": "tidelift" } ], - "time": "2024-11-06T09:25:12+00:00" + "time": "2024-11-06T14:23:19+00:00" }, { "name": "symfony/string", - "version": "v7.1.6", + "version": "v7.1.8", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "61b72d66bf96c360a727ae6232df5ac83c71f626" + "reference": "591ebd41565f356fcd8b090fe64dbb5878f50281" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/61b72d66bf96c360a727ae6232df5ac83c71f626", - "reference": "61b72d66bf96c360a727ae6232df5ac83c71f626", + "url": "https://api.github.com/repos/symfony/string/zipball/591ebd41565f356fcd8b090fe64dbb5878f50281", + "reference": "591ebd41565f356fcd8b090fe64dbb5878f50281", "shasum": "" }, "require": { @@ -8308,7 +8308,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v7.1.6" + "source": "https://github.com/symfony/string/tree/v7.1.8" }, "funding": [ { @@ -8324,7 +8324,7 @@ "type": "tidelift" } ], - "time": "2024-09-25T14:20:29+00:00" + "time": "2024-11-13T13:31:21+00:00" }, { "name": "textalk/websocket", From 7e7a4658cdaf885875bdacfb9a1c4476f030a0e4 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 14 Nov 2024 14:20:28 +0200 Subject: [PATCH 14/17] revert lock --- composer.lock | 62 +++++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/composer.lock b/composer.lock index 26da13625d..508de82034 100644 --- a/composer.lock +++ b/composer.lock @@ -2453,16 +2453,16 @@ }, { "name": "symfony/http-client", - "version": "v7.1.8", + "version": "v7.1.7", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "c30d91a1deac0dc3ed5e604683cf2e1dfc635b8a" + "reference": "90ab2a4992dcf5d1f19a9b8737eba36a7c305fd0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/c30d91a1deac0dc3ed5e604683cf2e1dfc635b8a", - "reference": "c30d91a1deac0dc3ed5e604683cf2e1dfc635b8a", + "url": "https://api.github.com/repos/symfony/http-client/zipball/90ab2a4992dcf5d1f19a9b8737eba36a7c305fd0", + "reference": "90ab2a4992dcf5d1f19a9b8737eba36a7c305fd0", "shasum": "" }, "require": { @@ -2527,7 +2527,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.1.8" + "source": "https://github.com/symfony/http-client/tree/v7.1.7" }, "funding": [ { @@ -2543,7 +2543,7 @@ "type": "tidelift" } ], - "time": "2024-11-13T13:40:27+00:00" + "time": "2024-11-05T16:45:54+00:00" }, { "name": "symfony/http-client-contracts", @@ -4806,16 +4806,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.39.25", + "version": "0.39.24", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "5b5323636a8d75a1c4faaae9728098dd6a6a47d1" + "reference": "412451c87f6ef17e24e9a5cf41721043d74c60c8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/5b5323636a8d75a1c4faaae9728098dd6a6a47d1", - "reference": "5b5323636a8d75a1c4faaae9728098dd6a6a47d1", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/412451c87f6ef17e24e9a5cf41721043d74c60c8", + "reference": "412451c87f6ef17e24e9a5cf41721043d74c60c8", "shasum": "" }, "require": { @@ -4823,7 +4823,7 @@ "ext-json": "*", "ext-mbstring": "*", "matthiasmullie/minify": "1.3.*", - "php": ">=8.3", + "php": ">=8.0", "twig/twig": "3.14.*" }, "require-dev": { @@ -4851,9 +4851,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.39.25" + "source": "https://github.com/appwrite/sdk-generator/tree/0.39.24" }, - "time": "2024-11-08T10:16:34+00:00" + "time": "2024-10-09T19:13:27+00:00" }, { "name": "doctrine/annotations", @@ -7576,16 +7576,16 @@ }, { "name": "symfony/console", - "version": "v7.1.8", + "version": "v7.1.7", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "ff04e5b5ba043d2badfb308197b9e6b42883fcd5" + "reference": "3284aafcac338b6e86fd955ee4d794cbe434151a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/ff04e5b5ba043d2badfb308197b9e6b42883fcd5", - "reference": "ff04e5b5ba043d2badfb308197b9e6b42883fcd5", + "url": "https://api.github.com/repos/symfony/console/zipball/3284aafcac338b6e86fd955ee4d794cbe434151a", + "reference": "3284aafcac338b6e86fd955ee4d794cbe434151a", "shasum": "" }, "require": { @@ -7649,7 +7649,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.1.8" + "source": "https://github.com/symfony/console/tree/v7.1.7" }, "funding": [ { @@ -7665,7 +7665,7 @@ "type": "tidelift" } ], - "time": "2024-11-06T14:23:19+00:00" + "time": "2024-11-05T15:34:55+00:00" }, { "name": "symfony/filesystem", @@ -8180,16 +8180,16 @@ }, { "name": "symfony/process", - "version": "v7.1.8", + "version": "v7.1.7", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "42783370fda6e538771f7c7a36e9fa2ee3a84892" + "reference": "9b8a40b7289767aa7117e957573c2a535efe6585" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/42783370fda6e538771f7c7a36e9fa2ee3a84892", - "reference": "42783370fda6e538771f7c7a36e9fa2ee3a84892", + "url": "https://api.github.com/repos/symfony/process/zipball/9b8a40b7289767aa7117e957573c2a535efe6585", + "reference": "9b8a40b7289767aa7117e957573c2a535efe6585", "shasum": "" }, "require": { @@ -8221,7 +8221,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.1.8" + "source": "https://github.com/symfony/process/tree/v7.1.7" }, "funding": [ { @@ -8237,20 +8237,20 @@ "type": "tidelift" } ], - "time": "2024-11-06T14:23:19+00:00" + "time": "2024-11-06T09:25:12+00:00" }, { "name": "symfony/string", - "version": "v7.1.8", + "version": "v7.1.6", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "591ebd41565f356fcd8b090fe64dbb5878f50281" + "reference": "61b72d66bf96c360a727ae6232df5ac83c71f626" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/591ebd41565f356fcd8b090fe64dbb5878f50281", - "reference": "591ebd41565f356fcd8b090fe64dbb5878f50281", + "url": "https://api.github.com/repos/symfony/string/zipball/61b72d66bf96c360a727ae6232df5ac83c71f626", + "reference": "61b72d66bf96c360a727ae6232df5ac83c71f626", "shasum": "" }, "require": { @@ -8308,7 +8308,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v7.1.8" + "source": "https://github.com/symfony/string/tree/v7.1.6" }, "funding": [ { @@ -8324,7 +8324,7 @@ "type": "tidelift" } ], - "time": "2024-11-13T13:31:21+00:00" + "time": "2024-09-25T14:20:29+00:00" }, { "name": "textalk/websocket", From 877a978451374589a2f64bd50d41380732c64b2b Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 19 Nov 2024 16:42:16 +0100 Subject: [PATCH 15/17] chore: update base to 0.9.5 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 07295f4afb..41810f5dc4 100755 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN composer install --ignore-platform-reqs --optimize-autoloader \ --no-plugins --no-scripts --prefer-dist \ `if [ "$TESTING" != "true" ]; then echo "--no-dev"; fi` -FROM appwrite/base:0.9.3 AS final +FROM appwrite/base:0.9.5 AS final LABEL maintainer="team@appwrite.io" From 6739f5cc55e85097e60685d97bbd35e6ef721946 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 22 Oct 2024 13:57:25 +0200 Subject: [PATCH 16/17] feat: move certificate generation to Adapter --- app/init.php | 15 ++ src/Appwrite/Certificates/Adapter.php | 14 ++ .../Certificates/Adapter/LetsEncrypt.php | 126 +++++++++++ src/Appwrite/Certificates/AdapterProvider.php | 19 ++ src/Appwrite/Platform/Tasks/Maintenance.php | 1 + .../Platform/Workers/Certificates.php | 206 ++++-------------- src/Appwrite/Platform/Workers/Deletes.php | 49 ++--- 7 files changed, 234 insertions(+), 196 deletions(-) create mode 100644 src/Appwrite/Certificates/Adapter.php create mode 100644 src/Appwrite/Certificates/Adapter/LetsEncrypt.php create mode 100644 src/Appwrite/Certificates/AdapterProvider.php diff --git a/app/init.php b/app/init.php index c9ec2e0061..8b3726fdf2 100644 --- a/app/init.php +++ b/app/init.php @@ -21,6 +21,8 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; +use Appwrite\Certificates\Adapter\LetsEncrypt; +use Appwrite\Certificates\AdapterProvider; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Certificate; @@ -1534,6 +1536,19 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); +App::setResource('letsEncryptCertificateAdapter', function () { + $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); + if (empty($email)) { + throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.'); + } + + return new LetsEncrypt($email); +}); + +App::setResource('adapterForCertificates', function ($letsEncrypt) { + return new AdapterProvider(fn (string $domain) => $letsEncrypt); +}, ['letsEncryptCertificateAdapter']); + App::setResource('deviceForLocal', function () { return new Local(); }); diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php new file mode 100644 index 0000000000..711e4c09b9 --- /dev/null +++ b/src/Appwrite/Certificates/Adapter.php @@ -0,0 +1,14 @@ +email = $email; + } + + + public function issueCertificate(string $certName, string $domain): ?string + { + $stdout = ''; + $stderr = ''; + + $staging = (App::isProduction()) ? '' : ' --dry-run'; + $exit = Console::execute( + "certbot certonly -v --webroot --noninteractive --agree-tos{$staging}" + . " --email " . $this->email + . " --cert-name " . $certName + . " -w " . APP_STORAGE_CERTIFICATES + . " -d {$domain}", + '', + $stdout, + $stderr + ); + + // Unexpected error, usually 5XX, API limits, ... + if ($exit !== 0) { + throw new Exception('Failed to issue a certificate with message: ' . $stderr); + } + + // Prepare folder in storage for domain + $path = APP_STORAGE_CERTIFICATES . '/' . $domain; + if (!\is_readable($path)) { + if (!\mkdir($path, 0755, true)) { + throw new Exception('Failed to create path for certificate.'); + } + } + + // Move generated files + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { + throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { + throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { + throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { + throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + $config = \implode(PHP_EOL, [ + "tls:", + " certificates:", + " - certFile: /storage/certificates/{$domain}/fullchain.pem", + " keyFile: /storage/certificates/{$domain}/privkey.pem" + ]); + + // Save configuration into Traefik using our new cert files + if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { + throw new Exception('Failed to save Traefik configuration.'); + } + + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? null; + $dt = (new \DateTime())->setTimestamp($validTo); + return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); + } + + public function isRenewRequired(string $domain, Log $log): bool + { + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + if (\file_exists($certPath)) { + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? 0; + + if (empty($validTo)) { + $log->addTag('certificateDomain', $domain); + throw new Exception('Unable to read certificate file (cert.pem).'); + } + + // LetsEncrypt allows renewal 30 days before expiry + $expiryInAdvance = (60 * 60 * 24 * 30); + if ($validTo - $expiryInAdvance > \time()) { + $log->addTag('certificateDomain', $domain); + $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData)); + return false; + } + } + + return true; + } + + public function deleteCertificate(string $domain): void + { + $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; + $checkTraversal = realpath($directory) === $directory; + + if ($checkTraversal && is_dir($directory)) { + // Delete files, so Traefik is aware of change + array_map('unlink', glob($directory . '/*.*')); + rmdir($directory); + Console::info("Deleted certificate files for {$domain}"); + } else { + Console::info("No certificate files found for {$domain}"); + } + } +} diff --git a/src/Appwrite/Certificates/AdapterProvider.php b/src/Appwrite/Certificates/AdapterProvider.php new file mode 100644 index 0000000000..bed73a0164 --- /dev/null +++ b/src/Appwrite/Certificates/AdapterProvider.php @@ -0,0 +1,19 @@ +decisionMaker = $decisionMaker; + } + + public function get(string $domain): Adapter + { + return call_user_func($this->decisionMaker, $domain); + } + +} diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index afb38f35fc..40e72dc683 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -149,6 +149,7 @@ class Maintenance extends Action $certificates = $dbForConsole->find('certificates', [ Query::lessThan('attempts', 5), // Maximum 5 attempts + Query::isNotNull('renewDate'), Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) ]); diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 21d967d9e1..655d09426f 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Workers; +use Appwrite\Certificates\AdapterProvider; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; @@ -11,7 +12,6 @@ use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; use Exception; use Throwable; -use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -48,7 +48,11 @@ class Certificates extends Action ->inject('queueForEvents') ->inject('queueForFunctions') ->inject('log') - ->callback(fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log)); + ->inject('adapterForCertificates') + ->callback( + fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates) => + $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $adapterForCertificates) + ); } /** @@ -58,11 +62,12 @@ class Certificates extends Action * @param Event $queueForEvents * @param Func $queueForFunctions * @param Log $log + * @param AdapterProvider $adapterForCertificates Retrieve the certificate adapter for the domain * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates): void { $payload = $message->getPayload() ?? []; @@ -76,7 +81,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck); + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $adapterForCertificates, $skipRenewCheck); } /** @@ -85,24 +90,24 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions + * @param AdapterProvider $adapterForCertificates Retrieve the certificate adapter for the domain * @param bool $skipRenewCheck * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain * 2. Get main domain * 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain) - * 4. Validate security email. Cannot be empty, required by LetsEncrypt - * 5. Validate renew date with certificate file, unless requested to skip by parameter - * 6. Issue a certificate using certbot CLI - * 7. Update 'log' attribute on certificate document with Certbot message - * 8. Create storage folder for certificate, if not ready already - * 9. Move certificates from Certbot location to our Storage - * 10. Create/Update our Storage with new Traefik config with new certificate paths + * 4. Validate renew date with certificate file, unless requested to skip by parameter + * 5. Issue a certificate using certbot CLI + * 6. Update 'log' attribute on certificate document with Certbot message + * 7. Create storage folder for certificate, if not ready already + * 8. Move certificates from Certbot location to our Storage + * 9. Create/Update our Storage with new Traefik config with new certificate paths * 11. Read certificate file and update 'renewDate' on certificate document * 12. Update 'issueDate' and 'attempts' on certificate * @@ -119,7 +124,7 @@ class Certificates extends Action * 2. Save document to database * 3. Update all domains documents with current certificate ID * - * Note: Renewals are checked and scheduled from maintenence worker + * Note: Renewals are checked and scheduled from maintenance worker */ // Get current certificate @@ -131,43 +136,32 @@ class Certificates extends Action $certificate->setAttribute('domain', $domain->get()); } + $certificateAdapter = $adapterForCertificates->get($domain->get()); + $success = false; try { - // Email for alerts is required by LetsEncrypt - $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); - if (empty($email)) { - throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue an SSL certificate.'); - } - // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { $mainDomain = $this->getMainDomain(); $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; $this->validateDomain($domain, $isMainDomain, $log); + + // If certificate exists already, double-check expiry date. Skip if job is forced + if (!$certificateAdapter->isRenewRequired($domain->get(), $log)) { + throw new Exception('Renew isn\'t required.'); + } } - // If certificate exists already, double-check expiry date. Skip if job is forced - if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) { - throw new Exception('Renew isn\'t required.'); - } - - // Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate - $folder = ID::unique(); - - // Generate certificate files using Let's Encrypt - $letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email); + // Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates. + $certName = ID::unique(); + $renewDate = $certificateAdapter->issueCertificate($certName, $domain->get()); // Command succeeded, store all data into document - $logs = 'Certificate successfully generated.'; - $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB - - - // Give certificates to Traefik - $this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData); + $certificate->setAttribute('logs', 'Certificate successfully generated.'); // Update certificate info stored in database - $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get())); + $certificate->setAttribute('renewDate', $renewDate); $certificate->setAttribute('attempts', 0); $certificate->setAttribute('issueDate', DateTime::now()); $success = true; @@ -181,7 +175,7 @@ class Certificates extends Action $attempts = $certificate->getAttribute('attempts', 0) + 1; $certificate->setAttribute('attempts', $attempts); - // Store cuttent time as renew date to ensure another attempt in next maintenance cycle + // Store current time as renew date to ensure another attempt in next maintenance cycle. $certificate->setAttribute('renewDate', DateTime::now()); // Send email to security email @@ -245,8 +239,8 @@ class Certificates extends Action } /** - * Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check: - * - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt) + * Internal domain validation functionality to prevent unnecessary attempts. We check: + * - Domain needs to be public and valid (prevents NFT domains that are not supported) * - Domain must have proper DNS record * * @param Domain $domain Domain which we validate @@ -292,136 +286,6 @@ class Certificates extends Action } } - /** - * Reads expiry date of certificate from file and decides if renewal is required or not. - * - * @param string $domain Domain for which we check certificate file - * @return bool True, if certificate needs to be renewed - * @throws Exception - */ - private function isRenewRequired(string $domain, Log $log): bool - { - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; - if (\file_exists($certPath)) { - $validTo = null; - - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? 0; - - if (empty($validTo)) { - $log->addTag('certificateDomain', $domain); - throw new Exception('Unable to read certificate file (cert.pem).'); - } - - // LetsEncrypt allows renewal 30 days before expiry - $expiryInAdvance = (60 * 60 * 24 * 30); - if ($validTo - $expiryInAdvance > \time()) { - $log->addTag('certificateDomain', $domain); - $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData)); - return false; - } - } - - return true; - } - - /** - * LetsEncrypt communication to issue certificate (using certbot CLI) - * - * @param string $folder Folder into which certificates should be generated - * @param string $domain Domain to generate certificate for - * @return array Named array with keys 'stdout' and 'stderr', both string - * @throws Exception - */ - private function issueCertificate(string $folder, string $domain, string $email): array - { - $stdout = ''; - $stderr = ''; - - $staging = (App::isProduction()) ? '' : ' --dry-run'; - $exit = Console::execute("certbot certonly -v --webroot --noninteractive --agree-tos{$staging}" - . " --email " . $email - . " --cert-name " . $folder - . " -w " . APP_STORAGE_CERTIFICATES - . " -d {$domain}", '', $stdout, $stderr); - - // Unexpected error, usually 5XX, API limits, ... - if ($exit !== 0) { - throw new Exception('Failed to issue a certificate with message: ' . $stderr); - } - - return [ - 'stdout' => $stdout, - 'stderr' => $stderr - ]; - } - - /** - * Read new renew date from certificate file generated by Let's Encrypt - * - * @param string $domain Domain which certificate was generated for - * @return string - * @throws \Utopia\Database\Exception - */ - private function getRenewDate(string $domain): string - { - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? null; - $dt = (new \DateTime())->setTimestamp($validTo); - return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days - } - - /** - * Method to take files from Let's Encrypt, and put it into Traefik. - * - * @param string $domain Domain which certificate was generated for - * @param string $folder Folder in which certificates were generated - * @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error - * @return void - * @throws Exception - */ - private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void - { - - // Prepare folder in storage for domain - $path = APP_STORAGE_CERTIFICATES . '/' . $domain; - if (!\is_readable($path)) { - if (!\mkdir($path, 0755, true)) { - throw new Exception('Failed to create path for certificate.'); - } - } - - // Move generated files - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - $config = \implode(PHP_EOL, [ - "tls:", - " certificates:", - " - certFile: /storage/certificates/{$domain}/fullchain.pem", - " keyFile: /storage/certificates/{$domain}/privkey.pem" - ]); - - // Save configuration into Traefik using our new cert files - if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { - throw new Exception('Failed to save Traefik configuration.'); - } - } - /** * Method to make sure information about error is delivered to admnistrator. * @@ -494,6 +358,10 @@ class Certificates extends Action $project = $dbForConsole->getDocument('projects', $projectId); + if ($project->isEmpty()) { + return; + } + /** Trigger Webhook */ $ruleModel = new Rule(); $queueForEvents diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 48e4014f1e..a85c435eb0 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Auth\Auth; +use Appwrite\Certificates\AdapterProvider; use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; @@ -50,18 +51,22 @@ class Deletes extends Action ->inject('deviceForFunctions') ->inject('deviceForBuilds') ->inject('deviceForCache') + ->inject('adapterForCertificates') ->inject('abuseRetention') ->inject('executionRetention') ->inject('auditRetention') ->inject('log') - ->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log)); + ->callback( + fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $abuseRetention, $executionRetention, $auditRetention, $log) + ); } /** * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -84,10 +89,10 @@ class Deletes extends Action case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { case DELETE_TYPE_PROJECTS: - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); + $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $adapterProvider, $document, $project); break; case DELETE_TYPE_DEPLOYMENTS: $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); @@ -102,7 +107,7 @@ class Deletes extends Action $this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project); break; case DELETE_TYPE_RULES: - $this->deleteRule($dbForConsole, $document); + $this->deleteRule($dbForConsole, $document, $adapterProvider); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -110,7 +115,7 @@ class Deletes extends Action } break; case DELETE_TYPE_TEAM_PROJECTS: - $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $document); + $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $adapterProvider, $document); break; case DELETE_TYPE_EXECUTIONS: $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); @@ -442,7 +447,7 @@ class Deletes extends Action * @throws Structure * @throws Exception */ - private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Document $document): void + private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, AdapterProvider $adapterProvider, Document $document): void { $projects = $dbForConsole->find('projects', [ @@ -455,7 +460,7 @@ class Deletes extends Action $deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); $deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $project); $dbForConsole->deleteDocument('projects', $project->getId()); } } @@ -473,7 +478,7 @@ class Deletes extends Action * @throws Authorization * @throws DatabaseException */ - private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void + private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, Document $document): void { $projectInternalId = $document->getInternalId(); $projectId = $document->getId(); @@ -536,8 +541,8 @@ class Deletes extends Action // Delete project and function rules $this->deleteByGroup('rules', [ Query::equal('projectInternalId', [$projectInternalId]) - ], $dbForConsole, function (Document $document) use ($dbForConsole) { - $this->deleteRule($dbForConsole, $document); + ], $dbForConsole, function (Document $document) use ($dbForConsole, $adapterProvider) { + $this->deleteRule($dbForConsole, $document, $adapterProvider); }); // Delete Keys @@ -738,7 +743,7 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void + private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, AdapterProvider $adapterProvider, Document $document, Document $project): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -753,8 +758,8 @@ class Deletes extends Action Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('projectInternalId', [$project->getInternalId()]) - ], $dbForConsole, function (Document $document) use ($project, $dbForConsole) { - $this->deleteRule($dbForConsole, $document); + ], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $adapterProvider) { + $this->deleteRule($dbForConsole, $document, $adapterProvider); }); /** @@ -1040,21 +1045,11 @@ class Deletes extends Action * @param Document $document rule document * @return void */ - private function deleteRule(Database $dbForConsole, Document $document): void + private function deleteRule(Database $dbForConsole, Document $document, AdapterProvider $adapterForCertificate): void { - $domain = $document->getAttribute('domain'); - $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; - $checkTraversal = realpath($directory) === $directory; - - if ($checkTraversal && is_dir($directory)) { - // Delete files, so Traefik is aware of change - array_map('unlink', glob($directory . '/*.*')); - rmdir($directory); - Console::info("Deleted certificate files for {$domain}"); - } else { - Console::info("No certificate files found for {$domain}"); - } + $adapter = $adapterForCertificate->get($domain); + $adapter->deleteCertificate($domain); // Delete certificate document, so Appwrite is aware of change if (isset($document['certificateId'])) { From 5c4ba9d2fd5e8ebf472d75a636a3c54e945aba5e Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Wed, 20 Nov 2024 11:10:57 +0100 Subject: [PATCH 17/17] refactor: use single adapter --- app/init.php | 9 +--- src/Appwrite/Certificates/AdapterProvider.php | 19 --------- .../{Adapter => }/LetsEncrypt.php | 3 +- .../Platform/Workers/Certificates.php | 24 +++++------ src/Appwrite/Platform/Workers/Deletes.php | 41 +++++++++---------- 5 files changed, 34 insertions(+), 62 deletions(-) delete mode 100644 src/Appwrite/Certificates/AdapterProvider.php rename src/Appwrite/Certificates/{Adapter => }/LetsEncrypt.php (98%) diff --git a/app/init.php b/app/init.php index 8b3726fdf2..a04bf391a5 100644 --- a/app/init.php +++ b/app/init.php @@ -21,8 +21,7 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; -use Appwrite\Certificates\Adapter\LetsEncrypt; -use Appwrite\Certificates\AdapterProvider; +use Appwrite\Certificates\LetsEncrypt; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Certificate; @@ -1536,7 +1535,7 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); -App::setResource('letsEncryptCertificateAdapter', function () { +App::setResource('certificates', function () { $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); if (empty($email)) { throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.'); @@ -1545,10 +1544,6 @@ App::setResource('letsEncryptCertificateAdapter', function () { return new LetsEncrypt($email); }); -App::setResource('adapterForCertificates', function ($letsEncrypt) { - return new AdapterProvider(fn (string $domain) => $letsEncrypt); -}, ['letsEncryptCertificateAdapter']); - App::setResource('deviceForLocal', function () { return new Local(); }); diff --git a/src/Appwrite/Certificates/AdapterProvider.php b/src/Appwrite/Certificates/AdapterProvider.php deleted file mode 100644 index bed73a0164..0000000000 --- a/src/Appwrite/Certificates/AdapterProvider.php +++ /dev/null @@ -1,19 +0,0 @@ -decisionMaker = $decisionMaker; - } - - public function get(string $domain): Adapter - { - return call_user_func($this->decisionMaker, $domain); - } - -} diff --git a/src/Appwrite/Certificates/Adapter/LetsEncrypt.php b/src/Appwrite/Certificates/LetsEncrypt.php similarity index 98% rename from src/Appwrite/Certificates/Adapter/LetsEncrypt.php rename to src/Appwrite/Certificates/LetsEncrypt.php index b2237189e9..3896eab022 100644 --- a/src/Appwrite/Certificates/Adapter/LetsEncrypt.php +++ b/src/Appwrite/Certificates/LetsEncrypt.php @@ -1,8 +1,7 @@ inject('queueForEvents') ->inject('queueForFunctions') ->inject('log') - ->inject('adapterForCertificates') + ->inject('certificates') ->callback( - fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates) => - $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $adapterForCertificates) + fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates) => + $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates) ); } @@ -62,12 +62,12 @@ class Certificates extends Action * @param Event $queueForEvents * @param Func $queueForFunctions * @param Log $log - * @param AdapterProvider $adapterForCertificates Retrieve the certificate adapter for the domain + * @param CertificatesAdapter $certificates * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates): void + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates): void { $payload = $message->getPayload() ?? []; @@ -81,7 +81,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $adapterForCertificates, $skipRenewCheck); + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates, $skipRenewCheck); } /** @@ -90,13 +90,13 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions - * @param AdapterProvider $adapterForCertificates Retrieve the certificate adapter for the domain + * @param CertificatesAdapter $certificates * @param bool $skipRenewCheck * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain @@ -136,8 +136,6 @@ class Certificates extends Action $certificate->setAttribute('domain', $domain->get()); } - $certificateAdapter = $adapterForCertificates->get($domain->get()); - $success = false; try { @@ -148,14 +146,14 @@ class Certificates extends Action $this->validateDomain($domain, $isMainDomain, $log); // If certificate exists already, double-check expiry date. Skip if job is forced - if (!$certificateAdapter->isRenewRequired($domain->get(), $log)) { + if (!$certificates->isRenewRequired($domain->get(), $log)) { throw new Exception('Renew isn\'t required.'); } } // Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates. $certName = ID::unique(); - $renewDate = $certificateAdapter->issueCertificate($certName, $domain->get()); + $renewDate = $certificates->issueCertificate($certName, $domain->get()); // Command succeeded, store all data into document $certificate->setAttribute('logs', 'Certificate successfully generated.'); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index a85c435eb0..5e930d0229 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -3,7 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Auth\Auth; -use Appwrite\Certificates\AdapterProvider; +use Appwrite\Certificates\Adapter as CertificatesAdapter; use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; @@ -51,14 +51,14 @@ class Deletes extends Action ->inject('deviceForFunctions') ->inject('deviceForBuilds') ->inject('deviceForCache') - ->inject('adapterForCertificates') + ->inject('certificates') ->inject('abuseRetention') ->inject('executionRetention') ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $abuseRetention, $executionRetention, $auditRetention, $log) + fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -89,10 +89,10 @@ class Deletes extends Action case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { case DELETE_TYPE_PROJECTS: - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $document); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $adapterProvider, $document, $project); + $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project); break; case DELETE_TYPE_DEPLOYMENTS: $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); @@ -107,7 +107,7 @@ class Deletes extends Action $this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project); break; case DELETE_TYPE_RULES: - $this->deleteRule($dbForConsole, $document, $adapterProvider); + $this->deleteRule($dbForConsole, $document, $certificates); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -115,7 +115,7 @@ class Deletes extends Action } break; case DELETE_TYPE_TEAM_PROJECTS: - $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $adapterProvider, $document); + $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $certificates, $document); break; case DELETE_TYPE_EXECUTIONS: $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); @@ -269,7 +269,7 @@ class Deletes extends Action MESSAGE_TYPE_EMAIL => 'emailTotal', MESSAGE_TYPE_SMS => 'smsTotal', MESSAGE_TYPE_PUSH => 'pushTotal', - default => throw new Exception('Invalid target provider type'), + default => throw new Exception('Invalid target CertificatesAdapter type'), }; $dbForProject->decreaseDocumentAttribute( 'topics', @@ -447,7 +447,7 @@ class Deletes extends Action * @throws Structure * @throws Exception */ - private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, AdapterProvider $adapterProvider, Document $document): void + private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void { $projects = $dbForConsole->find('projects', [ @@ -460,7 +460,7 @@ class Deletes extends Action $deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); $deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $project); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $project); $dbForConsole->deleteDocument('projects', $project->getId()); } } @@ -478,7 +478,7 @@ class Deletes extends Action * @throws Authorization * @throws DatabaseException */ - private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, Document $document): void + private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Document $document): void { $projectInternalId = $document->getInternalId(); $projectId = $document->getId(); @@ -541,8 +541,8 @@ class Deletes extends Action // Delete project and function rules $this->deleteByGroup('rules', [ Query::equal('projectInternalId', [$projectInternalId]) - ], $dbForConsole, function (Document $document) use ($dbForConsole, $adapterProvider) { - $this->deleteRule($dbForConsole, $document, $adapterProvider); + ], $dbForConsole, function (Document $document) use ($dbForConsole, $certificates) { + $this->deleteRule($dbForConsole, $document, $certificates); }); // Delete Keys @@ -743,7 +743,7 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, AdapterProvider $adapterProvider, Document $document, Document $project): void + private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -758,8 +758,8 @@ class Deletes extends Action Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('projectInternalId', [$project->getInternalId()]) - ], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $adapterProvider) { - $this->deleteRule($dbForConsole, $document, $adapterProvider); + ], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $certificates) { + $this->deleteRule($dbForConsole, $document, $certificates); }); /** @@ -1045,11 +1045,10 @@ class Deletes extends Action * @param Document $document rule document * @return void */ - private function deleteRule(Database $dbForConsole, Document $document, AdapterProvider $adapterForCertificate): void + private function deleteRule(Database $dbForConsole, Document $document, CertificatesAdapter $certificates): void { $domain = $document->getAttribute('domain'); - $adapter = $adapterForCertificate->get($domain); - $adapter->deleteCertificate($domain); + $certificates->deleteCertificate($domain); // Delete certificate document, so Appwrite is aware of change if (isset($document['certificateId'])) {