diff --git a/app/config/function-templates.php b/app/config/function-templates.php index 762c33dd9a..fec9d2227f 100644 --- a/app/config/function-templates.php +++ b/app/config/function-templates.php @@ -11,7 +11,7 @@ const TEMPLATE_RUNTIMES = [ ], 'DART' => [ 'name' => 'dart', - 'versions' => ['3.5', '3.3', '3.1', '3.0', '2.19', '2.18', '2.17', '2.16', '2.16'] + 'versions' => ['3.5', '3.3', '3.1', '3.0', '2.19', '2.18', '2.17', '2.16'] ], 'GO' => [ 'name' => 'go', diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 258d16cf46..a00324f3b7 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -20,7 +20,6 @@ use Utopia\Audit\Audit; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Conflict as ConflictException; use Utopia\Database\Exception\Duplicate as DuplicateException; @@ -399,19 +398,29 @@ function updateAttribute( } if (!empty($newKey) && $key !== $newKey) { - // Delete attribute and recreate since we can't modify IDs - $original = clone $attribute; - - $dbForProject->deleteDocument('attributes', $attribute->getId()); + $originalUid = $attribute->getId(); $attribute ->setAttribute('$id', ID::custom($db->getInternalId() . '_' . $collection->getInternalId() . '_' . $newKey)) ->setAttribute('key', $newKey); - try { - $attribute = $dbForProject->createDocument('attributes', $attribute); - } catch (DatabaseException|PDOException) { - $attribute = $dbForProject->createDocument('attributes', $original); + $dbForProject->updateDocument('attributes', $originalUid, $attribute); + + /** + * @var Document $index + */ + foreach ($collection->getAttribute('indexes') as $index) { + /** + * @var string[] $attributes + */ + $attributes = $index->getAttribute('attributes', []); + $found = \array_search($key, $attributes); + + if ($found !== false) { + $attributes[$found] = $newKey; + $index->setAttribute('attributes', $attributes); + $dbForProject->updateDocument('indexes', $index->getId(), $index); + } } } else { $attribute = $dbForProject->updateDocument('attributes', $db->getInternalId() . '_' . $collection->getInternalId() . '_' . $key, $attribute); @@ -2589,7 +2598,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') $attributeStatus = $oldAttributes[$attributeIndex]['status']; $attributeType = $oldAttributes[$attributeIndex]['type']; - $attributeSize = $oldAttributes[$attributeIndex]['size']; $attributeArray = $oldAttributes[$attributeIndex]['array'] ?? false; if ($attributeType === Database::VAR_RELATIONSHIP) { @@ -2603,10 +2611,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') $lengths[$i] = null; - if ($attributeType === Database::VAR_STRING) { - $lengths[$i] = $attributeSize; // set attribute size as index length only for strings - } - if ($attributeArray === true) { $lengths[$i] = Database::ARRAY_INDEX_LENGTH; $orders[$i] = null; diff --git a/app/init.php b/app/init.php index 3744bb055c..b3988774e3 100644 --- a/app/init.php +++ b/app/init.php @@ -21,6 +21,7 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; +use Appwrite\Certificates\LetsEncrypt; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Certificate; @@ -1538,6 +1539,15 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); +App::setResource('certificates', function () { + $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); + if (empty($email)) { + throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.'); + } + + return new LetsEncrypt($email); +}); + App::setResource('deviceForLocal', function () { return new Local(); }); diff --git a/app/worker.php b/app/worker.php index 25defba3f8..7899c1d5bc 100644 --- a/app/worker.php +++ b/app/worker.php @@ -283,6 +283,50 @@ Server::setResource( fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false ); +Server::setResource('logError', function (Registry $register, Document $project) { + return function (Throwable $error, string $namespace, string $action, ?array $extras) use ($register, $project) { + $logger = $register->get('logger'); + + if ($logger) { + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); + + $log = new Log(); + $log->setNamespace($namespace); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + + $log->addTag('code', $error->getCode()); + $log->addTag('verboseType', get_class($error)); + $log->addTag('projectId', $project->getId() ?? ''); + + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + + + foreach ($extras as $key => $value) { + $log->addExtra($key, $value); + } + + $log->setAction($action); + + $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + try { + $responseCode = $logger->addLog($log); + Console::info('Error log pushed with status code: ' . $responseCode); + } catch (Throwable $th) { + Console::error('Error pushing log: ' . $th->getMessage()); + } + } + + Console::warning("Failed: {$error->getMessage()}"); + Console::warning($error->getTraceAsString()); + }; +}, ['register', 'project']); $pools = $register->get('pools'); $platform = new Appwrite(); diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php new file mode 100644 index 0000000000..711e4c09b9 --- /dev/null +++ b/src/Appwrite/Certificates/Adapter.php @@ -0,0 +1,14 @@ +email = $email; + } + + + public function issueCertificate(string $certName, string $domain): ?string + { + $stdout = ''; + $stderr = ''; + + $staging = (App::isProduction()) ? '' : ' --dry-run'; + $exit = Console::execute( + "certbot certonly -v --webroot --noninteractive --agree-tos{$staging}" + . " --email " . $this->email + . " --cert-name " . $certName + . " -w " . APP_STORAGE_CERTIFICATES + . " -d {$domain}", + '', + $stdout, + $stderr + ); + + // Unexpected error, usually 5XX, API limits, ... + if ($exit !== 0) { + throw new Exception('Failed to issue a certificate with message: ' . $stderr); + } + + // Prepare folder in storage for domain + $path = APP_STORAGE_CERTIFICATES . '/' . $domain; + if (!\is_readable($path)) { + if (!\mkdir($path, 0755, true)) { + throw new Exception('Failed to create path for certificate.'); + } + } + + // Move generated files + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { + throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { + throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { + throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { + throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + $config = \implode(PHP_EOL, [ + "tls:", + " certificates:", + " - certFile: /storage/certificates/{$domain}/fullchain.pem", + " keyFile: /storage/certificates/{$domain}/privkey.pem" + ]); + + // Save configuration into Traefik using our new cert files + if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { + throw new Exception('Failed to save Traefik configuration.'); + } + + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? null; + $dt = (new \DateTime())->setTimestamp($validTo); + return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); + } + + public function isRenewRequired(string $domain, Log $log): bool + { + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + if (\file_exists($certPath)) { + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? 0; + + if (empty($validTo)) { + $log->addTag('certificateDomain', $domain); + throw new Exception('Unable to read certificate file (cert.pem).'); + } + + // LetsEncrypt allows renewal 30 days before expiry + $expiryInAdvance = (60 * 60 * 24 * 30); + if ($validTo - $expiryInAdvance > \time()) { + $log->addTag('certificateDomain', $domain); + $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData)); + return false; + } + } + + return true; + } + + public function deleteCertificate(string $domain): void + { + $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; + $checkTraversal = realpath($directory) === $directory; + + if ($checkTraversal && is_dir($directory)) { + // Delete files, so Traefik is aware of change + array_map('unlink', glob($directory . '/*.*')); + rmdir($directory); + Console::info("Deleted certificate files for {$domain}"); + } else { + Console::info("No certificate files found for {$domain}"); + } + } +} diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index afb38f35fc..40e72dc683 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -149,6 +149,7 @@ class Maintenance extends Action $certificates = $dbForConsole->find('certificates', [ Query::lessThan('attempts', 5), // Maximum 5 attempts + Query::isNotNull('renewDate'), Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) ]); diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index e55baa4e1e..c0884b0385 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Workers; +use Appwrite\Certificates\Adapter as CertificatesAdapter; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; @@ -11,7 +12,6 @@ use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; use Exception; use Throwable; -use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -22,7 +22,6 @@ use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Domains\Domain; -use Utopia\Fetch\Client; use Utopia\Locale\Locale; use Utopia\Logger\Log; use Utopia\Platform\Action; @@ -44,28 +43,31 @@ class Certificates extends Action $this ->desc('Certificates worker') ->inject('message') - ->inject('project') ->inject('dbForConsole') ->inject('queueForMails') ->inject('queueForEvents') ->inject('queueForFunctions') ->inject('log') - ->callback(fn (Message $message, Document $project, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $project, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log)); + ->inject('certificates') + ->callback( + fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates) => + $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates) + ); } /** * @param Message $message - * @param Document $project * @param Database $dbForConsole * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions * @param Log $log + * @param CertificatesAdapter $certificates * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Document $project, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates): void { $payload = $message->getPayload() ?? []; @@ -79,7 +81,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $project, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck); + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates, $skipRenewCheck); } /** @@ -88,24 +90,24 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions + * @param CertificatesAdapter $certificates * @param bool $skipRenewCheck * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Document $project, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain * 2. Get main domain * 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain) - * 4. Validate security email. Cannot be empty, required by LetsEncrypt - * 5. Validate renew date with certificate file, unless requested to skip by parameter - * 6. Issue a certificate using certbot CLI - * 7. Update 'log' attribute on certificate document with Certbot message - * 8. Create storage folder for certificate, if not ready already - * 9. Move certificates from Certbot location to our Storage - * 10. Create/Update our Storage with new Traefik config with new certificate paths + * 4. Validate renew date with certificate file, unless requested to skip by parameter + * 5. Issue a certificate using certbot CLI + * 6. Update 'log' attribute on certificate document with Certbot message + * 7. Create storage folder for certificate, if not ready already + * 8. Move certificates from Certbot location to our Storage + * 9. Create/Update our Storage with new Traefik config with new certificate paths * 11. Read certificate file and update 'renewDate' on certificate document * 12. Update 'issueDate' and 'attempts' on certificate * @@ -122,7 +124,7 @@ class Certificates extends Action * 2. Save document to database * 3. Update all domains documents with current certificate ID * - * Note: Renewals are checked and scheduled from maintenence worker + * Note: Renewals are checked and scheduled from maintenance worker */ // Get current certificate @@ -137,50 +139,27 @@ class Certificates extends Action $success = false; try { - // Email for alerts is required by LetsEncrypt - $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); - if (empty($email)) { - throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue an SSL certificate.'); - } - // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { $mainDomain = $this->getMainDomain(); $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; $this->validateDomain($domain, $isMainDomain, $log); + + // If certificate exists already, double-check expiry date. Skip if job is forced + if (!$certificates->isRenewRequired($domain->get(), $log)) { + throw new Exception('Renew isn\'t required.'); + } } - // If certificate exists already, double-check expiry date. Skip if job is forced - if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) { - throw new Exception('Renew isn\'t required.'); - } - - // Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate - $folder = ID::unique(); - - try { - // Generate certificate files using Let's Encrypt - $letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email); - - // Give certificates to Traefik - $this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData); - } catch (\Throwable $th) { - Console::error('Failed to generate Lets Encrypt certificate'); - } + // Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates. + $certName = ID::unique(); + $renewDate = $certificates->issueCertificate($certName, $domain->get()); // Command succeeded, store all data into document - $logs = 'Certificate successfully generated.'; - $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB - - try { - // TEMP: add custom hostnames to cloudflare - $this->addCustomHostnameToRegistrar($project, $domain->get()); - } catch (\Throwable $th) { - Console::error('Failed to add custom hostname to registrar: ' . $th->getMessage()); - } + $certificate->setAttribute('logs', 'Certificate successfully generated.'); // Update certificate info stored in database - $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get())); + $certificate->setAttribute('renewDate', $renewDate); $certificate->setAttribute('attempts', 0); $certificate->setAttribute('issueDate', DateTime::now()); $success = true; @@ -194,7 +173,7 @@ class Certificates extends Action $attempts = $certificate->getAttribute('attempts', 0) + 1; $certificate->setAttribute('attempts', $attempts); - // Store cuttent time as renew date to ensure another attempt in next maintenance cycle + // Store current time as renew date to ensure another attempt in next maintenance cycle. $certificate->setAttribute('renewDate', DateTime::now()); // Send email to security email @@ -210,35 +189,6 @@ class Certificates extends Action } } - /** - * Add custom hostname to Cloudflare registrar - * - * @param Document $project - * @param string $hostname - * @return void - * @throws Exception - */ - private function addCustomHostnameToRegistrar(Document $project, string $hostname): void - { - $client = new Client(); - $client - ->addHeader('content-type', Client::CONTENT_TYPE_APPLICATION_JSON) - ->addHeader('Authorization', 'Bearer ' . System::getEnv('_APP_SYSTEM_CLOUDFLARE_TOKEN')); - - $response = $client->fetch("https://api.cloudflare.com/client/v4/zones/b2d0e62383d3c0f6299efab107af2c7a/custom_hostnames", Client::METHOD_POST, [ - 'hostname' => $hostname, - 'ssl' => [ - "method" => "http", - "type" => "dv", - "wildcard" => false - ] - ]); - - if ($response->getStatusCode() !== 201) { - throw new Exception('Failed to add custom hostname to Cloudflare: ' . $response->getBody()); - } - } - /** * Save certificate data into database. * @@ -287,8 +237,8 @@ class Certificates extends Action } /** - * Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check: - * - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt) + * Internal domain validation functionality to prevent unnecessary attempts. We check: + * - Domain needs to be public and valid (prevents NFT domains that are not supported) * - Domain must have proper DNS record * * @param Domain $domain Domain which we validate @@ -334,136 +284,6 @@ class Certificates extends Action } } - /** - * Reads expiry date of certificate from file and decides if renewal is required or not. - * - * @param string $domain Domain for which we check certificate file - * @return bool True, if certificate needs to be renewed - * @throws Exception - */ - private function isRenewRequired(string $domain, Log $log): bool - { - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; - if (\file_exists($certPath)) { - $validTo = null; - - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? 0; - - if (empty($validTo)) { - $log->addTag('certificateDomain', $domain); - throw new Exception('Unable to read certificate file (cert.pem).'); - } - - // LetsEncrypt allows renewal 30 days before expiry - $expiryInAdvance = (60 * 60 * 24 * 30); - if ($validTo - $expiryInAdvance > \time()) { - $log->addTag('certificateDomain', $domain); - $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData)); - return false; - } - } - - return true; - } - - /** - * LetsEncrypt communication to issue certificate (using certbot CLI) - * - * @param string $folder Folder into which certificates should be generated - * @param string $domain Domain to generate certificate for - * @return array Named array with keys 'stdout' and 'stderr', both string - * @throws Exception - */ - private function issueCertificate(string $folder, string $domain, string $email): array - { - $stdout = ''; - $stderr = ''; - - $staging = (App::isProduction()) ? '' : ' --dry-run'; - $exit = Console::execute("certbot certonly -v --webroot --noninteractive --agree-tos{$staging}" - . " --email " . $email - . " --cert-name " . $folder - . " -w " . APP_STORAGE_CERTIFICATES - . " -d {$domain}", '', $stdout, $stderr); - - // Unexpected error, usually 5XX, API limits, ... - if ($exit !== 0) { - throw new Exception('Failed to issue a certificate with message: ' . $stderr); - } - - return [ - 'stdout' => $stdout, - 'stderr' => $stderr - ]; - } - - /** - * Read new renew date from certificate file generated by Let's Encrypt - * - * @param string $domain Domain which certificate was generated for - * @return string - * @throws \Utopia\Database\Exception - */ - private function getRenewDate(string $domain): string - { - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? null; - $dt = (new \DateTime())->setTimestamp($validTo); - return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days - } - - /** - * Method to take files from Let's Encrypt, and put it into Traefik. - * - * @param string $domain Domain which certificate was generated for - * @param string $folder Folder in which certificates were generated - * @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error - * @return void - * @throws Exception - */ - private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void - { - - // Prepare folder in storage for domain - $path = APP_STORAGE_CERTIFICATES . '/' . $domain; - if (!\is_readable($path)) { - if (!\mkdir($path, 0755, true)) { - throw new Exception('Failed to create path for certificate.'); - } - } - - // Move generated files - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - $config = \implode(PHP_EOL, [ - "tls:", - " certificates:", - " - certFile: /storage/certificates/{$domain}/fullchain.pem", - " keyFile: /storage/certificates/{$domain}/privkey.pem" - ]); - - // Save configuration into Traefik using our new cert files - if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { - throw new Exception('Failed to save Traefik configuration.'); - } - } - /** * Method to make sure information about error is delivered to admnistrator. * @@ -536,6 +356,10 @@ class Certificates extends Action $project = $dbForConsole->getDocument('projects', $projectId); + if ($project->isEmpty()) { + return; + } + /** Trigger Webhook */ $ruleModel = new Rule(); $queueForEvents diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index fe81114ea0..763967bb56 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -354,11 +354,9 @@ class Databases extends Action } } finally { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); - $dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); - $dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); } } } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index ddab8d0cf8..e4412d0718 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Auth\Auth; +use Appwrite\Certificates\Adapter as CertificatesAdapter; use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; @@ -50,18 +51,22 @@ class Deletes extends Action ->inject('deviceForFunctions') ->inject('deviceForBuilds') ->inject('deviceForCache') + ->inject('certificates') ->inject('abuseRetention') ->inject('executionRetention') ->inject('auditRetention') ->inject('log') - ->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log)); + ->callback( + fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) + ); } /** * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -84,10 +89,10 @@ class Deletes extends Action case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { case DELETE_TYPE_PROJECTS: - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); + $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project); break; case DELETE_TYPE_DEPLOYMENTS: $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); @@ -102,7 +107,7 @@ class Deletes extends Action $this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project); break; case DELETE_TYPE_RULES: - $this->deleteRule($dbForConsole, $document); + $this->deleteRule($dbForConsole, $document, $certificates); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -110,7 +115,7 @@ class Deletes extends Action } break; case DELETE_TYPE_TEAM_PROJECTS: - $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $document); + $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $certificates, $document); break; case DELETE_TYPE_EXECUTIONS: $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); @@ -264,7 +269,7 @@ class Deletes extends Action MESSAGE_TYPE_EMAIL => 'emailTotal', MESSAGE_TYPE_SMS => 'smsTotal', MESSAGE_TYPE_PUSH => 'pushTotal', - default => throw new Exception('Invalid target provider type'), + default => throw new Exception('Invalid target CertificatesAdapter type'), }; $dbForProject->decreaseDocumentAttribute( 'topics', @@ -442,7 +447,7 @@ class Deletes extends Action * @throws Structure * @throws Exception */ - private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Document $document): void + private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void { $projects = $dbForConsole->find('projects', [ @@ -455,7 +460,7 @@ class Deletes extends Action $deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); $deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $project); $dbForConsole->deleteDocument('projects', $project->getId()); } } @@ -473,7 +478,7 @@ class Deletes extends Action * @throws Authorization * @throws DatabaseException */ - private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void + private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Document $document): void { $projectInternalId = $document->getInternalId(); $projectId = $document->getId(); @@ -530,8 +535,8 @@ class Deletes extends Action // Delete project and function rules $this->deleteByGroup('rules', [ Query::equal('projectInternalId', [$projectInternalId]) - ], $dbForConsole, function (Document $document) use ($dbForConsole) { - $this->deleteRule($dbForConsole, $document); + ], $dbForConsole, function (Document $document) use ($dbForConsole, $certificates) { + $this->deleteRule($dbForConsole, $document, $certificates); }); // Delete Keys @@ -732,7 +737,7 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void + private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -747,8 +752,8 @@ class Deletes extends Action Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('projectInternalId', [$project->getInternalId()]) - ], $dbForConsole, function (Document $document) use ($project, $dbForConsole) { - $this->deleteRule($dbForConsole, $document); + ], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $certificates) { + $this->deleteRule($dbForConsole, $document, $certificates); }); /** @@ -1034,21 +1039,10 @@ class Deletes extends Action * @param Document $document rule document * @return void */ - private function deleteRule(Database $dbForConsole, Document $document): void + private function deleteRule(Database $dbForConsole, Document $document, CertificatesAdapter $certificates): void { - $domain = $document->getAttribute('domain'); - $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; - $checkTraversal = realpath($directory) === $directory; - - if ($checkTraversal && is_dir($directory)) { - // Delete files, so Traefik is aware of change - array_map('unlink', glob($directory . '/*.*')); - rmdir($directory); - Console::info("Deleted certificate files for {$domain}"); - } else { - Console::info("No certificate files found for {$domain}"); - } + $certificates->deleteCertificate($domain); // Delete certificate document, so Appwrite is aware of change if (isset($document['certificateId'])) { diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 4b1926ed26..fdd885effa 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -16,7 +16,6 @@ use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; -use Utopia\Logger\Log; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; @@ -37,6 +36,8 @@ class Migrations extends Action protected Document $project; + protected $logError; + public static function getName(): string { return 'migrations'; @@ -52,14 +53,14 @@ class Migrations extends Action ->inject('message') ->inject('dbForProject') ->inject('dbForConsole') - ->inject('log') - ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, Log $log) => $this->action($message, $dbForProject, $dbForConsole, $log)); + ->inject('logError') + ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, callable $logError) => $this->action($message, $dbForProject, $dbForConsole, $logError)); } /** * @throws Exception */ - public function action(Message $message, Database $dbForProject, Database $dbForConsole, Log $log): void + public function action(Message $message, Database $dbForProject, Database $dbForConsole, callable $logError): void { $payload = $message->getPayload() ?? []; @@ -78,6 +79,7 @@ class Migrations extends Action $this->dbForProject = $dbForProject; $this->dbForConsole = $dbForConsole; $this->project = $project; + $this->logError = $logError; /** * Handle Event execution. @@ -86,10 +88,7 @@ class Migrations extends Action return; } - $log->addTag('migrationId', $migration->getId()); - $log->addTag('projectId', $project->getId()); - - $this->processMigration($migration, $log); + $this->processMigration($migration); } /** @@ -259,7 +258,7 @@ class Migrations extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function processMigration(Document $migration, Log $log): void + protected function processMigration(Document $migration): void { $project = $this->project; $projectDocument = $this->dbForConsole->getDocument('projects', $project->getId()); @@ -285,8 +284,6 @@ class Migrations extends Action $migration->setAttribute('status', 'processing'); $this->updateMigrationDocument($migration, $projectDocument); - $log->addTag('type', $migration->getAttribute('source')); - $source = $this->processSource($migration); $destination = $this->processDestination($migration, $tempAPIKey->getAttribute('secret')); @@ -324,7 +321,6 @@ class Migrations extends Action $errorMessages = []; foreach ($sourceErrors as $error) { - /** @var $sourceErrors $error */ $message = "Error occurred while fetching '{$error->getResourceName()}:{$error->getResourceId()}' from source with message: '{$error->getMessage()}'"; if ($error->getPrevious()) { $message .= " Message: ".$error->getPrevious()->getMessage() . " File: ".$error->getPrevious()->getFile() . " Line: ".$error->getPrevious()->getLine(); @@ -344,7 +340,6 @@ class Migrations extends Action } $migration->setAttribute('errors', $errorMessages); - $log->addExtra('migrationErrors', json_encode($errorMessages)); $this->updateMigrationDocument($migration, $projectDocument); return; @@ -359,7 +354,6 @@ class Migrations extends Action if (! $migration->isEmpty()) { $migration->setAttribute('status', 'failed'); $migration->setAttribute('stage', 'finished'); - $migration->setAttribute('errors', [$th->getMessage()]); return; } @@ -379,7 +373,6 @@ class Migrations extends Action } $migration->setAttribute('errors', $errorMessages); - $log->addTag('migrationErrors', json_encode($errorMessages)); } } finally { if (! $tempAPIKey->isEmpty()) { @@ -394,7 +387,23 @@ class Migrations extends Action $destination->error(); $source->error(); - throw new Exception('Migration failed'); + foreach ($source->getErrors() as $error) { + call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ + 'migrationId' => $migration->getId() ?? '', + 'source' => $migration->getAttribute('source') ?? '', + 'resourceName' => $error->getResourceName(), + 'resourceGroup' => $error->getResourceGroup() + ]); + } + + foreach ($destination->getErrors() as $error) { + call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ + 'migrationId' => $migration->getId() ?? '', + 'source' => $migration->getAttribute('source') ?? '', + 'resourceName' => $error->getResourceName(), + 'resourceGroup' => $error->getResourceGroup() + ]); + } } if ($migration->getAttribute('status', '') === 'completed') {