From e2444734fdaaa777094247a32f76a1aa788bf80a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 7 Sep 2022 08:58:35 +0000 Subject: [PATCH 1/3] Fix certificate worker and retries --- app/workers/certificates.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 42ead45a26..e8071aa1e5 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -125,6 +125,9 @@ class CertificatesV1 extends Worker $attempts = $certificate->getAttribute('attempts', 0) + 1; $certificate->setAttribute('attempts', $attempts); + // Store cuttent time as renew date to ensure another attempt in next maintenance cycle + $certificate->setAttribute('renewDate', DateTime::now()); + // Send email to security email $this->notifyError($domain->get(), $e->getMessage(), $attempts); } finally { @@ -290,9 +293,9 @@ class CertificatesV1 extends Worker * * @param string $domain Domain which certificate was generated for * - * @return int + * @return string */ - private function getRenewDate(string $domain): int + private function getRenewDate(string $domain): string { $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; $certData = openssl_x509_parse(file_get_contents($certPath)); From e23752d4f3f37f9f8d006c62c73c1d68e42c1bba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 7 Sep 2022 10:44:57 +0000 Subject: [PATCH 2/3] Improve certbot communication --- app/workers/certificates.php | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/app/workers/certificates.php b/app/workers/certificates.php index e8071aa1e5..f932ba4bc3 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -8,6 +8,7 @@ use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\DateTime; +use Utopia\Database\ID; use Utopia\Database\Query; use Utopia\Domains\Domain; @@ -100,8 +101,11 @@ class CertificatesV1 extends Worker throw new Exception('Renew isn\'t required.'); } + // Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate + $folder = ID::unique(); + // Generate certificate files using Let's Encrypt - $letsEncryptData = $this->issueCertificate($domain->get(), $email); + $letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email); // Command succeeded, store all data into document // We store stderr too, because it may include warnings @@ -111,7 +115,7 @@ class CertificatesV1 extends Worker ])); // Give certificates to Traefik - $this->applyCertificateFiles($domain->get(), $letsEncryptData); + $this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData); // Update certificate info stored in database $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get())); @@ -262,11 +266,12 @@ class CertificatesV1 extends Worker /** * LetsEncrypt communication to issue certificate (using certbot CLI) * + * @param string $folder Folder into which certificates should be generated * @param string $domain Domain to generate certificate for * * @return array Named array with keys 'stdout' and 'stderr', both string */ - private function issueCertificate(string $domain, string $email): array + private function issueCertificate(string $folder, string $domain, string $email): array { $stdout = ''; $stderr = ''; @@ -274,6 +279,7 @@ class CertificatesV1 extends Worker $staging = (App::isProduction()) ? '' : ' --dry-run'; $exit = Console::execute("certbot certonly --webroot --noninteractive --agree-tos{$staging}" . " --email " . $email + . " --cert-name " . $folder . " -w " . APP_STORAGE_CERTIFICATES . " -d {$domain}", '', $stdout, $stderr); @@ -308,11 +314,12 @@ class CertificatesV1 extends Worker * Method to take files from Let's Encrypt, and put it into Traefik. * * @param string $domain Domain which certificate was generated for + * @param string $folder Folder in which certificates were generated * @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error * * @return void */ - private function applyCertificateFiles(string $domain, array $letsEncryptData): void + private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void { // Prepare folder in storage for domain $path = APP_STORAGE_CERTIFICATES . '/' . $domain; @@ -322,20 +329,20 @@ class CertificatesV1 extends Worker } } - // Move generated files from certbot into our storage - if (!@\rename('/etc/letsencrypt/live/' . $domain . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { + // Move generated files + if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); } - if (!@\rename('/etc/letsencrypt/live/' . $domain . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { + if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); } - if (!@\rename('/etc/letsencrypt/live/' . $domain . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { + if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); } - if (!@\rename('/etc/letsencrypt/live/' . $domain . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { + if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); } From 72068d5a93cf31575072856c0cc2f7e7e1fc7c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 7 Sep 2022 11:57:08 +0000 Subject: [PATCH 3/3] Disable 6th cert renew attempt --- app/tasks/maintenance.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 27a959ece4..42b5ed00dc 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -108,7 +108,7 @@ $cli $time = DateTime::now(); $certificates = $dbForConsole->find('certificates', [ - Query::lessThanEqual('attempts', 5), // Maximum 5 attempts + Query::lessThan('attempts', 5), // Maximum 5 attempts Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) ]);