Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

cert addition

Browse Source
This commit is contained in:
shimon
2023-01-05 15:05:38 +02:00
parent 8f230071c7
commit 4f5f983564
9 changed files with 412 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/controllers/api/edge.php
+5 -4
View File
@@ -33,7 +33,7 @@ App::post('/v1/edge/sync')
->desc('Purge cache keys')
->groups(['edge'])
->label('scope', 'public')
->param('keys', '', new ArrayList(new Assoc(), 500), 'Cache keys. an array containing alphanumerical cache keys')
->param('keys', '', new ArrayList(new Text(4056), 600), 'Cache keys. an array containing alphanumerical cache keys')
->inject('request')
->inject('response')
->inject('queueForCacheSyncIn')
@@ -43,11 +43,12 @@ App::post('/v1/edge/sync')
throw new Exception(Exception::KEY_NOT_FOUND);
}
foreach ($keys as $sync) {
foreach ($keys as $parts) {
$key = json_decode($parts);
$queueForCacheSyncIn
->enqueue([
'type' => $sync['type'],
'key' => $sync['key']
'type' => $key->type,
'key' => $key->key
]);
}
app/controllers/shared/api.php
+8
View File
@@ -408,6 +408,14 @@ App::shutdown()
}
}
// $queueForCacheSyncOut->enqueue([
// 'type' => 'certificate',
// 'key' => [
// 'domain' => 'appwrite.io',
// 'contents' => base64_encode(file_get_contents(APP_STORAGE_CERTIFICATES . '/appwrite.io.tar.gz')),
// ]
// ]);
$route = $utopia->match($request);
$requestParams = $route->getParamsValues();
$user = $audits->getUser();
app/worker.php
+19 -10
View File
@@ -2,6 +2,7 @@
require_once __DIR__ . '/init.php';
use Appwrite\Event\Certificate;
use Appwrite\Event\Func;
use Swoole\Runtime;
use Utopia\App;
@@ -11,7 +12,9 @@ use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Pools\Connection;
use Utopia\Queue\Adapter\Swoole;
use Utopia\Queue\Client;
use Utopia\Queue\Message;
use Utopia\Queue\Server;
use Utopia\Registry\Registry;
@@ -75,15 +78,21 @@ Server::setResource('cache', function (Registry $register) {
return new Cache(new Sharding($adapters));
}, ['register']);
Server::setResource('queueForFunctions', function (Registry $register) {
$pools = $register->get('pools');
return new Func(
$pools
->get('queue')
->pop()
->getResource()
);
}, ['register']);
Server::setResource('queue', function (Group $pools) {
return $pools->get('queue')->pop()->getResource();
}, ['pools']);
Server::setResource('queueForFunctions', function (Connection $queue) {
return new Func($queue);
}, ['queue']);
Server::setResource('queueForCertificates', function (Connection $queue) {
return new Certificate($queue);
}, ['queue']);
Server::setResource('queueForCacheSyncOut', function (Connection $queue) {
return new Client('v1-sync-out', $queue);
}, ['queue']);
Server::setResource('logger', function ($register) {
return $register->get('logger');
@@ -100,7 +109,7 @@ Server::setResource('pools', function ($register) {
$pools = $register->get('pools');
$connection = $pools->get('queue')->pop()->getResource();
$workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
$workerNumber =1;
if (empty(App::getEnv('QUEUE'))) {
throw new Exception('Please configure "QUEUE" environemnt variable.');
}
app/workers/certificates.php
+318 -293
View File
@@ -1,42 +1,36 @@
<?php
require_once __DIR__ . '/../worker.php';
use Appwrite\Event\Func;
use Appwrite\Event\Mail;
use Appwrite\Network\Validator\CNAME;
use Appwrite\Resque\Worker;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\DateTime;
use Utopia\Database\ID;
use Utopia\Database\Query;
use Utopia\Domains\Domain;
use Utopia\Queue\Client;
use Utopia\Queue\Message;
use Utopia\Queue\Server;
use Utopia\Database\DateTime;
require_once __DIR__ . '/../init.php';
Authorization::disable();
Authorization::setDefaultStatus(false);
Console::title('Certificates V1 Worker');
Console::success(APP_NAME . ' certificates worker v1 has started');
$database = null;
class CertificatesV1 extends Worker
{
/**
* Database connection shared across all methods of this file
*
* @var Database
*/
private Database $dbForConsole;
public function getName(): string
{
return "certificates";
}
public function init(): void
{
}
public function run(): void
{
Server::setResource('execute', function () {
return function (
Database $dbForConsole,
Document $document,
Domain $domain,
$queueForCacheSyncOut,
bool $skipRenewCheck = false,
) {
/**
* 1. Read arguments and validate domain
* 2. Get main domain
@@ -67,14 +61,8 @@ class CertificatesV1 extends Worker
* Note: Renewals are checked and scheduled from maintenence worker
*/
$this->dbForConsole = $this->getConsoleDB();
$skipCheck = $this->args['skipRenewCheck'] ?? false; // If true, we won't double-check expiry from cert file
$document = new Document($this->args['domain'] ?? []);
$domain = new Domain($document->getAttribute('domain', ''));
// Get current certificate
$certificate = $this->dbForConsole->findOne('certificates', [Query::equal('domain', [$domain->get()])]);
$certificate = $dbForConsole->findOne('certificates', [Query::equal('domain', [$domain->get()])]);
// If we don't have certificate for domain yet, let's create new document. At the end we save it
if (!$certificate) {
@@ -90,14 +78,14 @@ class CertificatesV1 extends Worker
}
// Validate domain and DNS records. Skip if job is forced
if (!$skipCheck) {
$mainDomain = $this->getMainDomain();
if (!$skipRenewCheck) {
$mainDomain = getMainDomain($dbForConsole);
$isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain;
$this->validateDomain($domain, $isMainDomain);
validateDomain($domain, $isMainDomain);
}
// If certificate exists already, double-check expiry date. Skip if job is forced
if (!$skipCheck && !$this->isRenewRequired($domain->get())) {
if (!$skipRenewCheck && !isRenewRequired($domain->get())) {
throw new Exception('Renew isn\'t required.');
}
@@ -105,7 +93,7 @@ class CertificatesV1 extends Worker
$folder = ID::unique();
// Generate certificate files using Let's Encrypt
$letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email);
$letsEncryptData = issueCertificate($folder, $domain->get(), $email);
// Command succeeded, store all data into document
// We store stderr too, because it may include warnings
@@ -115,12 +103,20 @@ class CertificatesV1 extends Worker
]));
// Give certificates to Traefik
$this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData);
applyCertificateFiles($folder, $domain->get(), $letsEncryptData);
// Update certificate info stored in database
$certificate->setAttribute('renewDate', $this->getRenewDate($domain->get()));
$certificate->setAttribute('renewDate', getRenewDate($domain->get()));
$certificate->setAttribute('attempts', 0);
$certificate->setAttribute('issueDate', DateTime::now());
$queueForCacheSyncOut->enqueue([
'type' => 'certificate',
'key' => [
'domain' => $domain,
'contents' => base64_encode(file_get_contents(APP_STORAGE_CERTIFICATES . '/' . $domain . '.tar.gz')),
]
]);
} catch (Throwable $e) {
// Set exception as log in certificate document
$certificate->setAttribute('log', $e->getMessage());
@@ -133,290 +129,319 @@ class CertificatesV1 extends Worker
$certificate->setAttribute('renewDate', DateTime::now());
// Send email to security email
$this->notifyError($domain->get(), $e->getMessage(), $attempts);
notifyError($domain->get(), $e->getMessage(), $attempts);
} finally {
// All actions result in new updatedAt date
$certificate->setAttribute('updated', DateTime::now());
// Save all changes we made to certificate document into database
$this->saveCertificateDocument($domain->get(), $certificate);
saveCertificateDocument($domain->get(), $certificate, $dbForConsole);
}
};
});
/**
* Save certificate data into database.
*
* @param string $domain Domain name that certificate is for
* @param Document $certificate Certificate document that we need to save
* @param Database $dbForConsole Database connection for console
*
* @return void
*/
function saveCertificateDocument(string $domain, Document $certificate, Database $dbForConsole): void
{
// Check if update or insert required
$certificateDocument = $dbForConsole->findOne('certificates', [Query::equal('domain', [$domain])]);
if (!empty($certificateDocument) && !$certificateDocument->isEmpty()) {
// Merge new data with current data
$certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy()));
$certificate = $dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate);
} else {
$certificate = $dbForConsole->createDocument('certificates', $certificate);
}
$certificateId = $certificate->getId();
updateDomainDocuments($certificateId, $domain, $dbForConsole);
}
/**
* Get main domain. Needed as we do different checks for main and non-main domains.
*
* @return null|string Returns main domain. If null, there is no main domain yet.
*/
function getMainDomain($dbForConsole): ?string
{
$envDomain = App::getEnv('_APP_DOMAIN', '');
if (!empty($envDomain) && $envDomain !== 'localhost') {
return $envDomain;
} else {
$domainDocument = $dbForConsole->findOne('domains', [Query::orderAsc('_id')]);
if ($domainDocument) {
return $domainDocument->getAttribute('domain');
}
}
public function shutdown(): void
{
return null;
}
/**
* Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check:
* - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt)
* - Domain must have proper DNS record
*
* @param Domain $domain Domain which we validate
* @param bool $isMainDomain In case of master domain, we look for different DNS configurations
*
* @return void
*/
function validateDomain(Domain $domain, bool $isMainDomain): void
{
if (empty($domain->get())) {
throw new Exception('Missing certificate domain.');
}
/**
* Save certificate data into database.
*
* @param string $domain Domain name that certificate is for
* @param Document $certificate Certificate document that we need to save
*
* @return void
*/
private function saveCertificateDocument(string $domain, Document $certificate): void
{
// Check if update or insert required
$certificateDocument = $this->dbForConsole->findOne('certificates', [Query::equal('domain', [$domain])]);
if (!empty($certificateDocument) && !$certificateDocument->isEmpty()) {
// Merge new data with current data
$certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy()));
$certificate = $this->dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate);
} else {
$certificate = $this->dbForConsole->createDocument('certificates', $certificate);
}
$certificateId = $certificate->getId();
$this->updateDomainDocuments($certificateId, $domain);
if (!$domain->isKnown() || $domain->isTest()) {
throw new Exception('Unknown public suffix for domain.');
}
/**
* Get main domain. Needed as we do different checks for main and non-main domains.
*
* @return null|string Returns main domain. If null, there is no main domain yet.
*/
private function getMainDomain(): ?string
{
$envDomain = App::getEnv('_APP_DOMAIN', '');
if (!empty($envDomain) && $envDomain !== 'localhost') {
return $envDomain;
} else {
$domainDocument = $this->dbForConsole->findOne('domains', [Query::orderAsc('_id')]);
if ($domainDocument) {
return $domainDocument->getAttribute('domain');
}
if (!$isMainDomain) {
// TODO: Would be awesome to also support A/AAAA records here. Maybe dry run?
// Validate if domain target is properly configured
$target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', ''));
if (!$target->isKnown() || $target->isTest()) {
throw new Exception('Unreachable CNAME target (' . $target->get() . '), please use a domain with a public suffix.');
}
return null;
// Verify domain with DNS records
$validator = new CNAME($target->get());
if (!$validator->isValid($domain->get())) {
throw new Exception('Failed to verify domain DNS records.');
}
} else {
// Main domain validation
// TODO: Would be awesome to check A/AAAA record here. Maybe dry run?
}
}
/**
* Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check:
* - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt)
* - Domain must have proper DNS record
*
* @param Domain $domain Domain which we validate
* @param bool $isMainDomain In case of master domain, we look for different DNS configurations
*
* @return void
*/
private function validateDomain(Domain $domain, bool $isMainDomain): void
{
if (empty($domain->get())) {
throw new Exception('Missing certificate domain.');
}
/**
* Reads expiry date of certificate from file and decides if renewal is required or not.
*
* @param string $domain Domain for which we check certificate file
*
* @return bool True, if certificate needs to be renewed
*/
function isRenewRequired(string $domain): bool
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
if (\file_exists($certPath)) {
$validTo = null;
if (!$domain->isKnown() || $domain->isTest()) {
throw new Exception('Unknown public suffix for domain.');
}
if (!$isMainDomain) {
// TODO: Would be awesome to also support A/AAAA records here. Maybe dry run?
// Validate if domain target is properly configured
$target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', ''));
if (!$target->isKnown() || $target->isTest()) {
throw new Exception('Unreachable CNAME target (' . $target->get() . '), please use a domain with a public suffix.');
}
// Verify domain with DNS records
$validator = new CNAME($target->get());
if (!$validator->isValid($domain->get())) {
throw new Exception('Failed to verify domain DNS records.');
}
} else {
// Main domain validation
// TODO: Would be awesome to check A/AAAA record here. Maybe dry run?
}
}
/**
* Reads expiry date of certificate from file and decides if renewal is required or not.
*
* @param string $domain Domain for which we check certificate file
*
* @return bool True, if certificate needs to be renewed
*/
private function isRenewRequired(string $domain): bool
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
if (\file_exists($certPath)) {
$validTo = null;
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? 0;
if (empty($validTo)) {
throw new Exception('Unable to read certificate file (cert.pem).');
}
// LetsEncrypt allows renewal 30 days before expiry
$expiryInAdvance = (60 * 60 * 24 * 30);
if ($validTo - $expiryInAdvance > \time()) {
return false;
}
}
return true;
}
/**
* LetsEncrypt communication to issue certificate (using certbot CLI)
*
* @param string $folder Folder into which certificates should be generated
* @param string $domain Domain to generate certificate for
*
* @return array Named array with keys 'stdout' and 'stderr', both string
*/
private function issueCertificate(string $folder, string $domain, string $email): array
{
$stdout = '';
$stderr = '';
$staging = (App::isProduction()) ? '' : ' --dry-run';
$exit = Console::execute("certbot certonly --webroot --noninteractive --agree-tos{$staging}"
. " --email " . $email
. " --cert-name " . $folder
. " -w " . APP_STORAGE_CERTIFICATES
. " -d {$domain}", '', $stdout, $stderr);
// Unexpected error, usually 5XX, API limits, ...
if ($exit !== 0) {
throw new Exception('Failed to issue a certificate with message: ' . $stderr);
}
return [
'stdout' => $stdout,
'stderr' => $stderr
];
}
/**
* Read new renew date from certificate file generated by Let's Encrypt
*
* @param string $domain Domain which certificate was generated for
*
* @return string
*/
private function getRenewDate(string $domain): string
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? null;
$dt = (new \DateTime())->setTimestamp($validTo);
return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days
}
$validTo = $certData['validTo_time_t'] ?? 0;
/**
* Method to take files from Let's Encrypt, and put it into Traefik.
*
* @param string $domain Domain which certificate was generated for
* @param string $folder Folder in which certificates were generated
* @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error
*
* @return void
*/
private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void
{
// Prepare folder in storage for domain
$path = APP_STORAGE_CERTIFICATES . '/' . $domain;
if (!\is_readable($path)) {
if (!\mkdir($path, 0755, true)) {
throw new Exception('Failed to create path for certificate.');
}
if (empty($validTo)) {
throw new Exception('Unable to read certificate file (cert.pem).');
}
// Move generated files
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) {
throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) {
throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) {
throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) {
throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
$config = \implode(PHP_EOL, [
"tls:",
" certificates:",
" - certFile: /storage/certificates/{$domain}/fullchain.pem",
" keyFile: /storage/certificates/{$domain}/privkey.pem"
]);
// Save configuration into Traefik using our new cert files
if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) {
throw new Exception('Failed to save Traefik configuration.');
// LetsEncrypt allows renewal 30 days before expiry
$expiryInAdvance = (60 * 60 * 24 * 30);
if ($validTo - $expiryInAdvance > \time()) {
return false;
}
}
/**
* Method to make sure information about error is delivered to admnistrator.
*
* @param string $domain Domain that caused the error
* @param string $errorMessage Verbose error message
* @param int $attempt How many times it failed already
*
* @return void
*/
private function notifyError(string $domain, string $errorMessage, int $attempt): void
{
// Log error into console
Console::warning('Cannot renew domain (' . $domain . ') on attempt no. ' . $attempt . ' certificate: ' . $errorMessage);
return true;
}
// Send mail to administratore mail
$mail = new Mail();
$mail
->setType(MAIL_TYPE_CERTIFICATE)
->setRecipient(App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'))
->setUrl('https://' . $domain)
->setLocale(App::getEnv('_APP_LOCALE', 'en'))
->setName('Appwrite Administrator')
->setPayload([
'domain' => $domain,
'error' => $errorMessage,
'attempt' => $attempt
])
->trigger();
/**
* LetsEncrypt communication to issue certificate (using certbot CLI)
*
* @param string $folder Folder into which certificates should be generated
* @param string $domain Domain to generate certificate for
*
* @return array Named array with keys 'stdout' and 'stderr', both string
*/
function issueCertificate(string $folder, string $domain, string $email): array
{
$stdout = '';
$stderr = '';
$staging = (App::isProduction()) ? '' : ' --dry-run';
$exit = Console::execute("certbot certonly --webroot --noninteractive --agree-tos{$staging}"
. " --email " . $email
. " --cert-name " . $folder
. " -w " . APP_STORAGE_CERTIFICATES
. " -d {$domain}", '', $stdout, $stderr);
// Unexpected error, usually 5XX, API limits, ...
if ($exit !== 0) {
throw new Exception('Failed to issue a certificate with message: ' . $stderr);
}
/**
* Update all existing domain documents so they have relation to correct certificate document.
* This solved issues:
* - when adding a domain for which there is already a certificate
* - when renew creates new document? It might?
* - overall makes it more reliable
*
* @param string $certificateId ID of a new or updated certificate document
* @param string $domain Domain that is affected by new certificate
*
* @return void
*/
private function updateDomainDocuments(string $certificateId, string $domain): void
{
$domains = $this->dbForConsole->find('domains', [
Query::equal('domain', [$domain]),
Query::limit(1000),
]);
return [
'stdout' => $stdout,
'stderr' => $stderr
];
}
foreach ($domains as $domainDocument) {
$domainDocument->setAttribute('updated', DateTime::now());
$domainDocument->setAttribute('certificateId', $certificateId);
/**
* Read new renew date from certificate file generated by Let's Encrypt
*
* @param string $domain Domain which certificate was generated for
*
* @return string
*/
function getRenewDate(string $domain): string
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? null;
$dt = (new \DateTime())->setTimestamp($validTo);
return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days
}
$this->dbForConsole->updateDocument('domains', $domainDocument->getId(), $domainDocument);
/**
* Method to take files from Let's Encrypt, and put it into Traefik.
*
* @param string $domain Domain which certificate was generated for
* @param string $folder Folder in which certificates were generated
* @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error
*
* @return void
*/
function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void
{
// Prepare folder in storage for domain
$path = APP_STORAGE_CERTIFICATES . '/' . $domain;
if (!\is_readable($path)) {
if (!\mkdir($path, 0755, true)) {
throw new Exception('Failed to create path for certificate.');
}
}
if ($domainDocument->getAttribute('projectId')) {
$this->dbForConsole->deleteCachedDocument('projects', $domainDocument->getAttribute('projectId'));
}
// Move generated files
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) {
throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) {
throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) {
throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) {
throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
$config = \implode(PHP_EOL, [
"tls:",
" certificates:",
" - certFile: /storage/certificates/{$domain}/fullchain.pem",
" keyFile: /storage/certificates/{$domain}/privkey.pem"
]);
// Save configuration into Traefik using our new cert files
if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) {
throw new Exception('Failed to save Traefik configuration.');
}
$stdout = '';
$stderr = '';
Console::execute('cd ' . APP_STORAGE_CERTIFICATES . " && tar --exclude $domain.tar.gz -czf $domain.tar.gz .", '', $stdout, $stderr);
}
/**
* Method to make sure information about error is delivered to admnistrator.
*
* @param string $domain Domain that caused the error
* @param string $errorMessage Verbose error message
* @param int $attempt How many times it failed already
*
* @return void
*/
function notifyError(string $domain, string $errorMessage, int $attempt): void
{
// Log error into console
Console::warning('Cannot renew domain (' . $domain . ') on attempt no. ' . $attempt . ' certificate: ' . $errorMessage);
// Send mail to administratore mail
$mail = new Mail();
$mail
->setType(MAIL_TYPE_CERTIFICATE)
->setRecipient(App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'))
->setUrl('https://' . $domain)
->setLocale(App::getEnv('_APP_LOCALE', 'en'))
->setName('Appwrite Administrator')
->setPayload([
'domain' => $domain,
'error' => $errorMessage,
'attempt' => $attempt
])
->trigger();
}
/**
* Update all existing domain documents so they have relation to correct certificate document.
* This solved issues:
* - when adding a domain for which there is already a certificate
* - when renew creates new document? It might?
* - overall makes it more reliable
*
* @param string $certificateId ID of a new or updated certificate document
* @param string $domain Domain that is affected by new certificate
* @param Database $dbForConsole Database instance for console
*
* @return void
*/
function updateDomainDocuments(string $certificateId, string $domain, Database $dbForConsole): void
{
$domains = $dbForConsole->find('domains', [
Query::equal('domain', [$domain]),
Query::limit(1000),
]);
foreach ($domains as $domainDocument) {
$domainDocument->setAttribute('updated', DateTime::now());
$domainDocument->setAttribute('certificateId', $certificateId);
$dbForConsole->updateDocument('domains', $domainDocument->getId(), $domainDocument);
if ($domainDocument->getAttribute('projectId')) {
$dbForConsole->deleteCachedDocument('projects', $domainDocument->getAttribute('projectId'));
}
}
}
$server->job()
->inject('message')
->inject('dbForConsole')
->inject('execute')
->action(function ($message, $dbForConsole, $execute, Client $queueForCacheSyncOut) use ($server) {
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
$document = new Document($payload['domain'] ?? []);
$domain = new Domain($document->getAttribute('domain', ''));
$skipRenewCheck = $payload['skipRenewCheck'] ?? false;
$execute(
dbForConsole: $dbForConsole,
document: $document,
domain: $domain,
queueForCacheSyncOut: $queueForCacheSyncOut,
skipRenewCheck: $skipRenewCheck,
);
});
$server->workerStart();
$server->start();
app/workers/sync-In.php
+23 -1
View File
@@ -36,6 +36,28 @@ $server->job()
options: $key['options']
);
break;
case 'certificate':
Console::log("[{$time}] Writing certificate for domain [{$key['domain']}]");
$path = APP_STORAGE_CERTIFICATES . '/__' . $key['domain'];
$filename = $key['domain'] . 'tar.gz';
if (!file_exists($path)) {
mkdir($path, 0755, true);
}
$result = file_put_contents($path . '/' . $filename, base64_decode($key['contents']));
if (empty($result)) {
Console::error('Can not write ' . $key['filename']);
break;
}
$stdout = '';
$stderr = '';
$result = Console::execute('cd ' . $path . ' && tar xvzf ' . $filename, '', $stdout, $stderr);
if ($result === 1) {
Console::error('Can not open ' . $filename);
}
break;
default:
break;
}
@@ -44,7 +66,7 @@ $server->job()
$server
->workerStart()
->action(function () {
Console::success("In [" . App::getEnv('_APP_REGION', 'nyc1') . "] edge cache purging worker Started");
Console::success("[" . App::getEnv('_APP_REGION', 'nyc1') . "] edge sync-in worker Started");
});
$server->start();
app/workers/sync-out.php
+30 -13
View File
@@ -34,24 +34,40 @@ const MAX_CURL_SEND_ATTEMPTS = 4;
/**
* @param string $url
* @param string $token
* @param array $payload
* @param array data
* @return int
*/
function call(string $url, string $token, array $payload): int
function call(string $url, string $token, array $data): int
{
$boundary = uniqid();
$delimiter = '-------------' . $boundary;
$payload = '';
$eol = "\r\n";
foreach ($data as $keys) {
$payload .= "--" . $delimiter . $eol
. 'Content-Disposition: form-data; name="keys[]"' . $eol . $eol
. json_encode($keys) . $eol;
}
$payload .= "--" . $delimiter . "--" . $eol;
var_dump($payload);
$status = 404;
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_HTTPHEADER, [
'Authorization: Bearer ' . $token,
'Content-Type: application/json'
'Content-type: multipart/form-data; boundary=' . $delimiter,
'Content-Length: ' . strlen($payload)
]);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_TIMEOUT, 5);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload));
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $payload);
curl_setopt($ch, CURLOPT_VERBOSE, true);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
for ($attempts = 0; $attempts < MAX_CURL_SEND_ATTEMPTS; $attempts++) {
$response = curl_exec($ch);
curl_exec($ch);
$status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if ($status === 200) {
@@ -66,29 +82,30 @@ function call(string $url, string $token, array $payload): int
return $status;
}
/**
* @throws Authorization
* @throws Structure
* @throws Exception|\Exception
*/
function handle($dbForConsole, $regions, $payload): void
function handle($dbForConsole, $regions, $data): void
{
$jwt = new JWT(App::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 600, 10);
$token = $jwt->encode([]);
foreach ($regions as $code => $region) {
var_dump($region);
$time = DateTime::now();
$status = call($region['domain'] . '/v1/edge/sync', $token, ['keys' => $payload]);
$status = call($region['domain'] . '/v1/edge/sync', $token, $data);
if ($status !== Response::STATUS_CODE_OK) {
Console::error("[{$time}] Request to {$code} has failed");
foreach ($payload as $sync) {
foreach ($data as $keys) {
$dbForConsole->createDocument('syncs', new Document([
'region' => App::getEnv('_APP_REGION'),
'target' => $code,
'type' => $sync['type'],
'key' => ['key' => $sync['key']],
'type' => $keys['type'],
'key' => ['key' => $keys['key']],
'status' => $status,
]));
}
@@ -160,7 +177,7 @@ $server
Console::log("[{$time}] Sending " . count($chunk) . " remains " . count($stack['keys']));
handle($dbForConsole, $stack['regions'], $chunk);
});
Console::success("Out [" . App::getEnv('_APP_REGION') . "] edge cache purging worker Started");
Console::success("[" . App::getEnv('_APP_REGION') . "] edge sync-out worker Started");
});
$server->start();
bin/worker-certificates
+1 -8
View File
@@ -1,10 +1,3 @@
#!/bin/sh
if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ]
then
REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}"
else
REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}"
fi
INTERVAL=1 QUEUE='v1-certificates' APP_INCLUDE='/usr/src/code/app/workers/certificates.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php
QUEUE=v1-certificates php /usr/src/code/app/workers/certificates.php $@
docker-compose.yml
+2
View File
@@ -266,6 +266,7 @@ services:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- appwrite-certificates:/storage/certificates:rw
- ./vendor/utopia-php/pools:/usr/src/code/vendor/utopia-php/pools
depends_on:
- mariadb
@@ -300,6 +301,7 @@ services:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- appwrite-certificates:/storage/certificates:rw
#- ./vendor/utopia-php/cache:/usr/src/code/vendor/utopia-php/cache
depends_on:
src/Appwrite/Event/Certificate.php
+6 -4
View File
@@ -2,17 +2,18 @@
namespace Appwrite\Event;
use Resque;
use Utopia\Database\Document;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
class Certificate extends Event
{
protected bool $skipRenewCheck = false;
protected ?Document $domain = null;
public function __construct()
public function __construct(protected Connection $connection)
{
parent::__construct(Event::CERTIFICATES_QUEUE_NAME, Event::CERTIFICATES_CLASS_NAME);
parent::__construct(Event::FUNCTIONS_QUEUE_NAME, Event::FUNCTIONS_CLASS_NAME);
}
/**
@@ -69,7 +70,8 @@ class Certificate extends Event
*/
public function trigger(): string|bool
{
return Resque::enqueue($this->queue, $this->class, [
$client = new Client($this->queue, $this->connection);
return $client->enqueue([
'project' => $this->project,
'domain' => $this->domain,
'skipRenewCheck' => $this->skipRenewCheck