Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

test merge cloudflare certs

Browse Source
This commit is contained in:
Fabian Gruber
2024-11-20 14:15:19 +01:00
parent d32a1c35ab 5c4ba9d2fd
commit 733d1fde46
11 changed files with 295 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/config/function-templates.php
+1 -1
View File
@@ -11,7 +11,7 @@ const TEMPLATE_RUNTIMES = [
],
'DART' => [
'name' => 'dart',
'versions' => ['3.5', '3.3', '3.1', '3.0', '2.19', '2.18', '2.17', '2.16', '2.16']
'versions' => ['3.5', '3.3', '3.1', '3.0', '2.19', '2.18', '2.17', '2.16']
],
'GO' => [
'name' => 'go',
app/controllers/api/databases.php
+18 -14
View File
@@ -20,7 +20,6 @@ use Utopia\Audit\Audit;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception as DatabaseException;
use Utopia\Database\Exception\Authorization as AuthorizationException;
use Utopia\Database\Exception\Conflict as ConflictException;
use Utopia\Database\Exception\Duplicate as DuplicateException;
@@ -399,19 +398,29 @@ function updateAttribute(
}
if (!empty($newKey) && $key !== $newKey) {
// Delete attribute and recreate since we can't modify IDs
$original = clone $attribute;
$dbForProject->deleteDocument('attributes', $attribute->getId());
$originalUid = $attribute->getId();
$attribute
->setAttribute('$id', ID::custom($db->getInternalId() . '_' . $collection->getInternalId() . '_' . $newKey))
->setAttribute('key', $newKey);
try {
$attribute = $dbForProject->createDocument('attributes', $attribute);
} catch (DatabaseException|PDOException) {
$attribute = $dbForProject->createDocument('attributes', $original);
$dbForProject->updateDocument('attributes', $originalUid, $attribute);
/**
* @var Document $index
*/
foreach ($collection->getAttribute('indexes') as $index) {
/**
* @var string[] $attributes
*/
$attributes = $index->getAttribute('attributes', []);
$found = \array_search($key, $attributes);
if ($found !== false) {
$attributes[$found] = $newKey;
$index->setAttribute('attributes', $attributes);
$dbForProject->updateDocument('indexes', $index->getId(), $index);
}
}
} else {
$attribute = $dbForProject->updateDocument('attributes', $db->getInternalId() . '_' . $collection->getInternalId() . '_' . $key, $attribute);
@@ -2589,7 +2598,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
$attributeStatus = $oldAttributes[$attributeIndex]['status'];
$attributeType = $oldAttributes[$attributeIndex]['type'];
$attributeSize = $oldAttributes[$attributeIndex]['size'];
$attributeArray = $oldAttributes[$attributeIndex]['array'] ?? false;
if ($attributeType === Database::VAR_RELATIONSHIP) {
@@ -2603,10 +2611,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
$lengths[$i] = null;
if ($attributeType === Database::VAR_STRING) {
$lengths[$i] = $attributeSize; // set attribute size as index length only for strings
}
if ($attributeArray === true) {
$lengths[$i] = Database::ARRAY_INDEX_LENGTH;
$orders[$i] = null;
app/init.php
+10
View File
@@ -21,6 +21,7 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) {
use Ahc\Jwt\JWT;
use Ahc\Jwt\JWTException;
use Appwrite\Auth\Auth;
use Appwrite\Certificates\LetsEncrypt;
use Appwrite\Event\Audit;
use Appwrite\Event\Build;
use Appwrite\Event\Certificate;
@@ -1538,6 +1539,15 @@ App::setResource('cache', function (Group $pools) {
return new Cache(new Sharding($adapters));
}, ['pools']);
App::setResource('certificates', function () {
$email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'));
if (empty($email)) {
throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.');
}
return new LetsEncrypt($email);
});
App::setResource('deviceForLocal', function () {
return new Local();
});
app/worker.php
+44
View File
@@ -283,6 +283,50 @@ Server::setResource(
fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false
);
Server::setResource('logError', function (Registry $register, Document $project) {
return function (Throwable $error, string $namespace, string $action, ?array $extras) use ($register, $project) {
$logger = $register->get('logger');
if ($logger) {
$version = System::getEnv('_APP_VERSION', 'UNKNOWN');
$log = new Log();
$log->setNamespace($namespace);
$log->setServer(\gethostname());
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
$log->addTag('code', $error->getCode());
$log->addTag('verboseType', get_class($error));
$log->addTag('projectId', $project->getId() ?? '');
$log->addExtra('file', $error->getFile());
$log->addExtra('line', $error->getLine());
$log->addExtra('trace', $error->getTraceAsString());
foreach ($extras as $key => $value) {
$log->addExtra($key, $value);
}
$log->setAction($action);
$isProduction = System::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
try {
$responseCode = $logger->addLog($log);
Console::info('Error log pushed with status code: ' . $responseCode);
} catch (Throwable $th) {
Console::error('Error pushing log: ' . $th->getMessage());
}
}
Console::warning("Failed: {$error->getMessage()}");
Console::warning($error->getTraceAsString());
};
}, ['register', 'project']);
$pools = $register->get('pools');
$platform = new Appwrite();
src/Appwrite/Certificates/Adapter.php
+14
View File
@@ -0,0 +1,14 @@
<?php
namespace Appwrite\Certificates;
use Utopia\Logger\Log;
interface Adapter
{
public function issueCertificate(string $certName, string $domain): ?string;
public function isRenewRequired(string $domain, Log $log): bool;
public function deleteCertificate(string $domain): void;
}
src/Appwrite/Certificates/LetsEncrypt.php
+125
View File
@@ -0,0 +1,125 @@
<?php
namespace Appwrite\Certificates;
use Exception;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Logger\Log;
class LetsEncrypt implements Adapter
{
private string $email;
public function __construct(string $email)
{
$this->email = $email;
}
public function issueCertificate(string $certName, string $domain): ?string
{
$stdout = '';
$stderr = '';
$staging = (App::isProduction()) ? '' : ' --dry-run';
$exit = Console::execute(
"certbot certonly -v --webroot --noninteractive --agree-tos{$staging}"
. " --email " . $this->email
. " --cert-name " . $certName
. " -w " . APP_STORAGE_CERTIFICATES
. " -d {$domain}",
'',
$stdout,
$stderr
);
// Unexpected error, usually 5XX, API limits, ...
if ($exit !== 0) {
throw new Exception('Failed to issue a certificate with message: ' . $stderr);
}
// Prepare folder in storage for domain
$path = APP_STORAGE_CERTIFICATES . '/' . $domain;
if (!\is_readable($path)) {
if (!\mkdir($path, 0755, true)) {
throw new Exception('Failed to create path for certificate.');
}
}
// Move generated files
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) {
throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) {
throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) {
throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) {
throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
$config = \implode(PHP_EOL, [
"tls:",
" certificates:",
" - certFile: /storage/certificates/{$domain}/fullchain.pem",
" keyFile: /storage/certificates/{$domain}/privkey.pem"
]);
// Save configuration into Traefik using our new cert files
if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) {
throw new Exception('Failed to save Traefik configuration.');
}
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? null;
$dt = (new \DateTime())->setTimestamp($validTo);
return DateTime::addSeconds($dt, -60 * 60 * 24 * 30);
}
public function isRenewRequired(string $domain, Log $log): bool
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
if (\file_exists($certPath)) {
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? 0;
if (empty($validTo)) {
$log->addTag('certificateDomain', $domain);
throw new Exception('Unable to read certificate file (cert.pem).');
}
// LetsEncrypt allows renewal 30 days before expiry
$expiryInAdvance = (60 * 60 * 24 * 30);
if ($validTo - $expiryInAdvance > \time()) {
$log->addTag('certificateDomain', $domain);
$log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
return false;
}
}
return true;
}
public function deleteCertificate(string $domain): void
{
$directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
$checkTraversal = realpath($directory) === $directory;
if ($checkTraversal && is_dir($directory)) {
// Delete files, so Traefik is aware of change
array_map('unlink', glob($directory . '/*.*'));
rmdir($directory);
Console::info("Deleted certificate files for {$domain}");
} else {
Console::info("No certificate files found for {$domain}");
}
}
}
src/Appwrite/Platform/Tasks/Maintenance.php
+1
View File
@@ -149,6 +149,7 @@ class Maintenance extends Action
$certificates = $dbForConsole->find('certificates', [
Query::lessThan('attempts', 5), // Maximum 5 attempts
Query::isNotNull('renewDate'),
Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew)
Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains)
]);
src/Appwrite/Platform/Workers/Certificates.php
+35 -211
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Certificates\Adapter as CertificatesAdapter;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Mail;
@@ -11,7 +12,6 @@ use Appwrite\Template\Template;
use Appwrite\Utopia\Response\Model\Rule;
use Exception;
use Throwable;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
@@ -22,7 +22,6 @@ use Utopia\Database\Exception\Structure;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Domains\Domain;
use Utopia\Fetch\Client;
use Utopia\Locale\Locale;
use Utopia\Logger\Log;
use Utopia\Platform\Action;
@@ -44,28 +43,31 @@ class Certificates extends Action
$this
->desc('Certificates worker')
->inject('message')
->inject('project')
->inject('dbForConsole')
->inject('queueForMails')
->inject('queueForEvents')
->inject('queueForFunctions')
->inject('log')
->callback(fn (Message $message, Document $project, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $project, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log));
->inject('certificates')
->callback(
fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates) =>
$this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates)
);
}
/**
* @param Message $message
* @param Document $project
* @param Database $dbForConsole
* @param Mail $queueForMails
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param Log $log
* @param CertificatesAdapter $certificates
* @return void
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Document $project, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void
public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates): void
{
$payload = $message->getPayload() ?? [];
@@ -79,7 +81,7 @@ class Certificates extends Action
$log->addTag('domain', $domain->get());
$this->execute($domain, $project, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck);
$this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates, $skipRenewCheck);
}
/**
@@ -88,24 +90,24 @@ class Certificates extends Action
* @param Mail $queueForMails
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param CertificatesAdapter $certificates
* @param bool $skipRenewCheck
* @return void
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
private function execute(Domain $domain, Document $project, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void
private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void
{
/**
* 1. Read arguments and validate domain
* 2. Get main domain
* 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain)
* 4. Validate security email. Cannot be empty, required by LetsEncrypt
* 5. Validate renew date with certificate file, unless requested to skip by parameter
* 6. Issue a certificate using certbot CLI
* 7. Update 'log' attribute on certificate document with Certbot message
* 8. Create storage folder for certificate, if not ready already
* 9. Move certificates from Certbot location to our Storage
* 10. Create/Update our Storage with new Traefik config with new certificate paths
* 4. Validate renew date with certificate file, unless requested to skip by parameter
* 5. Issue a certificate using certbot CLI
* 6. Update 'log' attribute on certificate document with Certbot message
* 7. Create storage folder for certificate, if not ready already
* 8. Move certificates from Certbot location to our Storage
* 9. Create/Update our Storage with new Traefik config with new certificate paths
* 11. Read certificate file and update 'renewDate' on certificate document
* 12. Update 'issueDate' and 'attempts' on certificate
*
@@ -122,7 +124,7 @@ class Certificates extends Action
* 2. Save document to database
* 3. Update all domains documents with current certificate ID
*
* Note: Renewals are checked and scheduled from maintenence worker
* Note: Renewals are checked and scheduled from maintenance worker
*/
// Get current certificate
@@ -137,50 +139,27 @@ class Certificates extends Action
$success = false;
try {
// Email for alerts is required by LetsEncrypt
$email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'));
if (empty($email)) {
throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue an SSL certificate.');
}
// Validate domain and DNS records. Skip if job is forced
if (!$skipRenewCheck) {
$mainDomain = $this->getMainDomain();
$isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain;
$this->validateDomain($domain, $isMainDomain, $log);
// If certificate exists already, double-check expiry date. Skip if job is forced
if (!$certificates->isRenewRequired($domain->get(), $log)) {
throw new Exception('Renew isn\'t required.');
}
}
// If certificate exists already, double-check expiry date. Skip if job is forced
if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) {
throw new Exception('Renew isn\'t required.');
}
// Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate
$folder = ID::unique();
try {
// Generate certificate files using Let's Encrypt
$letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email);
// Give certificates to Traefik
$this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData);
} catch (\Throwable $th) {
Console::error('Failed to generate Lets Encrypt certificate');
}
// Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates.
$certName = ID::unique();
$renewDate = $certificates->issueCertificate($certName, $domain->get());
// Command succeeded, store all data into document
$logs = 'Certificate successfully generated.';
$certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB
try {
// TEMP: add custom hostnames to cloudflare
$this->addCustomHostnameToRegistrar($project, $domain->get());
} catch (\Throwable $th) {
Console::error('Failed to add custom hostname to registrar: ' . $th->getMessage());
}
$certificate->setAttribute('logs', 'Certificate successfully generated.');
// Update certificate info stored in database
$certificate->setAttribute('renewDate', $this->getRenewDate($domain->get()));
$certificate->setAttribute('renewDate', $renewDate);
$certificate->setAttribute('attempts', 0);
$certificate->setAttribute('issueDate', DateTime::now());
$success = true;
@@ -194,7 +173,7 @@ class Certificates extends Action
$attempts = $certificate->getAttribute('attempts', 0) + 1;
$certificate->setAttribute('attempts', $attempts);
// Store cuttent time as renew date to ensure another attempt in next maintenance cycle
// Store current time as renew date to ensure another attempt in next maintenance cycle.
$certificate->setAttribute('renewDate', DateTime::now());
// Send email to security email
@@ -210,35 +189,6 @@ class Certificates extends Action
}
}
/**
* Add custom hostname to Cloudflare registrar
*
* @param Document $project
* @param string $hostname
* @return void
* @throws Exception
*/
private function addCustomHostnameToRegistrar(Document $project, string $hostname): void
{
$client = new Client();
$client
->addHeader('content-type', Client::CONTENT_TYPE_APPLICATION_JSON)
->addHeader('Authorization', 'Bearer ' . System::getEnv('_APP_SYSTEM_CLOUDFLARE_TOKEN'));
$response = $client->fetch("https://api.cloudflare.com/client/v4/zones/b2d0e62383d3c0f6299efab107af2c7a/custom_hostnames", Client::METHOD_POST, [
'hostname' => $hostname,
'ssl' => [
"method" => "http",
"type" => "dv",
"wildcard" => false
]
]);
if ($response->getStatusCode() !== 201) {
throw new Exception('Failed to add custom hostname to Cloudflare: ' . $response->getBody());
}
}
/**
* Save certificate data into database.
*
@@ -287,8 +237,8 @@ class Certificates extends Action
}
/**
* Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check:
* - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt)
* Internal domain validation functionality to prevent unnecessary attempts. We check:
* - Domain needs to be public and valid (prevents NFT domains that are not supported)
* - Domain must have proper DNS record
*
* @param Domain $domain Domain which we validate
@@ -334,136 +284,6 @@ class Certificates extends Action
}
}
/**
* Reads expiry date of certificate from file and decides if renewal is required or not.
*
* @param string $domain Domain for which we check certificate file
* @return bool True, if certificate needs to be renewed
* @throws Exception
*/
private function isRenewRequired(string $domain, Log $log): bool
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
if (\file_exists($certPath)) {
$validTo = null;
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? 0;
if (empty($validTo)) {
$log->addTag('certificateDomain', $domain);
throw new Exception('Unable to read certificate file (cert.pem).');
}
// LetsEncrypt allows renewal 30 days before expiry
$expiryInAdvance = (60 * 60 * 24 * 30);
if ($validTo - $expiryInAdvance > \time()) {
$log->addTag('certificateDomain', $domain);
$log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
return false;
}
}
return true;
}
/**
* LetsEncrypt communication to issue certificate (using certbot CLI)
*
* @param string $folder Folder into which certificates should be generated
* @param string $domain Domain to generate certificate for
* @return array Named array with keys 'stdout' and 'stderr', both string
* @throws Exception
*/
private function issueCertificate(string $folder, string $domain, string $email): array
{
$stdout = '';
$stderr = '';
$staging = (App::isProduction()) ? '' : ' --dry-run';
$exit = Console::execute("certbot certonly -v --webroot --noninteractive --agree-tos{$staging}"
. " --email " . $email
. " --cert-name " . $folder
. " -w " . APP_STORAGE_CERTIFICATES
. " -d {$domain}", '', $stdout, $stderr);
// Unexpected error, usually 5XX, API limits, ...
if ($exit !== 0) {
throw new Exception('Failed to issue a certificate with message: ' . $stderr);
}
return [
'stdout' => $stdout,
'stderr' => $stderr
];
}
/**
* Read new renew date from certificate file generated by Let's Encrypt
*
* @param string $domain Domain which certificate was generated for
* @return string
* @throws \Utopia\Database\Exception
*/
private function getRenewDate(string $domain): string
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? null;
$dt = (new \DateTime())->setTimestamp($validTo);
return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days
}
/**
* Method to take files from Let's Encrypt, and put it into Traefik.
*
* @param string $domain Domain which certificate was generated for
* @param string $folder Folder in which certificates were generated
* @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error
* @return void
* @throws Exception
*/
private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void
{
// Prepare folder in storage for domain
$path = APP_STORAGE_CERTIFICATES . '/' . $domain;
if (!\is_readable($path)) {
if (!\mkdir($path, 0755, true)) {
throw new Exception('Failed to create path for certificate.');
}
}
// Move generated files
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) {
throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) {
throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) {
throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) {
throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
$config = \implode(PHP_EOL, [
"tls:",
" certificates:",
" - certFile: /storage/certificates/{$domain}/fullchain.pem",
" keyFile: /storage/certificates/{$domain}/privkey.pem"
]);
// Save configuration into Traefik using our new cert files
if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) {
throw new Exception('Failed to save Traefik configuration.');
}
}
/**
* Method to make sure information about error is delivered to admnistrator.
*
@@ -536,6 +356,10 @@ class Certificates extends Action
$project = $dbForConsole->getDocument('projects', $projectId);
if ($project->isEmpty()) {
return;
}
/** Trigger Webhook */
$ruleModel = new Rule();
$queueForEvents
src/Appwrite/Platform/Workers/Databases.php
-2
View File
@@ -354,11 +354,9 @@ class Databases extends Action
}
} finally {
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
$dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) {
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId());
$dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId());
}
}
}
src/Appwrite/Platform/Workers/Deletes.php
+22 -28
View File
@@ -3,6 +3,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Auth\Auth;
use Appwrite\Certificates\Adapter as CertificatesAdapter;
use Appwrite\Extend\Exception;
use Executor\Executor;
use Throwable;
@@ -50,18 +51,22 @@ class Deletes extends Action
->inject('deviceForFunctions')
->inject('deviceForBuilds')
->inject('deviceForCache')
->inject('certificates')
->inject('abuseRetention')
->inject('executionRetention')
->inject('auditRetention')
->inject('log')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log));
->callback(
fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) =>
$this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log)
);
}
/**
* @throws Exception
* @throws Throwable
*/
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void
{
$payload = $message->getPayload() ?? [];
@@ -84,10 +89,10 @@ class Deletes extends Action
case DELETE_TYPE_DOCUMENT:
switch ($document->getCollection()) {
case DELETE_TYPE_PROJECTS:
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document);
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document);
break;
case DELETE_TYPE_FUNCTIONS:
$this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project);
$this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project);
break;
case DELETE_TYPE_DEPLOYMENTS:
$this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project);
@@ -102,7 +107,7 @@ class Deletes extends Action
$this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project);
break;
case DELETE_TYPE_RULES:
$this->deleteRule($dbForConsole, $document);
$this->deleteRule($dbForConsole, $document, $certificates);
break;
default:
Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
@@ -110,7 +115,7 @@ class Deletes extends Action
}
break;
case DELETE_TYPE_TEAM_PROJECTS:
$this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $document);
$this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $certificates, $document);
break;
case DELETE_TYPE_EXECUTIONS:
$this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
@@ -264,7 +269,7 @@ class Deletes extends Action
MESSAGE_TYPE_EMAIL => 'emailTotal',
MESSAGE_TYPE_SMS => 'smsTotal',
MESSAGE_TYPE_PUSH => 'pushTotal',
default => throw new Exception('Invalid target provider type'),
default => throw new Exception('Invalid target CertificatesAdapter type'),
};
$dbForProject->decreaseDocumentAttribute(
'topics',
@@ -442,7 +447,7 @@ class Deletes extends Action
* @throws Structure
* @throws Exception
*/
private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Document $document): void
private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void
{
$projects = $dbForConsole->find('projects', [
@@ -455,7 +460,7 @@ class Deletes extends Action
$deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId());
$deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId());
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project);
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $project);
$dbForConsole->deleteDocument('projects', $project->getId());
}
}
@@ -473,7 +478,7 @@ class Deletes extends Action
* @throws Authorization
* @throws DatabaseException
*/
private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void
private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Document $document): void
{
$projectInternalId = $document->getInternalId();
$projectId = $document->getId();
@@ -530,8 +535,8 @@ class Deletes extends Action
// Delete project and function rules
$this->deleteByGroup('rules', [
Query::equal('projectInternalId', [$projectInternalId])
], $dbForConsole, function (Document $document) use ($dbForConsole) {
$this->deleteRule($dbForConsole, $document);
], $dbForConsole, function (Document $document) use ($dbForConsole, $certificates) {
$this->deleteRule($dbForConsole, $document, $certificates);
});
// Delete Keys
@@ -732,7 +737,7 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void
private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
@@ -747,8 +752,8 @@ class Deletes extends Action
Query::equal('resourceType', ['function']),
Query::equal('resourceInternalId', [$functionInternalId]),
Query::equal('projectInternalId', [$project->getInternalId()])
], $dbForConsole, function (Document $document) use ($project, $dbForConsole) {
$this->deleteRule($dbForConsole, $document);
], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $certificates) {
$this->deleteRule($dbForConsole, $document, $certificates);
});
/**
@@ -1034,21 +1039,10 @@ class Deletes extends Action
* @param Document $document rule document
* @return void
*/
private function deleteRule(Database $dbForConsole, Document $document): void
private function deleteRule(Database $dbForConsole, Document $document, CertificatesAdapter $certificates): void
{
$domain = $document->getAttribute('domain');
$directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
$checkTraversal = realpath($directory) === $directory;
if ($checkTraversal && is_dir($directory)) {
// Delete files, so Traefik is aware of change
array_map('unlink', glob($directory . '/*.*'));
rmdir($directory);
Console::info("Deleted certificate files for {$domain}");
} else {
Console::info("No certificate files found for {$domain}");
}
$certificates->deleteCertificate($domain);
// Delete certificate document, so Appwrite is aware of change
if (isset($document['certificateId'])) {
src/Appwrite/Platform/Workers/Migrations.php
+25 -16
View File
@@ -16,7 +16,6 @@ use Utopia\Database\Exception\Conflict;
use Utopia\Database\Exception\Restricted;
use Utopia\Database\Exception\Structure;
use Utopia\Database\Helpers\ID;
use Utopia\Logger\Log;
use Utopia\Migration\Destination;
use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite;
use Utopia\Migration\Exception as MigrationException;
@@ -37,6 +36,8 @@ class Migrations extends Action
protected Document $project;
protected $logError;
public static function getName(): string
{
return 'migrations';
@@ -52,14 +53,14 @@ class Migrations extends Action
->inject('message')
->inject('dbForProject')
->inject('dbForConsole')
->inject('log')
->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, Log $log) => $this->action($message, $dbForProject, $dbForConsole, $log));
->inject('logError')
->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, callable $logError) => $this->action($message, $dbForProject, $dbForConsole, $logError));
}
/**
* @throws Exception
*/
public function action(Message $message, Database $dbForProject, Database $dbForConsole, Log $log): void
public function action(Message $message, Database $dbForProject, Database $dbForConsole, callable $logError): void
{
$payload = $message->getPayload() ?? [];
@@ -78,6 +79,7 @@ class Migrations extends Action
$this->dbForProject = $dbForProject;
$this->dbForConsole = $dbForConsole;
$this->project = $project;
$this->logError = $logError;
/**
* Handle Event execution.
@@ -86,10 +88,7 @@ class Migrations extends Action
return;
}
$log->addTag('migrationId', $migration->getId());
$log->addTag('projectId', $project->getId());
$this->processMigration($migration, $log);
$this->processMigration($migration);
}
/**
@@ -259,7 +258,7 @@ class Migrations extends Action
* @throws \Utopia\Database\Exception
* @throws Exception
*/
protected function processMigration(Document $migration, Log $log): void
protected function processMigration(Document $migration): void
{
$project = $this->project;
$projectDocument = $this->dbForConsole->getDocument('projects', $project->getId());
@@ -285,8 +284,6 @@ class Migrations extends Action
$migration->setAttribute('status', 'processing');
$this->updateMigrationDocument($migration, $projectDocument);
$log->addTag('type', $migration->getAttribute('source'));
$source = $this->processSource($migration);
$destination = $this->processDestination($migration, $tempAPIKey->getAttribute('secret'));
@@ -324,7 +321,6 @@ class Migrations extends Action
$errorMessages = [];
foreach ($sourceErrors as $error) {
/** @var $sourceErrors $error */
$message = "Error occurred while fetching '{$error->getResourceName()}:{$error->getResourceId()}' from source with message: '{$error->getMessage()}'";
if ($error->getPrevious()) {
$message .= " Message: ".$error->getPrevious()->getMessage() . " File: ".$error->getPrevious()->getFile() . " Line: ".$error->getPrevious()->getLine();
@@ -344,7 +340,6 @@ class Migrations extends Action
}
$migration->setAttribute('errors', $errorMessages);
$log->addExtra('migrationErrors', json_encode($errorMessages));
$this->updateMigrationDocument($migration, $projectDocument);
return;
@@ -359,7 +354,6 @@ class Migrations extends Action
if (! $migration->isEmpty()) {
$migration->setAttribute('status', 'failed');
$migration->setAttribute('stage', 'finished');
$migration->setAttribute('errors', [$th->getMessage()]);
return;
}
@@ -379,7 +373,6 @@ class Migrations extends Action
}
$migration->setAttribute('errors', $errorMessages);
$log->addTag('migrationErrors', json_encode($errorMessages));
}
} finally {
if (! $tempAPIKey->isEmpty()) {
@@ -394,7 +387,23 @@ class Migrations extends Action
$destination->error();
$source->error();
throw new Exception('Migration failed');
foreach ($source->getErrors() as $error) {
call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [
'migrationId' => $migration->getId() ?? '',
'source' => $migration->getAttribute('source') ?? '',
'resourceName' => $error->getResourceName(),
'resourceGroup' => $error->getResourceGroup()
]);
}
foreach ($destination->getErrors() as $error) {
call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [
'migrationId' => $migration->getId() ?? '',
'source' => $migration->getAttribute('source') ?? '',
'resourceName' => $error->getResourceName(),
'resourceGroup' => $error->getResourceGroup()
]);
}
}
if ($migration->getAttribute('status', '') === 'completed') {