Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement better cname validation logging

Browse Source
This commit is contained in:
Matej Bačo
2024-01-23 15:40:32 +00:00
parent d220c2bdd7
commit 5bd106afaf
9 changed files with 104 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/config/errors.php
+1
View File
@@ -680,6 +680,7 @@ return [
'name' => Exception::RULE_VERIFICATION_FAILED,
'description' => 'Domain verification failed. Please check if your DNS records are correct and try again.',
'code' => 401,
'publish' => true
],
Exception::PROJECT_SMTP_CONFIG_INVALID => [
'name' => Exception::PROJECT_SMTP_CONFIG_INVALID,
app/controllers/api/proxy.php
+10 -1
View File
@@ -14,6 +14,7 @@ use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Database\Validator\UID;
use Utopia\Domains\Domain;
use Utopia\Logger\Log;
use Utopia\Validator\Domain as ValidatorDomain;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
@@ -278,7 +279,8 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
->inject('queueForEvents')
->inject('project')
->inject('dbForConsole')
->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole) {
->inject('log')
->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole, Log $log) {
$rule = $dbForConsole->getDocument('rules', $ruleId);
if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) {
@@ -298,7 +300,14 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
$validator = new CNAME($target->get()); // Verify Domain with DNS records
$domain = new Domain($rule->getAttribute('domain', ''));
$validationStart = \microtime(true);
if (!$validator->isValid($domain->get())) {
$log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart));
$log->addTag('dnsDomain', $domain->get());
$error = $validator->getDnsResponse();
$log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error));
throw new Exception(Exception::RULE_VERIFICATION_FAILED);
}
app/controllers/general.php
+41 -50
View File
@@ -603,9 +603,8 @@ App::error()
->inject('response')
->inject('project')
->inject('logger')
->inject('loggerBreadcrumbs')
->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, array $loggerBreadcrumbs) {
->inject('log')
->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log) {
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
$route = $utopia->getRoute();
$publish = true;
@@ -615,54 +614,46 @@ App::error()
}
if ($logger && $publish) {
if ($error->getCode() >= 500 || $error->getCode() === 0) {
try {
/** @var Utopia\Database\Document $user */
$user = $utopia->getResource('user');
} catch (\Throwable $th) {
// All good, user is optional information for logger
}
$log = new Utopia\Logger\Log();
if (isset($user) && !$user->isEmpty()) {
$log->setUser(new User($user->getId()));
}
$log->setNamespace("http");
$log->setServer(\gethostname());
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
$log->addTag('database', $project->getAttribute('database', 'console'));
$log->addTag('method', $route->getMethod());
$log->addTag('url', $route->getPath());
$log->addTag('verboseType', get_class($error));
$log->addTag('code', $error->getCode());
$log->addTag('projectId', $project->getId());
$log->addTag('hostname', $request->getHostname());
$log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', '')));
$log->addExtra('file', $error->getFile());
$log->addExtra('line', $error->getLine());
$log->addExtra('trace', $error->getTraceAsString());
$log->addExtra('detailedTrace', $error->getTrace());
$log->addExtra('roles', Authorization::getRoles());
$action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
$log->setAction($action);
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
foreach ($loggerBreadcrumbs as $loggerBreadcrumb) {
$log->addBreadcrumb($loggerBreadcrumb);
}
$responseCode = $logger->addLog($log);
Console::info('Log pushed with status code: ' . $responseCode);
try {
/** @var Utopia\Database\Document $user */
$user = $utopia->getResource('user');
} catch (\Throwable $th) {
// All good, user is optional information for logger
}
if (isset($user) && !$user->isEmpty()) {
$log->setUser(new User($user->getId()));
}
$log->setNamespace("http");
$log->setServer(\gethostname());
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
$log->addTag('database', $project->getAttribute('database', 'console'));
$log->addTag('method', $route->getMethod());
$log->addTag('url', $route->getPath());
$log->addTag('verboseType', get_class($error));
$log->addTag('code', $error->getCode());
$log->addTag('projectId', $project->getId());
$log->addTag('hostname', $request->getHostname());
$log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', '')));
$log->addExtra('file', $error->getFile());
$log->addExtra('line', $error->getLine());
$log->addExtra('trace', $error->getTraceAsString());
$log->addExtra('detailedTrace', $error->getTrace());
$log->addExtra('roles', Authorization::getRoles());
$action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
$log->setAction($action);
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
$responseCode = $logger->addLog($log);
Console::info('Log pushed with status code: ' . $responseCode);
}
$code = $error->getCode();
app/http.php
+1 -6
View File
@@ -263,10 +263,9 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
// All good, user is optional information for logger
}
$loggerBreadcrumbs = $app->getResource("loggerBreadcrumbs");
$route = $app->getRoute();
$log = new Utopia\Logger\Log();
$log = $app->getResource("log");
if (isset($user) && !$user->isEmpty()) {
$log->setUser(new User($user->getId()));
@@ -298,10 +297,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
foreach ($loggerBreadcrumbs as $loggerBreadcrumb) {
$log->addBreadcrumb($loggerBreadcrumb);
}
$responseCode = $logger->addLog($log);
Console::info('Log pushed with status code: ' . $responseCode);
}
app/init.php
+2 -4
View File
@@ -76,6 +76,7 @@ use Appwrite\Hooks\Hooks;
use MaxMind\Db\Reader;
use PHPMailer\PHPMailer\PHPMailer;
use Swoole\Database\PDOProxy;
use Utopia\Logger\Log;
use Utopia\Queue;
use Utopia\Queue\Connection;
use Utopia\Storage\Storage;
@@ -864,6 +865,7 @@ foreach ($locales as $locale) {
]);
// Runtime Execution
App::setResource('log', fn() => new Log());
App::setResource('logger', function ($register) {
return $register->get('logger');
}, ['register']);
@@ -872,10 +874,6 @@ App::setResource('hooks', function ($register) {
return $register->get('hooks');
}, ['register']);
App::setResource('loggerBreadcrumbs', function () {
return [];
});
App::setResource('register', fn() => $register);
App::setResource('locale', fn() => new Locale(App::getEnv('_APP_LOCALE', 'en')));
src/Appwrite/Extend/Exception.php
+3 -1
View File
@@ -238,13 +238,15 @@ class Exception extends \Exception
protected string $type = '';
protected array $errors = [];
protected bool $publish = true;
protected bool $publish;
public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
{
$this->errors = Config::getParam('errors');
$this->type = $type;
$this->publish = !isset($code) ? true : $code >= 500 || $code === 0;
if (isset($this->errors[$type])) {
$this->code = $this->errors[$type]['code'];
$this->message = $this->errors[$type]['description'];
src/Appwrite/Network/Validator/CNAME.php
+14
View File
@@ -6,6 +6,11 @@ use Utopia\Validator;
class CNAME extends Validator
{
/**
* @var mixed
*/
protected mixed $dnsResponse;
/**
* @var string
*/
@@ -27,6 +32,14 @@ class CNAME extends Validator
return 'Invalid CNAME record';
}
/**
* @return mixed
*/
public function getDnsResponse(): mixed
{
return $this->dnsResponse;
}
/**
* Check if CNAME record target value matches selected target
*
@@ -42,6 +55,7 @@ class CNAME extends Validator
try {
$records = \dns_get_record($domain, DNS_CNAME);
$this->dnsResponse = $records;
} catch (\Throwable $th) {
return false;
}
src/Appwrite/Platform/Tasks/SSL.php
+7 -3
View File
@@ -7,6 +7,7 @@ use Appwrite\Event\Certificate;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Document;
use Utopia\Validator\Boolean;
use Utopia\Validator\Hostname;
class SSL extends Action
@@ -21,19 +22,22 @@ class SSL extends Action
$this
->desc('Validate server certificates')
->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true)
->param('skip-check', true, new Boolean(true), 'If DNS and renew check should be skipped. Defaults to true, and when true, all jobs will result in certificate generation attempt.', true)
->inject('queueForCertificates')
->callback(fn (string $domain, Certificate $queueForCertificates) => $this->action($domain, $queueForCertificates));
->callback(fn (string $domain, bool|string $skipCheck, Certificate $queueForCertificates) => $this->action($domain, $skipCheck, $queueForCertificates));
}
public function action(string $domain, Certificate $queueForCertificates): void
public function action(string $domain, bool|string $skipCheck, Certificate $queueForCertificates): void
{
$skipCheck = \strval($skipCheck) === 'true';
Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain);
$queueForCertificates
->setDomain(new Document([
'domain' => $domain
]))
->setSkipRenewCheck(true)
->setSkipRenewCheck($skipCheck)
->trigger();
}
}
src/Appwrite/Platform/Workers/Certificates.php
+25 -8
View File
@@ -23,6 +23,7 @@ use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Domains\Domain;
use Utopia\Locale\Locale;
use Utopia\Logger\Log;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
@@ -45,7 +46,8 @@ class Certificates extends Action
->inject('queueForMails')
->inject('queueForEvents')
->inject('queueForFunctions')
->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions));
->inject('log')
->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log));
}
/**
@@ -58,7 +60,7 @@ class Certificates extends Action
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions): void
public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void
{
$payload = $message->getPayload() ?? [];
@@ -70,7 +72,7 @@ class Certificates extends Action
$domain = new Domain($document->getAttribute('domain', ''));
$skipRenewCheck = $payload['skipRenewCheck'] ?? false;
$this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $skipRenewCheck);
$this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck);
}
/**
@@ -84,7 +86,7 @@ class Certificates extends Action
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, bool $skipRenewCheck = false): void
private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void
{
/**
* 1. Read arguments and validate domain
@@ -138,11 +140,11 @@ class Certificates extends Action
if (!$skipRenewCheck) {
$mainDomain = $this->getMainDomain();
$isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain;
$this->validateDomain($domain, $isMainDomain);
$this->validateDomain($domain, $isMainDomain, $log);
}
// If certificate exists already, double-check expiry date. Skip if job is forced
if (!$skipRenewCheck && !$this->isRenewRequired($domain->get())) {
if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) {
throw new Exception('Renew isn\'t required.');
}
@@ -167,6 +169,7 @@ class Certificates extends Action
$success = true;
} catch (Throwable $e) {
$logs = $e->getMessage();
$finalException = $e;
// Set exception as log in certificate document
$certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB
@@ -187,6 +190,10 @@ class Certificates extends Action
// Save all changes we made to certificate document into database
$this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForConsole, $queueForEvents, $queueForFunctions);
}
if (isset($finalException)) {
throw $finalException;
}
}
/**
@@ -247,7 +254,7 @@ class Certificates extends Action
* @return void
* @throws Exception
*/
private function validateDomain(Domain $domain, bool $isMainDomain): void
private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void
{
if (empty($domain->get())) {
throw new Exception('Missing certificate domain.');
@@ -267,8 +274,15 @@ class Certificates extends Action
}
// Verify domain with DNS records
$validationStart = \microtime(true);
$validator = new CNAME($target->get());
if (!$validator->isValid($domain->get())) {
$log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart));
$log->addTag('dnsDomain', $domain->get());
$error = $validator->getDnsResponse();
$log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error));
throw new Exception('Failed to verify domain DNS records.');
}
} else {
@@ -284,7 +298,7 @@ class Certificates extends Action
* @return bool True, if certificate needs to be renewed
* @throws Exception
*/
private function isRenewRequired(string $domain): bool
private function isRenewRequired(string $domain, Log $log): bool
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
if (\file_exists($certPath)) {
@@ -294,12 +308,15 @@ class Certificates extends Action
$validTo = $certData['validTo_time_t'] ?? 0;
if (empty($validTo)) {
$log->addTag('certificateDomain', $domain);
throw new Exception('Unable to read certificate file (cert.pem).');
}
// LetsEncrypt allows renewal 30 days before expiry
$expiryInAdvance = (60 * 60 * 24 * 30);
if ($validTo - $expiryInAdvance > \time()) {
$log->addTag('certificateDomain', $domain);
$log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
return false;
}
}