From 54c24b24e603c96e73cfa61dba0b074ebe0ac425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 6 Nov 2024 10:52:43 +0000 Subject: [PATCH 01/25] Improve preview domains local development flow --- CONTRIBUTING.md | 12 ++++++++++++ app/controllers/general.php | 33 ++++++++++++++++++++------------- app/init.php | 11 +++++++++++ 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b92361e51a..e89aa369cf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -613,6 +613,18 @@ If you need to clear the cache, you can do so by running the following command: docker compose exec redis redis-cli FLUSHALL ``` +## Using preview domains locally + +Appwrite Functions are automatically given a domain you can visit to execute the function. This domain has format `[SOMETHING].functions.localhost` unless you changed `_APP_DOMAIN_FUNCTIONS` environment variable. This default value works great when running Appwrite locally, but it can be impossible to use preview domains with Cloud woekspaces such as Gitpod or GitHub Codespaces. + +To use preview domains on Cloud workspaces, you can visit hostname provided by them, and supply function's preview domain as URL parameter: + +``` +https://8080-appwrite-appwrite-mjeb3ebilwv.ws-eu116.gitpod.io/ping?preview=672b3c7eab1ac523ccf5.functions.localhost +``` + +The path was set to `/ping` intentionally. Visiting `/` for preview domains might trigger Console background worker, and trigger redirect to Console without our preview URL param. Visiting different path ensures this doesnt happen. + ## Tutorials From time to time, our team will add tutorials that will help contributors find their way in the Appwrite source code. Below is a list of currently available tutorials: diff --git a/app/controllers/general.php b/app/controllers/general.php index b08ecb3d12..1d1e4055e4 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -46,11 +46,14 @@ Config::setParam('domainVerification', false); Config::setParam('cookieDomain', 'localhost'); Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) +function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { $utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml'); $host = $request->getHostname() ?? ''; + if (!empty($previewHost)) { + $host = $previewHost; + } $route = Authorization::skip( fn () => $dbForConsole->find('rules', [ @@ -462,15 +465,16 @@ App::init() ->inject('queueForCertificates') ->inject('queueForFunctions') ->inject('isResourceBlocked') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked) { + ->inject('previewHost') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHost) { /* * Appwrite Router */ $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain) { - if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked)) { + if ($host !== $mainDomain || !empty($previewHost)) { + if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost)) { return; } } @@ -681,15 +685,16 @@ App::options() ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) { + ->inject('previewHost') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { /* * Appwrite Router */ $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain) { - if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked)) { + if ($host !== $mainDomain || !empty($previewHost)) { + if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost)) { return; } } @@ -976,15 +981,16 @@ App::get('/robots.txt') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) { + ->inject('previewHost') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); - if ($host === $mainDomain || $host === 'localhost') { + if (($host === $mainDomain || $host === 'localhost') && empty($previewHost)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked); + router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost); } }); @@ -1003,15 +1009,16 @@ App::get('/humans.txt') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) { + ->inject('previewHost') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); - if ($host === $mainDomain || $host === 'localhost') { + if (($host === $mainDomain || $host === 'localhost') && empty($previewHost)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked); + router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost); } }); diff --git a/app/init.php b/app/init.php index d062e218e9..433d2092c5 100644 --- a/app/init.php +++ b/app/init.php @@ -1827,3 +1827,14 @@ App::setResource( 'isResourceBlocked', fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false ); + +App::setResource('previewHost', function (Request $request) { + if (App::isDevelopment()) { + $host = $request->getQuery('preview') ?? ''; + if (!empty($host)) { + return $host; + } + } + + return ''; +}, ['request']); From ca887f018e5343fe8f50d21e98776ce6da17ae54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 6 Nov 2024 11:29:24 +0000 Subject: [PATCH 02/25] fix domain execution response size --- app/controllers/general.php | 42 +++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 1d1e4055e4..467c8b0aca 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -355,26 +355,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo throw $th; } } finally { - $fileSize = 0; - $file = $request->getFiles('file'); - if (!empty($file)) { - $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; - } - - $queueForUsage - ->addMetric(METRIC_NETWORK_REQUESTS, 1) - ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) - ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) - ->addMetric(METRIC_EXECUTIONS, 1) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) - ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function - ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->setProject($project) - ->trigger() - ; - $queueForFunctions ->setType(Func::TYPE_ASYNC_WRITE) ->setExecution($execution) @@ -409,6 +389,28 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo ->setStatusCode($execution['responseStatusCode'] ?? 200) ->send($body); + $fileSize = 0; + $file = $request->getFiles('file'); + if (!empty($file)) { + $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + } + + \var_dump($response->getSize()); + + $queueForUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) + ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) + ->addMetric(METRIC_EXECUTIONS, 1) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) + ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function + ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) + ->setProject($project) + ->trigger() + ; + return true; } elseif ($type === 'api') { $utopia->getRoute()?->label('error', ''); From 87cfd7df3ba527cb26ad624c353bb00e8e6c9f68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 6 Nov 2024 11:39:44 +0000 Subject: [PATCH 03/25] Remove leftover --- app/controllers/general.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 467c8b0aca..e453fa3200 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -395,8 +395,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - \var_dump($response->getSize()); - $queueForUsage ->addMetric(METRIC_NETWORK_REQUESTS, 1) ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) From c9e6ef820211c73506d1129e3887b7f8732384b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 6 Nov 2024 16:05:58 +0000 Subject: [PATCH 04/25] Rename based on PR reviews --- app/controllers/general.php | 38 ++++++++++++++++++------------------- app/init.php | 4 ++-- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 1d1e4055e4..a8ca5985f8 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -46,13 +46,13 @@ Config::setParam('domainVerification', false); Config::setParam('cookieDomain', 'localhost'); Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) +function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml'); $host = $request->getHostname() ?? ''; - if (!empty($previewHost)) { - $host = $previewHost; + if (!empty($previewHostname)) { + $host = $previewHostname; } $route = Authorization::skip( @@ -465,16 +465,16 @@ App::init() ->inject('queueForCertificates') ->inject('queueForFunctions') ->inject('isResourceBlocked') - ->inject('previewHost') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHost) { + ->inject('previewHostname') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHostname) { /* * Appwrite Router */ $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain || !empty($previewHost)) { - if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost)) { + if ($host !== $mainDomain || !empty($previewHostname)) { + if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { return; } } @@ -685,16 +685,16 @@ App::options() ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->inject('previewHost') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { + ->inject('previewHostname') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { /* * Appwrite Router */ $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain || !empty($previewHost)) { - if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost)) { + if ($host !== $mainDomain || !empty($previewHostname)) { + if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { return; } } @@ -981,16 +981,16 @@ App::get('/robots.txt') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->inject('previewHost') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { + ->inject('previewHostname') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); - if (($host === $mainDomain || $host === 'localhost') && empty($previewHost)) { + if (($host === $mainDomain || $host === 'localhost') && empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost); + router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); } }); @@ -1009,16 +1009,16 @@ App::get('/humans.txt') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->inject('previewHost') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { + ->inject('previewHostname') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); - if (($host === $mainDomain || $host === 'localhost') && empty($previewHost)) { + if (($host === $mainDomain || $host === 'localhost') && empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost); + router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); } }); diff --git a/app/init.php b/app/init.php index 433d2092c5..c7a87c8dc9 100644 --- a/app/init.php +++ b/app/init.php @@ -1828,9 +1828,9 @@ App::setResource( fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false ); -App::setResource('previewHost', function (Request $request) { +App::setResource('previewHostname', function (Request $request) { if (App::isDevelopment()) { - $host = $request->getQuery('preview') ?? ''; + $host = $request->getQuery('appwrite-hostname') ?? ''; if (!empty($host)) { return $host; } From 842e23220ae3250f502be62a3e9c9cb974e2fc6b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 19 Nov 2024 07:51:50 +0545 Subject: [PATCH 05/25] update project last activity on error hook --- app/controllers/general.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/controllers/general.php b/app/controllers/general.php index 0bbfa2b694..f7f15138a6 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -702,6 +702,22 @@ App::error() ->inject('log') ->inject('queueForUsage') ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) { + /** + * Update project last activity + */ + if (!$project->isEmpty() && $project->getId() !== 'console') { + try { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); + } + } catch(Throwable $th) { + Console::error('[Error] updating project\'s last activity'); + Console::error($th->getMessage()); + } + } + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); From e8079cfd295a5423639a0e4babba6773c1136e65 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 19 Nov 2024 06:59:30 +0000 Subject: [PATCH 06/25] lint --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index f7f15138a6..4bf45f3860 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -712,7 +712,7 @@ App::error() $project->setAttribute('accessedAt', DateTime::now()); Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); } - } catch(Throwable $th) { + } catch (Throwable $th) { Console::error('[Error] updating project\'s last activity'); Console::error($th->getMessage()); } From 2da2b5bb7fe2423c6f14639d17496bb46daf0151 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 20 Nov 2024 04:25:37 +0000 Subject: [PATCH 07/25] reset error hook --- app/controllers/general.php | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 4bf45f3860..0bbfa2b694 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -702,22 +702,6 @@ App::error() ->inject('log') ->inject('queueForUsage') ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) { - /** - * Update project last activity - */ - if (!$project->isEmpty() && $project->getId() !== 'console') { - try { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); - } - } catch (Throwable $th) { - Console::error('[Error] updating project\'s last activity'); - Console::error($th->getMessage()); - } - } - $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); From c33c5732e0758a5c9413292dc1b9af925e60c285 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 20 Nov 2024 04:29:57 +0000 Subject: [PATCH 08/25] move accessed at to init --- app/controllers/shared/api.php | 63 ++++++++++++++++------------------ 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 6d87940ff7..a78bf1d88f 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -155,12 +155,13 @@ App::init() ->inject('utopia') ->inject('request') ->inject('dbForConsole') + ->inject('dbForProject') ->inject('project') ->inject('user') ->inject('session') ->inject('servers') ->inject('mode') - ->action(function (App $utopia, Request $request, Database $dbForConsole, Document $project, Document $user, ?Document $session, array $servers, string $mode) { + ->action(function (App $utopia, Request $request, Database $dbForConsole, Database $dbForProject, Document $project, Document $user, ?Document $session, array $servers, string $mode) { $route = $utopia->getRoute(); if ($project->isEmpty()) { @@ -301,6 +302,33 @@ App::init() Authorization::setRole($authRole); } + /** + * Update project last activity + */ + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); + } + } + + /** + * Update user last activity + */ + if (!$user->isEmpty()) { + $accessedAt = $user->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { + $user->setAttribute('accessedAt', DateTime::now()); + + if (APP_MODE_ADMIN !== $mode) { + $dbForProject->updateDocument('users', $user->getId(), $user); + } else { + $dbForConsole->updateDocument('users', $user->getId(), $user); + } + } + } + $service = $route->getLabel('sdk.namespace', ''); if (!empty($service)) { if ( @@ -588,9 +616,7 @@ App::shutdown() ->inject('queueForMessaging') ->inject('dbForProject') ->inject('queueForFunctions') - ->inject('mode') - ->inject('dbForConsole') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Database $dbForProject, Func $queueForFunctions) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -746,8 +772,6 @@ App::shutdown() } } - - if ($project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; @@ -766,33 +790,6 @@ App::shutdown() ->setProject($project) ->trigger(); } - - /** - * Update project last activity - */ - if (!$project->isEmpty() && $project->getId() !== 'console') { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); - } - } - - /** - * Update user last activity - */ - if (!$user->isEmpty()) { - $accessedAt = $user->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { - $user->setAttribute('accessedAt', DateTime::now()); - - if (APP_MODE_ADMIN !== $mode) { - $dbForProject->updateDocument('users', $user->getId(), $user); - } else { - $dbForConsole->updateDocument('users', $user->getId(), $user); - } - } - } }); App::init() From 2fcfd1d7c939777aa7e3be40309e3a504260b4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 22 Nov 2024 13:57:42 +0000 Subject: [PATCH 09/25] Add safe workers --- app/http.php | 159 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 157 insertions(+), 2 deletions(-) diff --git a/app/http.php b/app/http.php index 4bfacd4cac..c80fccee54 100644 --- a/app/http.php +++ b/app/http.php @@ -9,6 +9,7 @@ use Swoole\Http\Request as SwooleRequest; use Swoole\Http\Response as SwooleResponse; use Swoole\Http\Server; use Swoole\Process; +use Swoole\Table; use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\Audit\Audit; @@ -16,11 +17,13 @@ use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Duplicate; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; +use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Logger\Log; use Utopia\Logger\Log\User; @@ -28,6 +31,12 @@ use Utopia\Pools\Group; use Utopia\Swoole\Files; use Utopia\System\System; +const DOMAIN_SYNC_TIMER = 30; // 30 seconds + +$domains = new Table(1_000_000); // 1 million rows +$domains->column('value', Table::TYPE_INT, 1); +$domains->create(); + $http = new Server( host: "0.0.0.0", port: System::getEnv('PORT', 80), @@ -35,11 +44,12 @@ $http = new Server( ); $payloadSize = 12 * (1024 * 1024); // 12MB - adding slight buffer for headers and other data that might be sent with the payload - update later with valid testing -$workerNumber = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); +$totalWorkers = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); $http ->set([ - 'worker_num' => $workerNumber, + 'worker_num' => $totalWorkers, + 'dispatch_func' => 'dispatch', 'open_http2_protocol' => true, 'http_compression' => false, 'package_max_length' => $payloadSize, @@ -58,6 +68,93 @@ $http->on(Constant::EVENT_AFTER_RELOAD, function ($server, $workerId) { Console::success('Reload completed...'); }); +/** + * Assigns HTTP requests to worker threads by analyzing its payload/content. + * + * Routes requests as 'safe' or 'risky' based on specific content patterns (like POST actions or certain domains) + * to optimize load distribution between the workers. Utilizes `$safeThreadsPercent` to manage risk by assigning + * riskier tasks to a dedicated worker subset. Prefers idle workers, with fallback to random selection if necessary. + * doc: https://openswoole.com/docs/modules/swoole-server/configuration#dispatch_func + * + * @param Server $server Swoole server instance. + * @param int $fd client ID + * @param int $type the type of data and its current state + * @param string|null $data Request content for categorization. + * @global int $totalThreads Total number of workers. + * @return int Chosen worker ID for the request. + */ +function dispatch(Server $server, int $fd, int $type, $data = null): int +{ + global $totalWorkers, $domains; + + // If data is not set we can send request to any worker + // first we try to pick idle worker, if not we randomly pick a worker + if ($data === null) { + for ($i = 0; $i < $totalWorkers; $i++) { + if ($server->getWorkerStatus($i) === SWOOLE_WORKER_IDLE) { + return $i; + } + } + return rand(0, $totalWorkers - 1); + } + + $riskyWorkersPercent = intval(System::getEnv('_APP_RISKY_WORKERS_PERCENT', 80)) / 100; // Decimal form 0 to 1 + + // Each worker has numeric ID, starting from 0 and incrementing + // From 0 to riskyWorkers, we consider safe workers + // From riskyWorkers to totalWorkers, we consider risky workers + $riskyWorkers = (int) floor($totalWorkers * $riskyWorkersPercent); // Absolute amount of risky workers + + $domain = ''; + // max up to 3 as first line has request details and second line has host + $lines = explode("\n", $data, 3); + $request = $lines[0]; + if (count($lines) > 1) { + $domain = trim(explode('Host: ', $lines[1])[1]); + } + + // Sync executions are considered risky + $risky = false; + if (str_starts_with($request, 'POST') && str_contains($request, '/executions')) { + $risky = true; + } elseif (str_ends_with($domain, System::getEnv('_APP_DOMAIN_FUNCTIONS'))) { + $risky = true; + } elseif ($domains->get(md5($domain), 'value') === 1) { + // executions request coming from custom domain + $risky = true; + } + + if ($risky) { + // If risky request, only consider risky workers + for ($j = $riskyWorkers; $j < $totalWorkers; $j++) { + /** Reference https://openswoole.com/docs/modules/swoole-server-getWorkerStatus#description */ + if ($server->getWorkerStatus($j) === SWOOLE_WORKER_IDLE) { + // If idle worker found, give to him + return $j; + } + } + + // If no idle workers, give to random risky worker + $worker = rand($riskyWorkers, $totalWorkers - 1); + Console::warning("swoole_dispatch: Risky branch: did not find a idle worker, picking random worker {$worker}"); + return $worker; + } + + // If safe request, give to any idle worker + // Its fine to pick risky worker here, because it's idle. Idle is never actually risky + for ($i = 0; $i < $totalWorkers; $i++) { + if ($server->getWorkerStatus($i) === SWOOLE_WORKER_IDLE) { + return $i; + } + } + + // If no idle worker found, give to random safe worker + // We avoid risky workers here, as it could be in work - not idle. Thats exactly when they are risky. + $worker = rand(0, $riskyWorkers - 1); + Console::warning("swoole_dispatch: Non-risky branch: did not find a idle worker, picking random worker {$worker}"); + return $worker; +} + include __DIR__ . '/controllers/general.php'; $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $register) { @@ -213,6 +310,9 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg Console::success('Server started successfully (max payload is ' . number_format($payloadSize) . ' bytes)'); Console::info("Master pid {$http->master_pid}, manager pid {$http->manager_pid}"); + // Start the task that starts fetching custom domains + $http->task([], 0); + // listen ctrl + c Process::signal(2, function () use ($http) { Console::log('Stop by Ctrl+C'); @@ -330,4 +430,59 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool } }); +// Fetch domains every `DOMAIN_SYNC_TIMER` seconds and update in the memory +$http->on('Task', function () use ($register, $domains) { + $lastSyncUpdate = null; + $pools = $register->get('pools'); + App::setResource('pools', fn () => $pools); + $app = new App('UTC'); + + /** @var Utopia\Database\Database $dbForConsole */ + $dbForConsole = $app->getResource('dbForConsole'); + + Console::loop(function () use ($dbForConsole, $domains, &$lastSyncUpdate) { + try { + $time = DateTime::now(); + $limit = 1000; + $sum = $limit; + $latestDocument = null; + + while ($sum === $limit) { + $queries = [Query::limit($limit)]; + if ($latestDocument !== null) { + $queries[] = Query::cursorAfter($latestDocument); + } + if ($lastSyncUpdate != null) { + $queries[] = Query::greaterThanEqual('$updatedAt', $lastSyncUpdate); + } + $queries[] = Query::equal('resourceType', ['function']); + $results = []; + try { + $results = Authorization::skip(fn () => $dbForConsole->find('rules', $queries)); + } catch (Throwable $th) { + Console::error($th->getMessage()); + } + + $sum = count($results); + foreach ($results as $document) { + $domain = $document->getAttribute('domain'); + if (str_ends_with($domain, System::getEnv('_APP_DOMAIN_FUNCTIONS'))) { + continue; + } + $domains->set(md5($domain), ['value' => 1]); + } + $latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null; + } + $lastSyncUpdate = $time; + if ($sum > 0) { + Console::log("Sync domains tick: {$sum} domains were updated"); + } + } catch (Throwable $th) { + Console::error($th->getMessage()); + } + }, DOMAIN_SYNC_TIMER, 0, function ($error) { + Console::error($error); + }); +}); + $http->start(); From 33dd4df6945cc0003bf534e3bb262351189e125e Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 22 Nov 2024 17:16:51 +0000 Subject: [PATCH 10/25] chore: backwards compatibility for 1.6.x --- app/controllers/api/functions.php | 3 +- app/controllers/api/proxy.php | 17 ++++++++-- app/controllers/general.php | 32 ++++++++++++++++--- .../Platform/Workers/Certificates.php | 10 ++++-- 4 files changed, 52 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 2309defe5a..5dc67677ac 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -328,7 +328,8 @@ App::post('/v1/functions') if (!empty($functionsDomain)) { $routeSubdomain = ID::unique(); $domain = "{$routeSubdomain}.{$functionsDomain}"; - $ruleId = md5($domain); + // TODO: @christyjacob remove once we migrate the rules in 1.7.x + $ruleId = version_compare(APP_VERSION_STABLE, '1.7.0', '<') ? ID::unique() : md5($domain); $rule = Authorization::skip( fn () => $dbForConsole->createDocument('rules', new Document([ diff --git a/app/controllers/api/proxy.php b/app/controllers/api/proxy.php index 56fd31e88c..e015876935 100644 --- a/app/controllers/api/proxy.php +++ b/app/controllers/api/proxy.php @@ -11,6 +11,7 @@ use Utopia\App; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception\Query as QueryException; +use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\Query\Cursor; use Utopia\Database\Validator\UID; @@ -59,8 +60,16 @@ App::post('/v1/proxy/rules') throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'This domain name is not allowed. Please pick another one.'); } - $ruleId = md5($domain); - $document = $dbForConsole->getDocument('rules', $ruleId); + // TODO: @christyjacob remove once we migrate the rules in 1.7.x + if (version_compare(APP_VERSION_STABLE, '1.7.0', '<')) { + $document = $dbForConsole->findOne('rules', [ + Query::equal('domain', [$domain]), + ]); + } else { + $ruleId = md5($domain); + $document = $dbForConsole->getDocument('rules', $ruleId); + } + if (!$document->isEmpty()) { if ($document->getAttribute('projectId') === $project->getId()) { @@ -101,7 +110,9 @@ App::post('/v1/proxy/rules') throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Domain may not start with http:// or https://.'); } - $ruleId = md5($domain->get()); + // TODO: @christyjacob remove once we migrate the rules in 1.7.x + $ruleId = version_compare(APP_VERSION_STABLE, '1.7.0', '<') ? ID::unique() : md5($domain->get()); + $rule = new Document([ '$id' => $ruleId, 'projectId' => $project->getId(), diff --git a/app/controllers/general.php b/app/controllers/general.php index 564a5b7df4..4c4761bba9 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -29,6 +29,7 @@ use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; +use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Domains\Domain; use Utopia\DSN\DSN; @@ -51,7 +52,17 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $host = $request->getHostname() ?? ''; - $route = Authorization::skip(fn () => $dbForConsole->getDocument('rules', md5($host))); + // TODO: @christyjacob remove once we migrate the rules in 1.7.x + if (version_compare(APP_VERSION_STABLE, '1.7.0', '<')) { + $route = Authorization::skip( + fn () => $dbForConsole->find('rules', [ + Query::equal('domain', [$host]), + Query::limit(1) + ]) + )[0] ?? new Document(); + } else { + $route = Authorization::skip(fn () => $dbForConsole->getDocument('rules', md5($host))); + } if ($route->isEmpty()) { if ($host === System::getEnv('_APP_DOMAIN_FUNCTIONS', '')) { @@ -512,18 +523,31 @@ App::init() if (!empty($envDomain) && $envDomain !== 'localhost') { $mainDomain = $envDomain; } else { - $domainDocument = $dbForConsole->getDocument('rules', md5($envDomain)); + // TODO: @christyjacob remove once we migrate the rules in 1.7.x + if (version_compare(APP_VERSION_STABLE, '1.7.0', '<')) { + $domainDocument = $dbForConsole->findOne('rules', [Query::orderAsc('$id')]); + } else { + $domainDocument = $dbForConsole->getDocument('rules', md5($envDomain)); + } $mainDomain = !$domainDocument->isEmpty() ? $domainDocument->getAttribute('domain') : $domain->get(); } if ($mainDomain !== $domain->get()) { Console::warning($domain->get() . ' is not a main domain. Skipping SSL certificate generation.'); } else { - $domainDocument = $dbForConsole->getDocument('rules', md5($domain->get())); + // TODO: @christyjacob remove once we migrate the rules in 1.7.x + if (version_compare(APP_VERSION_STABLE, '1.7.0', '<')) { + $domainDocument = $dbForConsole->findOne('rules', [ + Query::equal('domain', [$domain->get()]) + ]); + } else { + $domainDocument = $dbForConsole->getDocument('rules', md5($domain->get())); + } if ($domainDocument->isEmpty()) { $domainDocument = new Document([ - '$id' => md5($domain->get()), + // TODO: @christyjacob remove once we migrate the rules in 1.7.x + '$id' => version_compare(APP_VERSION_STABLE, '1.7.0', '<') ? ID::unique() : md5($domain->get()), 'domain' => $domain->get(), 'resourceType' => 'api', 'status' => 'verifying', diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 21d967d9e1..6f1a710bc0 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -477,8 +477,14 @@ class Certificates extends Action */ private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void { - - $rule = $dbForConsole->getDocument('rules', md5($domain)); + // TODO: @christyjacob remove once we migrate the rules in 1.7.x + if (version_compare(APP_VERSION_STABLE, '1.7.0', '<')) { + $rule = $dbForConsole->findOne('rules', [ + Query::equal('domain', [$domain]), + ]); + } else { + $rule = $dbForConsole->getDocument('rules', md5($domain)); + } if (!$rule->isEmpty()) { $rule->setAttribute('certificateId', $certificateId); From ae7cb3624007349ca1600328f75c0f9d74a6a18a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 22 Nov 2024 17:18:23 +0000 Subject: [PATCH 11/25] chore: linter --- app/controllers/api/proxy.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/proxy.php b/app/controllers/api/proxy.php index e015876935..029bc5e8b3 100644 --- a/app/controllers/api/proxy.php +++ b/app/controllers/api/proxy.php @@ -69,7 +69,7 @@ App::post('/v1/proxy/rules') $ruleId = md5($domain); $document = $dbForConsole->getDocument('rules', $ruleId); } - + if (!$document->isEmpty()) { if ($document->getAttribute('projectId') === $project->getId()) { @@ -112,7 +112,7 @@ App::post('/v1/proxy/rules') // TODO: @christyjacob remove once we migrate the rules in 1.7.x $ruleId = version_compare(APP_VERSION_STABLE, '1.7.0', '<') ? ID::unique() : md5($domain->get()); - + $rule = new Document([ '$id' => $ruleId, 'projectId' => $project->getId(), From 014c613c939543388cddd35946bbba510a79115b Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:52:27 +0000 Subject: [PATCH 12/25] fix: update secret returned from users.createSession() 1. Include at least 1 factor because the minumum number of factors required when mfa is disabled is 1. 2. Purge the cached user document to ensure the new session is included in subsequent requests for the user. 3. Fix the encoding of the secret to match other parts of the codebase. --- app/controllers/api/users.php | 6 +++++- tests/e2e/Services/Users/UsersBase.php | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 42f7a59f54..bdb24572eb 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1804,6 +1804,7 @@ App::post('/v1/users/:userId/sessions') 'provider' => Auth::SESSION_PROVIDER_SERVER, 'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak 'userAgent' => $request->getUserAgent('UNKNOWN'), + 'factors' => ['server'], 'ip' => $request->getIP(), 'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--', 'expire' => $expire, @@ -1816,8 +1817,11 @@ App::post('/v1/users/:userId/sessions') $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session = $dbForProject->createDocument('sessions', $session); + + $dbForProject->purgeCachedDocument('users', $user->getId()); + $session - ->setAttribute('secret', $secret) + ->setAttribute('secret', Auth::encodeSession($user->getId(), $secret)) ->setAttribute('countryName', $countryName); $queueForEvents diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index bd0a8ef937..bbf9a5e2df 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -310,6 +310,14 @@ trait UsersBase $this->assertNotEmpty($session['secret']); $this->assertNotEmpty($session['expire']); $this->assertEquals('server', $session['provider']); + + $response = $this->client->call(Client::METHOD_GET, '/account', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-session' => $session['secret'] + ]); + + $this->assertEquals(200, $response['headers']['status-code']); } From 98d78163a2b8de2c0ccf90ef5131fd28945bc526 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Fri, 22 Nov 2024 22:39:24 +0000 Subject: [PATCH 13/25] Bump console to version 5.2.25 --- app/views/install/compose.phtml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 8d7fecb479..361685d1ed 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -167,7 +167,7 @@ $image = $this->getParam('image', ''); appwrite-console: <<: *x-logging container_name: appwrite-console - image: /console:5.0.12 + image: /console:5.2.25 restart: unless-stopped networks: - appwrite diff --git a/docker-compose.yml b/docker-compose.yml index bae2cc7811..36a642f8f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -199,7 +199,7 @@ services: appwrite-console: <<: *x-logging container_name: appwrite-console - image: appwrite/console:5.0.12 + image: appwrite/console:5.2.25 restart: unless-stopped networks: - appwrite From 07b3cbaec9de9ae586b224a0de2ce401027d8dcf Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Fri, 22 Nov 2024 22:47:02 +0000 Subject: [PATCH 14/25] Bump appwrite version to 1.6.1 --- README-CN.md | 6 +++--- README.md | 6 +++--- app/init.php | 2 +- src/Appwrite/Migration/Migration.php | 1 + 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README-CN.md b/README-CN.md index a5eac1be03..92a9bf9806 100644 --- a/README-CN.md +++ b/README-CN.md @@ -67,7 +67,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` ### Windows @@ -79,7 +79,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` #### PowerShell @@ -89,7 +89,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` 运行后,可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后,服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index a9856a7310..ab57e65c4c 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` ### Windows @@ -87,7 +87,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` #### PowerShell @@ -97,7 +97,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/init.php b/app/init.php index 047e9402b6..d629f1a0b3 100644 --- a/app/init.php +++ b/app/init.php @@ -123,7 +123,7 @@ const APP_USER_ACCESS = 24 * 60 * 60; // 24 hours const APP_PROJECT_ACCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 4318; -const APP_VERSION_STABLE = '1.6.0'; +const APP_VERSION_STABLE = '1.6.1'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index cee1b2d263..19f69b1a4f 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -91,6 +91,7 @@ abstract class Migration '1.5.10' => 'V20', '1.5.11' => 'V20', '1.6.0' => 'V21', + '1.6.1' => 'V21', ]; /** From 9d3bc1f80ae985e295f9e7b0fa62fdd0e901d7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 25 Nov 2024 11:04:04 +0100 Subject: [PATCH 15/25] Fix swoole task warning --- app/http.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/http.php b/app/http.php index c80fccee54..7387b3a43b 100644 --- a/app/http.php +++ b/app/http.php @@ -54,6 +54,7 @@ $http 'http_compression' => false, 'package_max_length' => $payloadSize, 'buffer_output_size' => $payloadSize, + 'task_worker_num' => 1, // required for the task to fetch domains background ]); $http->on(Constant::EVENT_WORKER_START, function ($server, $workerId) { From cb385674a52ea9978243a334dd2c677dc3aacfc1 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Mon, 25 Nov 2024 15:58:22 +0000 Subject: [PATCH 16/25] Add 1.6.1 to CHANGES.md --- CHANGES.md | 114 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 9b6172eeab..62db3d525e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,117 @@ +# Version 1.6.1 + +## What's Changed + +### Notable changes + +* Remove JPEG fallback for webp by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8746 +* Add heic and avif support by @lohanidamodar in https://github.com/appwrite/appwrite/pull/7718 +* Add new runtimes by @Meldiron in https://github.com/appwrite/appwrite/pull/8771 +* Remove audits deletion by @shimonewman in https://github.com/appwrite/appwrite/pull/8766 +* Bump assistant by @loks0n in https://github.com/appwrite/appwrite/pull/8801 +* Change max queries values to 500 by @fogelito in https://github.com/appwrite/appwrite/pull/8802 +* Allow '.wav' as 'audio/x-wav' as well by @basert in https://github.com/appwrite/appwrite/pull/8846 +* Use 1 instead of 0.5 cpu for default function specification by @loks0n in https://github.com/appwrite/appwrite/pull/8848 +* Update function runtimes by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8781 +* Add a realtime heartbeat by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8943 + +### Fixes + +* Trigger functions event only if event is not paused by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8526 +* Update docker-compose to restart usage-dump by @feschaffa in https://github.com/appwrite/appwrite/pull/8642 +* Fix typo in scheduler base by @fogelito in https://github.com/appwrite/appwrite/pull/8691 +* Add domain and force HTTPS env vars to mail worker by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8722 +* Fix webp by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8732 +* Ignore junction tables by @fogelito in https://github.com/appwrite/appwrite/pull/8728 +* Fix logger throwing fatal error by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8724 +* Fix missing protocol for testing SMTP by @byawitz in https://github.com/appwrite/appwrite/pull/8749 +* Make create execution async loose by @loks0n in https://github.com/appwrite/appwrite/pull/8707 +* Fix invalid cursor value by @fogelito in https://github.com/appwrite/appwrite/pull/8109 +* Fix target deletes by @abnegate in https://github.com/appwrite/appwrite/pull/8833 +* Fix translation commas by @loks0n in https://github.com/appwrite/appwrite/pull/8892 +* Fix Migrations having source creds being overwritten and add Migration tests by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8897 +* Fix validator usage for updating string size by @abnegate in https://github.com/appwrite/appwrite/pull/8890 +* Fix create user event not triggering by @loks0n in https://github.com/appwrite/appwrite/pull/8718 +* Improve error handling and logging in the database worker by @fogelito in https://github.com/appwrite/appwrite/pull/8944 +* Remove inaccurate info about leaving the URL parameter empty by @ebenezerdon in https://github.com/appwrite/appwrite/pull/8963 +* Ensure indexes are updated when updating an attribute key by @fogelito in https://github.com/appwrite/appwrite/pull/8971 +* Remove duplicate dart-2.16 runtime template by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8972 +* Fix team invites with existing session by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/9006 +* Improve handling of HTTP requests by dispatching to safe workers by @Meldiron in https://github.com/appwrite/appwrite/pull/9016 +* Fix users create session secret by @stnguyen90 in https://github.com/appwrite/appwrite/pull/9019 +* Fix swoole task warning by @Meldiron in https://github.com/appwrite/appwrite/pull/9025 + +### Miscellaneous + +* Update Init copy by @adityaoberai in https://github.com/appwrite/appwrite/pull/8557 +* Fix security scan permissions and comment by @EVDOG4LIFE in https://github.com/appwrite/appwrite/pull/8525 +* Add Trivy security scans by @btme0011 in https://github.com/appwrite/appwrite/pull/6876 +* Update database stack by @abnegate in https://github.com/appwrite/appwrite/pull/8564 +* Bump database by @abnegate in https://github.com/appwrite/appwrite/pull/8573 +* Sync main with 1.5.x by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8589 +* Add AWS to one-click installs by @byawitz in https://github.com/appwrite/appwrite/pull/8593 +* Update Init copy in readme by @adityaoberai in https://github.com/appwrite/appwrite/pull/8618 +* Sync main into 1.6.x by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8685 +* Sync 1.6.x into main by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8686 +* Feat coroutines by @Meldiron in https://github.com/appwrite/appwrite/pull/7826 +* Sync main into 1.6.x by @Meldiron in https://github.com/appwrite/appwrite/pull/8719 +* Sentence casing endpoint API reference by @choir241 in https://github.com/appwrite/appwrite/pull/8617 +* DB storage metrics by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8404 +* Fix exception thrown when optional array attribute does not exist by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8391 +* Add projects channels to realtime by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8735 +* Base for console roles support by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8565 +* Remove DB disk storage calculation by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8745 +* Messaging adapter default values by @shimonewman in https://github.com/appwrite/appwrite/pull/8742 +* Add payload response type by @loks0n in https://github.com/appwrite/appwrite/pull/8720 +* Fix flaky functions tests by @loks0n in https://github.com/appwrite/appwrite/pull/8682 +* Migrations Backups by @fogelito in https://github.com/appwrite/appwrite/pull/8186 +* Add test for response and request filters by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8697 +* Bump version in SECURITY.md by @EVDOG4LIFE in https://github.com/appwrite/appwrite/pull/8755 +* Add originalId attribute to databases collection by @fogelito in https://github.com/appwrite/appwrite/pull/8764 +* Fix Walter References by @ItzNotABug in https://github.com/appwrite/appwrite/pull/8757 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8769 +* Move new attributes by @abnegate in https://github.com/appwrite/appwrite/pull/8777 +* Add ping endpoint by @loks0n in https://github.com/appwrite/appwrite/pull/8761 +* Fix GitHub action caching by @loks0n in https://github.com/appwrite/appwrite/pull/8772 +* Chore release ruby SDK by @abnegate in https://github.com/appwrite/appwrite/pull/8767 +* Call migration success on success by @abnegate in https://github.com/appwrite/appwrite/pull/8782 +* Update utopia-php/system to 0.9.0 by @basert in https://github.com/appwrite/appwrite/pull/8780 +* Move createDocument from api to worker by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8776 +* Add missing indexes by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8803 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8809 +* Fix typo in BLR region by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8756 +* Add tests for project variables by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8815 +* Replace 'Expires' with 'Cache-Control: private' header to avoid CDN caching by @basert in https://github.com/appwrite/appwrite/pull/8836 +* Allow blocking based on resource attributes by @basert in https://github.com/appwrite/appwrite/pull/8812 +* Check if resource is blocked inside functions worker by @basert in https://github.com/appwrite/appwrite/pull/8855 +* Fix missing allow attribute by @abnegate in https://github.com/appwrite/appwrite/pull/8889 +* Revert function execution order by @basert in https://github.com/appwrite/appwrite/pull/8857 +* Use resource type constants by @basert in https://github.com/appwrite/appwrite/pull/8895 +* Update Database lib by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8680 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8917 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8923 +* Update database for transaction counter fixes with retries by @abnegate in https://github.com/appwrite/appwrite/pull/8927 +* Validate string permissions by @fogelito in https://github.com/appwrite/appwrite/pull/8929 +* Add PubSub adapter support by @basert in https://github.com/appwrite/appwrite/pull/8905 +* List memberships as client by @loks0n in https://github.com/appwrite/appwrite/pull/8913 +* Fix XDebug Extension not being removed by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8891 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8946 +* Use utopia compression by @loks0n in https://github.com/appwrite/appwrite/pull/8938 +* Make compression minimum size configurable by @loks0n in https://github.com/appwrite/appwrite/pull/8947 +* Revert "Update database" by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8949 +* Fix setpaused by @loks0n in https://github.com/appwrite/appwrite/pull/8948 +* Use getDocument instead of find() for rules by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8951 +* Remove double fetch from migrations worker by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8956 +* Fix memberships privacy MFA by @loks0n in https://github.com/appwrite/appwrite/pull/8969 +* Add telemetry by @basert in https://github.com/appwrite/appwrite/pull/8960 +* Send migration errors individually by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8959 +* Add console sdk previews by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8990 +* Unset index length by @fogelito in https://github.com/appwrite/appwrite/pull/8978 +* Update base to 0.9.5 by @basert in https://github.com/appwrite/appwrite/pull/9005 +* Sync main into 1.6.x by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/9011 +* Improved shared tables V2 by @abnegate in https://github.com/appwrite/appwrite/pull/9013 +* Ensure backwards compatibility for 1.6.x by @christyjacob4 in https://github.com/appwrite/appwrite/pull/9018 + # Version 1.6.0 ## What's Changed From 580a9f331fc07b31ae2c5a068f5d6fc370334afb Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 26 Nov 2024 08:59:03 +0000 Subject: [PATCH 17/25] fix user update --- app/controllers/shared/api.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index a78bf1d88f..68c32cc969 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -316,7 +316,7 @@ App::init() /** * Update user last activity */ - if (!$user->isEmpty()) { + if (!empty($user->getId())) { $accessedAt = $user->getAttribute('accessedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { $user->setAttribute('accessedAt', DateTime::now()); From 3a9ba8a6ad6a76454bc9dad586373cd949fd037a Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 22 Oct 2024 13:57:25 +0200 Subject: [PATCH 18/25] feat: move certificate generation to Adapter --- app/init.php | 15 ++ src/Appwrite/Certificates/Adapter.php | 14 ++ .../Certificates/Adapter/LetsEncrypt.php | 126 +++++++++++ src/Appwrite/Certificates/AdapterProvider.php | 19 ++ src/Appwrite/Platform/Tasks/Maintenance.php | 1 + .../Platform/Workers/Certificates.php | 206 ++++-------------- src/Appwrite/Platform/Workers/Deletes.php | 49 ++--- 7 files changed, 234 insertions(+), 196 deletions(-) create mode 100644 src/Appwrite/Certificates/Adapter.php create mode 100644 src/Appwrite/Certificates/Adapter/LetsEncrypt.php create mode 100644 src/Appwrite/Certificates/AdapterProvider.php diff --git a/app/init.php b/app/init.php index 047e9402b6..55f1356d62 100644 --- a/app/init.php +++ b/app/init.php @@ -21,6 +21,8 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; +use Appwrite\Certificates\Adapter\LetsEncrypt; +use Appwrite\Certificates\AdapterProvider; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Certificate; @@ -1531,6 +1533,19 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); +App::setResource('letsEncryptCertificateAdapter', function () { + $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); + if (empty($email)) { + throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.'); + } + + return new LetsEncrypt($email); +}); + +App::setResource('adapterForCertificates', function ($letsEncrypt) { + return new AdapterProvider(fn (string $domain) => $letsEncrypt); +}, ['letsEncryptCertificateAdapter']); + App::setResource('deviceForLocal', function () { return new Local(); }); diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php new file mode 100644 index 0000000000..711e4c09b9 --- /dev/null +++ b/src/Appwrite/Certificates/Adapter.php @@ -0,0 +1,14 @@ +email = $email; + } + + + public function issueCertificate(string $certName, string $domain): ?string + { + $stdout = ''; + $stderr = ''; + + $staging = (App::isProduction()) ? '' : ' --dry-run'; + $exit = Console::execute( + "certbot certonly -v --webroot --noninteractive --agree-tos{$staging}" + . " --email " . $this->email + . " --cert-name " . $certName + . " -w " . APP_STORAGE_CERTIFICATES + . " -d {$domain}", + '', + $stdout, + $stderr + ); + + // Unexpected error, usually 5XX, API limits, ... + if ($exit !== 0) { + throw new Exception('Failed to issue a certificate with message: ' . $stderr); + } + + // Prepare folder in storage for domain + $path = APP_STORAGE_CERTIFICATES . '/' . $domain; + if (!\is_readable($path)) { + if (!\mkdir($path, 0755, true)) { + throw new Exception('Failed to create path for certificate.'); + } + } + + // Move generated files + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { + throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { + throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { + throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { + throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + $config = \implode(PHP_EOL, [ + "tls:", + " certificates:", + " - certFile: /storage/certificates/{$domain}/fullchain.pem", + " keyFile: /storage/certificates/{$domain}/privkey.pem" + ]); + + // Save configuration into Traefik using our new cert files + if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { + throw new Exception('Failed to save Traefik configuration.'); + } + + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? null; + $dt = (new \DateTime())->setTimestamp($validTo); + return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); + } + + public function isRenewRequired(string $domain, Log $log): bool + { + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + if (\file_exists($certPath)) { + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? 0; + + if (empty($validTo)) { + $log->addTag('certificateDomain', $domain); + throw new Exception('Unable to read certificate file (cert.pem).'); + } + + // LetsEncrypt allows renewal 30 days before expiry + $expiryInAdvance = (60 * 60 * 24 * 30); + if ($validTo - $expiryInAdvance > \time()) { + $log->addTag('certificateDomain', $domain); + $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData)); + return false; + } + } + + return true; + } + + public function deleteCertificate(string $domain): void + { + $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; + $checkTraversal = realpath($directory) === $directory; + + if ($checkTraversal && is_dir($directory)) { + // Delete files, so Traefik is aware of change + array_map('unlink', glob($directory . '/*.*')); + rmdir($directory); + Console::info("Deleted certificate files for {$domain}"); + } else { + Console::info("No certificate files found for {$domain}"); + } + } +} diff --git a/src/Appwrite/Certificates/AdapterProvider.php b/src/Appwrite/Certificates/AdapterProvider.php new file mode 100644 index 0000000000..bed73a0164 --- /dev/null +++ b/src/Appwrite/Certificates/AdapterProvider.php @@ -0,0 +1,19 @@ +decisionMaker = $decisionMaker; + } + + public function get(string $domain): Adapter + { + return call_user_func($this->decisionMaker, $domain); + } + +} diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index afb38f35fc..40e72dc683 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -149,6 +149,7 @@ class Maintenance extends Action $certificates = $dbForConsole->find('certificates', [ Query::lessThan('attempts', 5), // Maximum 5 attempts + Query::isNotNull('renewDate'), Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) ]); diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 6f1a710bc0..ecb4cbd127 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Workers; +use Appwrite\Certificates\AdapterProvider; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; @@ -11,7 +12,6 @@ use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; use Exception; use Throwable; -use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -48,7 +48,11 @@ class Certificates extends Action ->inject('queueForEvents') ->inject('queueForFunctions') ->inject('log') - ->callback(fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log)); + ->inject('adapterForCertificates') + ->callback( + fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates) => + $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $adapterForCertificates) + ); } /** @@ -58,11 +62,12 @@ class Certificates extends Action * @param Event $queueForEvents * @param Func $queueForFunctions * @param Log $log + * @param AdapterProvider $adapterForCertificates Retrieve the certificate adapter for the domain * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates): void { $payload = $message->getPayload() ?? []; @@ -76,7 +81,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck); + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $adapterForCertificates, $skipRenewCheck); } /** @@ -85,24 +90,24 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions + * @param AdapterProvider $adapterForCertificates Retrieve the certificate adapter for the domain * @param bool $skipRenewCheck * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain * 2. Get main domain * 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain) - * 4. Validate security email. Cannot be empty, required by LetsEncrypt - * 5. Validate renew date with certificate file, unless requested to skip by parameter - * 6. Issue a certificate using certbot CLI - * 7. Update 'log' attribute on certificate document with Certbot message - * 8. Create storage folder for certificate, if not ready already - * 9. Move certificates from Certbot location to our Storage - * 10. Create/Update our Storage with new Traefik config with new certificate paths + * 4. Validate renew date with certificate file, unless requested to skip by parameter + * 5. Issue a certificate using certbot CLI + * 6. Update 'log' attribute on certificate document with Certbot message + * 7. Create storage folder for certificate, if not ready already + * 8. Move certificates from Certbot location to our Storage + * 9. Create/Update our Storage with new Traefik config with new certificate paths * 11. Read certificate file and update 'renewDate' on certificate document * 12. Update 'issueDate' and 'attempts' on certificate * @@ -119,7 +124,7 @@ class Certificates extends Action * 2. Save document to database * 3. Update all domains documents with current certificate ID * - * Note: Renewals are checked and scheduled from maintenence worker + * Note: Renewals are checked and scheduled from maintenance worker */ // Get current certificate @@ -131,43 +136,32 @@ class Certificates extends Action $certificate->setAttribute('domain', $domain->get()); } + $certificateAdapter = $adapterForCertificates->get($domain->get()); + $success = false; try { - // Email for alerts is required by LetsEncrypt - $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); - if (empty($email)) { - throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue an SSL certificate.'); - } - // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { $mainDomain = $this->getMainDomain(); $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; $this->validateDomain($domain, $isMainDomain, $log); + + // If certificate exists already, double-check expiry date. Skip if job is forced + if (!$certificateAdapter->isRenewRequired($domain->get(), $log)) { + throw new Exception('Renew isn\'t required.'); + } } - // If certificate exists already, double-check expiry date. Skip if job is forced - if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) { - throw new Exception('Renew isn\'t required.'); - } - - // Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate - $folder = ID::unique(); - - // Generate certificate files using Let's Encrypt - $letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email); + // Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates. + $certName = ID::unique(); + $renewDate = $certificateAdapter->issueCertificate($certName, $domain->get()); // Command succeeded, store all data into document - $logs = 'Certificate successfully generated.'; - $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB - - - // Give certificates to Traefik - $this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData); + $certificate->setAttribute('logs', 'Certificate successfully generated.'); // Update certificate info stored in database - $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get())); + $certificate->setAttribute('renewDate', $renewDate); $certificate->setAttribute('attempts', 0); $certificate->setAttribute('issueDate', DateTime::now()); $success = true; @@ -181,7 +175,7 @@ class Certificates extends Action $attempts = $certificate->getAttribute('attempts', 0) + 1; $certificate->setAttribute('attempts', $attempts); - // Store cuttent time as renew date to ensure another attempt in next maintenance cycle + // Store current time as renew date to ensure another attempt in next maintenance cycle. $certificate->setAttribute('renewDate', DateTime::now()); // Send email to security email @@ -245,8 +239,8 @@ class Certificates extends Action } /** - * Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check: - * - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt) + * Internal domain validation functionality to prevent unnecessary attempts. We check: + * - Domain needs to be public and valid (prevents NFT domains that are not supported) * - Domain must have proper DNS record * * @param Domain $domain Domain which we validate @@ -292,136 +286,6 @@ class Certificates extends Action } } - /** - * Reads expiry date of certificate from file and decides if renewal is required or not. - * - * @param string $domain Domain for which we check certificate file - * @return bool True, if certificate needs to be renewed - * @throws Exception - */ - private function isRenewRequired(string $domain, Log $log): bool - { - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; - if (\file_exists($certPath)) { - $validTo = null; - - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? 0; - - if (empty($validTo)) { - $log->addTag('certificateDomain', $domain); - throw new Exception('Unable to read certificate file (cert.pem).'); - } - - // LetsEncrypt allows renewal 30 days before expiry - $expiryInAdvance = (60 * 60 * 24 * 30); - if ($validTo - $expiryInAdvance > \time()) { - $log->addTag('certificateDomain', $domain); - $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData)); - return false; - } - } - - return true; - } - - /** - * LetsEncrypt communication to issue certificate (using certbot CLI) - * - * @param string $folder Folder into which certificates should be generated - * @param string $domain Domain to generate certificate for - * @return array Named array with keys 'stdout' and 'stderr', both string - * @throws Exception - */ - private function issueCertificate(string $folder, string $domain, string $email): array - { - $stdout = ''; - $stderr = ''; - - $staging = (App::isProduction()) ? '' : ' --dry-run'; - $exit = Console::execute("certbot certonly -v --webroot --noninteractive --agree-tos{$staging}" - . " --email " . $email - . " --cert-name " . $folder - . " -w " . APP_STORAGE_CERTIFICATES - . " -d {$domain}", '', $stdout, $stderr); - - // Unexpected error, usually 5XX, API limits, ... - if ($exit !== 0) { - throw new Exception('Failed to issue a certificate with message: ' . $stderr); - } - - return [ - 'stdout' => $stdout, - 'stderr' => $stderr - ]; - } - - /** - * Read new renew date from certificate file generated by Let's Encrypt - * - * @param string $domain Domain which certificate was generated for - * @return string - * @throws \Utopia\Database\Exception - */ - private function getRenewDate(string $domain): string - { - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? null; - $dt = (new \DateTime())->setTimestamp($validTo); - return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days - } - - /** - * Method to take files from Let's Encrypt, and put it into Traefik. - * - * @param string $domain Domain which certificate was generated for - * @param string $folder Folder in which certificates were generated - * @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error - * @return void - * @throws Exception - */ - private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void - { - - // Prepare folder in storage for domain - $path = APP_STORAGE_CERTIFICATES . '/' . $domain; - if (!\is_readable($path)) { - if (!\mkdir($path, 0755, true)) { - throw new Exception('Failed to create path for certificate.'); - } - } - - // Move generated files - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - $config = \implode(PHP_EOL, [ - "tls:", - " certificates:", - " - certFile: /storage/certificates/{$domain}/fullchain.pem", - " keyFile: /storage/certificates/{$domain}/privkey.pem" - ]); - - // Save configuration into Traefik using our new cert files - if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { - throw new Exception('Failed to save Traefik configuration.'); - } - } - /** * Method to make sure information about error is delivered to admnistrator. * @@ -500,6 +364,10 @@ class Certificates extends Action $project = $dbForConsole->getDocument('projects', $projectId); + if ($project->isEmpty()) { + return; + } + /** Trigger Webhook */ $ruleModel = new Rule(); $queueForEvents diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 6c11116a4c..b4570e3929 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Auth\Auth; +use Appwrite\Certificates\AdapterProvider; use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; @@ -50,18 +51,22 @@ class Deletes extends Action ->inject('deviceForFunctions') ->inject('deviceForBuilds') ->inject('deviceForCache') + ->inject('adapterForCertificates') ->inject('abuseRetention') ->inject('executionRetention') ->inject('auditRetention') ->inject('log') - ->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log)); + ->callback( + fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $abuseRetention, $executionRetention, $auditRetention, $log) + ); } /** * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -84,10 +89,10 @@ class Deletes extends Action case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { case DELETE_TYPE_PROJECTS: - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); + $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $adapterProvider, $document, $project); break; case DELETE_TYPE_DEPLOYMENTS: $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); @@ -102,7 +107,7 @@ class Deletes extends Action $this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project); break; case DELETE_TYPE_RULES: - $this->deleteRule($dbForConsole, $document); + $this->deleteRule($dbForConsole, $document, $adapterProvider); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -110,7 +115,7 @@ class Deletes extends Action } break; case DELETE_TYPE_TEAM_PROJECTS: - $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $document); + $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $adapterProvider, $document); break; case DELETE_TYPE_EXECUTIONS: $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); @@ -442,7 +447,7 @@ class Deletes extends Action * @throws Structure * @throws Exception */ - private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Document $document): void + private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, AdapterProvider $adapterProvider, Document $document): void { $projects = $dbForConsole->find('projects', [ @@ -455,7 +460,7 @@ class Deletes extends Action $deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); $deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $project); $dbForConsole->deleteDocument('projects', $project->getId()); } } @@ -473,7 +478,7 @@ class Deletes extends Action * @throws Authorization * @throws DatabaseException */ - private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void + private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, Document $document): void { $projectInternalId = $document->getInternalId(); $projectId = $document->getId(); @@ -537,8 +542,8 @@ class Deletes extends Action // Delete project and function rules $this->deleteByGroup('rules', [ Query::equal('projectInternalId', [$projectInternalId]) - ], $dbForConsole, function (Document $document) use ($dbForConsole) { - $this->deleteRule($dbForConsole, $document); + ], $dbForConsole, function (Document $document) use ($dbForConsole, $adapterProvider) { + $this->deleteRule($dbForConsole, $document, $adapterProvider); }); // Delete Keys @@ -746,7 +751,7 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void + private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, AdapterProvider $adapterProvider, Document $document, Document $project): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -761,8 +766,8 @@ class Deletes extends Action Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('projectInternalId', [$project->getInternalId()]) - ], $dbForConsole, function (Document $document) use ($project, $dbForConsole) { - $this->deleteRule($dbForConsole, $document); + ], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $adapterProvider) { + $this->deleteRule($dbForConsole, $document, $adapterProvider); }); /** @@ -1048,21 +1053,11 @@ class Deletes extends Action * @param Document $document rule document * @return void */ - private function deleteRule(Database $dbForConsole, Document $document): void + private function deleteRule(Database $dbForConsole, Document $document, AdapterProvider $adapterForCertificate): void { - $domain = $document->getAttribute('domain'); - $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; - $checkTraversal = realpath($directory) === $directory; - - if ($checkTraversal && is_dir($directory)) { - // Delete files, so Traefik is aware of change - array_map('unlink', glob($directory . '/*.*')); - rmdir($directory); - Console::info("Deleted certificate files for {$domain}"); - } else { - Console::info("No certificate files found for {$domain}"); - } + $adapter = $adapterForCertificate->get($domain); + $adapter->deleteCertificate($domain); // Delete certificate document, so Appwrite is aware of change if (isset($document['certificateId'])) { From 98984a808500264fb2b3ce14801a7fb062e33d39 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Wed, 20 Nov 2024 11:10:57 +0100 Subject: [PATCH 19/25] refactor: use single adapter --- app/init.php | 15 ------- app/worker.php | 11 ++++- docker-compose.yml | 1 + src/Appwrite/Certificates/AdapterProvider.php | 19 --------- .../{Adapter => }/LetsEncrypt.php | 3 +- .../Platform/Workers/Certificates.php | 24 +++++------ src/Appwrite/Platform/Workers/Deletes.php | 41 +++++++++---------- 7 files changed, 43 insertions(+), 71 deletions(-) delete mode 100644 src/Appwrite/Certificates/AdapterProvider.php rename src/Appwrite/Certificates/{Adapter => }/LetsEncrypt.php (98%) diff --git a/app/init.php b/app/init.php index 55f1356d62..047e9402b6 100644 --- a/app/init.php +++ b/app/init.php @@ -21,8 +21,6 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; -use Appwrite\Certificates\Adapter\LetsEncrypt; -use Appwrite\Certificates\AdapterProvider; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Certificate; @@ -1533,19 +1531,6 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); -App::setResource('letsEncryptCertificateAdapter', function () { - $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); - if (empty($email)) { - throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.'); - } - - return new LetsEncrypt($email); -}); - -App::setResource('adapterForCertificates', function ($letsEncrypt) { - return new AdapterProvider(fn (string $domain) => $letsEncrypt); -}, ['letsEncryptCertificateAdapter']); - App::setResource('deviceForLocal', function () { return new Local(); }); diff --git a/app/worker.php b/app/worker.php index 7899c1d5bc..280e170028 100644 --- a/app/worker.php +++ b/app/worker.php @@ -2,6 +2,7 @@ require_once __DIR__ . '/init.php'; +use Appwrite\Certificates\LetsEncrypt; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Certificate; @@ -16,7 +17,6 @@ use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; -use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -283,6 +283,15 @@ Server::setResource( fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false ); +Server::setResource('certificates', function () { + $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); + if (empty($email)) { + throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.'); + } + + return new LetsEncrypt($email); +}); + Server::setResource('logError', function (Registry $register, Document $project) { return function (Throwable $error, string $namespace, string $action, ?array $extras) use ($register, $project) { $logger = $register->get('logger'); diff --git a/docker-compose.yml b/docker-compose.yml index 36a642f8f6..ef8d898250 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -385,6 +385,7 @@ services: - _APP_EXECUTOR_HOST - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 + - _APP_EMAIL_CERTIFICATES appwrite-worker-databases: entrypoint: worker-databases diff --git a/src/Appwrite/Certificates/AdapterProvider.php b/src/Appwrite/Certificates/AdapterProvider.php deleted file mode 100644 index bed73a0164..0000000000 --- a/src/Appwrite/Certificates/AdapterProvider.php +++ /dev/null @@ -1,19 +0,0 @@ -decisionMaker = $decisionMaker; - } - - public function get(string $domain): Adapter - { - return call_user_func($this->decisionMaker, $domain); - } - -} diff --git a/src/Appwrite/Certificates/Adapter/LetsEncrypt.php b/src/Appwrite/Certificates/LetsEncrypt.php similarity index 98% rename from src/Appwrite/Certificates/Adapter/LetsEncrypt.php rename to src/Appwrite/Certificates/LetsEncrypt.php index b2237189e9..3896eab022 100644 --- a/src/Appwrite/Certificates/Adapter/LetsEncrypt.php +++ b/src/Appwrite/Certificates/LetsEncrypt.php @@ -1,8 +1,7 @@ inject('queueForEvents') ->inject('queueForFunctions') ->inject('log') - ->inject('adapterForCertificates') + ->inject('certificates') ->callback( - fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates) => - $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $adapterForCertificates) + fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates) => + $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates) ); } @@ -62,12 +62,12 @@ class Certificates extends Action * @param Event $queueForEvents * @param Func $queueForFunctions * @param Log $log - * @param AdapterProvider $adapterForCertificates Retrieve the certificate adapter for the domain + * @param CertificatesAdapter $certificates * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates): void + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates): void { $payload = $message->getPayload() ?? []; @@ -81,7 +81,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $adapterForCertificates, $skipRenewCheck); + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates, $skipRenewCheck); } /** @@ -90,13 +90,13 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions - * @param AdapterProvider $adapterForCertificates Retrieve the certificate adapter for the domain + * @param CertificatesAdapter $certificates * @param bool $skipRenewCheck * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain @@ -136,8 +136,6 @@ class Certificates extends Action $certificate->setAttribute('domain', $domain->get()); } - $certificateAdapter = $adapterForCertificates->get($domain->get()); - $success = false; try { @@ -148,14 +146,14 @@ class Certificates extends Action $this->validateDomain($domain, $isMainDomain, $log); // If certificate exists already, double-check expiry date. Skip if job is forced - if (!$certificateAdapter->isRenewRequired($domain->get(), $log)) { + if (!$certificates->isRenewRequired($domain->get(), $log)) { throw new Exception('Renew isn\'t required.'); } } // Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates. $certName = ID::unique(); - $renewDate = $certificateAdapter->issueCertificate($certName, $domain->get()); + $renewDate = $certificates->issueCertificate($certName, $domain->get()); // Command succeeded, store all data into document $certificate->setAttribute('logs', 'Certificate successfully generated.'); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index b4570e3929..e6002bf75f 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -3,7 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Auth\Auth; -use Appwrite\Certificates\AdapterProvider; +use Appwrite\Certificates\Adapter as CertificatesAdapter; use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; @@ -51,14 +51,14 @@ class Deletes extends Action ->inject('deviceForFunctions') ->inject('deviceForBuilds') ->inject('deviceForCache') - ->inject('adapterForCertificates') + ->inject('certificates') ->inject('abuseRetention') ->inject('executionRetention') ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $abuseRetention, $executionRetention, $auditRetention, $log) + fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -89,10 +89,10 @@ class Deletes extends Action case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { case DELETE_TYPE_PROJECTS: - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $document); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $adapterProvider, $document, $project); + $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project); break; case DELETE_TYPE_DEPLOYMENTS: $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); @@ -107,7 +107,7 @@ class Deletes extends Action $this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project); break; case DELETE_TYPE_RULES: - $this->deleteRule($dbForConsole, $document, $adapterProvider); + $this->deleteRule($dbForConsole, $document, $certificates); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -115,7 +115,7 @@ class Deletes extends Action } break; case DELETE_TYPE_TEAM_PROJECTS: - $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $adapterProvider, $document); + $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $certificates, $document); break; case DELETE_TYPE_EXECUTIONS: $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); @@ -269,7 +269,7 @@ class Deletes extends Action MESSAGE_TYPE_EMAIL => 'emailTotal', MESSAGE_TYPE_SMS => 'smsTotal', MESSAGE_TYPE_PUSH => 'pushTotal', - default => throw new Exception('Invalid target provider type'), + default => throw new Exception('Invalid target CertificatesAdapter type'), }; $dbForProject->decreaseDocumentAttribute( 'topics', @@ -447,7 +447,7 @@ class Deletes extends Action * @throws Structure * @throws Exception */ - private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, AdapterProvider $adapterProvider, Document $document): void + private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void { $projects = $dbForConsole->find('projects', [ @@ -460,7 +460,7 @@ class Deletes extends Action $deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); $deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $project); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $project); $dbForConsole->deleteDocument('projects', $project->getId()); } } @@ -478,7 +478,7 @@ class Deletes extends Action * @throws Authorization * @throws DatabaseException */ - private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, Document $document): void + private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Document $document): void { $projectInternalId = $document->getInternalId(); $projectId = $document->getId(); @@ -542,8 +542,8 @@ class Deletes extends Action // Delete project and function rules $this->deleteByGroup('rules', [ Query::equal('projectInternalId', [$projectInternalId]) - ], $dbForConsole, function (Document $document) use ($dbForConsole, $adapterProvider) { - $this->deleteRule($dbForConsole, $document, $adapterProvider); + ], $dbForConsole, function (Document $document) use ($dbForConsole, $certificates) { + $this->deleteRule($dbForConsole, $document, $certificates); }); // Delete Keys @@ -751,7 +751,7 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, AdapterProvider $adapterProvider, Document $document, Document $project): void + private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -766,8 +766,8 @@ class Deletes extends Action Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('projectInternalId', [$project->getInternalId()]) - ], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $adapterProvider) { - $this->deleteRule($dbForConsole, $document, $adapterProvider); + ], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $certificates) { + $this->deleteRule($dbForConsole, $document, $certificates); }); /** @@ -1053,11 +1053,10 @@ class Deletes extends Action * @param Document $document rule document * @return void */ - private function deleteRule(Database $dbForConsole, Document $document, AdapterProvider $adapterForCertificate): void + private function deleteRule(Database $dbForConsole, Document $document, CertificatesAdapter $certificates): void { $domain = $document->getAttribute('domain'); - $adapter = $adapterForCertificate->get($domain); - $adapter->deleteCertificate($domain); + $certificates->deleteCertificate($domain); // Delete certificate document, so Appwrite is aware of change if (isset($document['certificateId'])) { From 442459f6a482121f573168458cafb767875467ec Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 26 Nov 2024 16:14:38 +0200 Subject: [PATCH 20/25] fix restoration --- composer.json | 2 +- composer.lock | 39 ++++++++++++++++++++++++--------------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/composer.json b/composer.json index e3fbcabc83..5fb781083c 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.12.*", - "utopia-php/migration": "0.6.*", + "utopia-php/migration": "dev-0.6.x-fix-structore-validation as 0.6.13", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.5.*", diff --git a/composer.lock b/composer.lock index 87d8b47583..c742097e13 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ae3b9a491c9870a4897cdf712caba1e3", + "content-hash": "8e458c412c8a8ce3ce018b7a2c690832", "packages": [ { "name": "adhocore/jwt", @@ -157,16 +157,16 @@ }, { "name": "appwrite/php-runtimes", - "version": "0.16.4", + "version": "0.16.5", "source": { "type": "git", "url": "https://github.com/appwrite/runtimes.git", - "reference": "7e4741337b9373f77210396e68eca539018cabd1" + "reference": "1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/runtimes/zipball/7e4741337b9373f77210396e68eca539018cabd1", - "reference": "7e4741337b9373f77210396e68eca539018cabd1", + "url": "https://api.github.com/repos/appwrite/runtimes/zipball/1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9", + "reference": "1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9", "shasum": "" }, "require": { @@ -206,9 +206,9 @@ ], "support": { "issues": "https://github.com/appwrite/runtimes/issues", - "source": "https://github.com/appwrite/runtimes/tree/0.16.4" + "source": "https://github.com/appwrite/runtimes/tree/0.16.5" }, - "time": "2024-10-26T10:39:59+00:00" + "time": "2024-11-25T15:17:06+00:00" }, { "name": "beberlei/assert", @@ -3928,16 +3928,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.12", + "version": "dev-0.6.x-fix-structore-validation", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "9a8c905af4cece5c5ec9542a5b534befce067260" + "reference": "ed735136dd10a88ab8601a9831e9cbcf3b6d6560" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/9a8c905af4cece5c5ec9542a5b534befce067260", - "reference": "9a8c905af4cece5c5ec9542a5b534befce067260", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/ed735136dd10a88ab8601a9831e9cbcf3b6d6560", + "reference": "ed735136dd10a88ab8601a9831e9cbcf3b6d6560", "shasum": "" }, "require": { @@ -3978,9 +3978,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.12" + "source": "https://github.com/utopia-php/migration/tree/0.6.x-fix-structore-validation" }, - "time": "2024-11-12T00:31:53+00:00" + "time": "2024-11-26T11:04:12+00:00" }, { "name": "utopia-php/mongo", @@ -8555,9 +8555,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/migration", + "version": "dev-0.6.x-fix-structore-validation", + "alias": "0.6.13", + "alias_normalized": "0.6.13.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/migration": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From 135ff1a72400487c827033e3943a96dd322bea4e Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 26 Nov 2024 17:05:17 +0200 Subject: [PATCH 21/25] bomb migration 0.6.13 --- composer.json | 2 +- composer.lock | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/composer.json b/composer.json index 5fb781083c..e3fbcabc83 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.12.*", - "utopia-php/migration": "dev-0.6.x-fix-structore-validation as 0.6.13", + "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.5.*", diff --git a/composer.lock b/composer.lock index c742097e13..bdb41c2478 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8e458c412c8a8ce3ce018b7a2c690832", + "content-hash": "ae3b9a491c9870a4897cdf712caba1e3", "packages": [ { "name": "adhocore/jwt", @@ -3928,16 +3928,16 @@ }, { "name": "utopia-php/migration", - "version": "dev-0.6.x-fix-structore-validation", + "version": "0.6.13", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "ed735136dd10a88ab8601a9831e9cbcf3b6d6560" + "reference": "68d9b0a9477755afcda607e7e8109785cae17a13" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/ed735136dd10a88ab8601a9831e9cbcf3b6d6560", - "reference": "ed735136dd10a88ab8601a9831e9cbcf3b6d6560", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/68d9b0a9477755afcda607e7e8109785cae17a13", + "reference": "68d9b0a9477755afcda607e7e8109785cae17a13", "shasum": "" }, "require": { @@ -3978,9 +3978,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.x-fix-structore-validation" + "source": "https://github.com/utopia-php/migration/tree/0.6.13" }, - "time": "2024-11-26T11:04:12+00:00" + "time": "2024-11-26T13:57:53+00:00" }, { "name": "utopia-php/mongo", @@ -8555,18 +8555,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/migration", - "version": "dev-0.6.x-fix-structore-validation", - "alias": "0.6.13", - "alias_normalized": "0.6.13.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/migration": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From f0b6c729c925efc5148153ab2f3ad2d6b0d20a69 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 26 Nov 2024 14:54:27 +0100 Subject: [PATCH 22/25] feat: add more tags to sentry --- app/cli.php | 2 +- app/controllers/general.php | 5 ++++- app/http.php | 5 ++++- app/realtime.php | 2 +- app/worker.php | 4 ++-- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/app/cli.php b/app/cli.php index b0f053d3c6..fd49d2fed4 100644 --- a/app/cli.php +++ b/app/cli.php @@ -181,7 +181,7 @@ CLI::setResource('logError', function (Registry $register) { $log = new Log(); $log->setNamespace($namespace); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); diff --git a/app/controllers/general.php b/app/controllers/general.php index 4c4761bba9..9e589452ec 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -861,6 +861,8 @@ App::error() if (isset($user) && !$user->isEmpty()) { $log->setUser(new User($user->getId())); + } else { + $log->setUser(new User('guest-' . hash('sha256', $request->getIP()))); } try { @@ -871,7 +873,7 @@ App::error() } $log->setNamespace("http"); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); @@ -892,6 +894,7 @@ App::error() $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD"); $log->setAction($action); + $log->addTag('service', $action); $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); diff --git a/app/http.php b/app/http.php index 7387b3a43b..c4e48a7f69 100644 --- a/app/http.php +++ b/app/http.php @@ -370,10 +370,12 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool if (isset($user) && !$user->isEmpty()) { $log->setUser(new User($user->getId())); + } else { + $log->setUser(new User('guest-' . hash('sha256', $request->getIP()))); } $log->setNamespace("http"); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($th->getMessage()); @@ -393,6 +395,7 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD"); $log->setAction($action); + $log->addTag('service', $action); $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); diff --git a/app/realtime.php b/app/realtime.php index cfb9bf9a16..03eaa53bcc 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -184,7 +184,7 @@ $logError = function (Throwable $error, string $action) use ($register) { $log = new Log(); $log->setNamespace("realtime"); - $log->setServer(gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); diff --git a/app/worker.php b/app/worker.php index 280e170028..7e4eafeea2 100644 --- a/app/worker.php +++ b/app/worker.php @@ -301,7 +301,7 @@ Server::setResource('logError', function (Registry $register, Document $project) $log = new Log(); $log->setNamespace($namespace); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); @@ -394,7 +394,7 @@ $worker if ($logger) { $log->setNamespace("appwrite-worker"); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); From ca6cd349fb6e033af13d69b76dd6d1568df70658 Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 27 Nov 2024 14:35:58 +0200 Subject: [PATCH 23/25] Remove failed attribute --- src/Appwrite/Platform/Workers/Databases.php | 28 ++++++++++----------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 763967bb56..b027b3e707 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -251,23 +251,21 @@ class Databases extends Action try { try { - if ($status !== 'failed') { - if ($type === Database::VAR_RELATIONSHIP) { - if ($options['twoWay']) { - $relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']); - if ($relatedCollection->isEmpty()) { - throw new DatabaseException('Collection not found'); - } - $relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']); + if ($type === Database::VAR_RELATIONSHIP) { + if ($options['twoWay']) { + $relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']); + if ($relatedCollection->isEmpty()) { + throw new DatabaseException('Collection not found'); } - - if (!$dbForProject->deleteRelationship('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) { - $dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'stuck')); - throw new DatabaseException('Failed to delete Relationship'); - } - } elseif (!$dbForProject->deleteAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) { - throw new DatabaseException('Failed to delete Attribute'); + $relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']); } + + if (!$dbForProject->deleteRelationship('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) { + $dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'stuck')); + throw new DatabaseException('Failed to delete Relationship'); + } + } elseif (!$dbForProject->deleteAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) { + throw new DatabaseException('Failed to delete Attribute'); } $dbForProject->deleteDocument('attributes', $attribute->getId()); From 98864b04e133918553ce7525fc7d88ed5c086cce Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 27 Nov 2024 18:02:37 +0200 Subject: [PATCH 24/25] fix $options --- app/controllers/api/databases.php | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index a00324f3b7..aad072c50a 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -290,15 +290,9 @@ function updateAttribute( $attribute->setAttribute('size', $size); } - $formatOptions = $attribute->getAttribute('formatOptions'); - switch ($attribute->getAttribute('format')) { case APP_DATABASE_ATTRIBUTE_INT_RANGE: case APP_DATABASE_ATTRIBUTE_FLOAT_RANGE: - if ($min === $formatOptions['min'] && $max === $formatOptions['max']) { - break; - } - if ($min > $max) { throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Minimum value must be lesser than maximum value'); } @@ -385,7 +379,7 @@ function updateAttribute( size: $size, required: $required, default: $default, - formatOptions: $options ?? null, + formatOptions: $options, newKey: $newKey ?? null ); } catch (TruncateException) { From c062a8f1299d0980ae39130cbb8401aa62933072 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Wed, 27 Nov 2024 16:47:32 +0000 Subject: [PATCH 25/25] Bump console to version 5.2.27 --- app/views/install/compose.phtml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 361685d1ed..ad6d883c4c 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -167,7 +167,7 @@ $image = $this->getParam('image', ''); appwrite-console: <<: *x-logging container_name: appwrite-console - image: /console:5.2.25 + image: /console:5.2.27 restart: unless-stopped networks: - appwrite diff --git a/docker-compose.yml b/docker-compose.yml index ef8d898250..f9a79ee84d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -199,7 +199,7 @@ services: appwrite-console: <<: *x-logging container_name: appwrite-console - image: appwrite/console:5.2.25 + image: appwrite/console:5.2.27 restart: unless-stopped networks: - appwrite