From 54c24b24e603c96e73cfa61dba0b074ebe0ac425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 6 Nov 2024 10:52:43 +0000 Subject: [PATCH 01/19] Improve preview domains local development flow --- CONTRIBUTING.md | 12 ++++++++++++ app/controllers/general.php | 33 ++++++++++++++++++++------------- app/init.php | 11 +++++++++++ 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b92361e51a..e89aa369cf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -613,6 +613,18 @@ If you need to clear the cache, you can do so by running the following command: docker compose exec redis redis-cli FLUSHALL ``` +## Using preview domains locally + +Appwrite Functions are automatically given a domain you can visit to execute the function. This domain has format `[SOMETHING].functions.localhost` unless you changed `_APP_DOMAIN_FUNCTIONS` environment variable. This default value works great when running Appwrite locally, but it can be impossible to use preview domains with Cloud woekspaces such as Gitpod or GitHub Codespaces. + +To use preview domains on Cloud workspaces, you can visit hostname provided by them, and supply function's preview domain as URL parameter: + +``` +https://8080-appwrite-appwrite-mjeb3ebilwv.ws-eu116.gitpod.io/ping?preview=672b3c7eab1ac523ccf5.functions.localhost +``` + +The path was set to `/ping` intentionally. Visiting `/` for preview domains might trigger Console background worker, and trigger redirect to Console without our preview URL param. Visiting different path ensures this doesnt happen. + ## Tutorials From time to time, our team will add tutorials that will help contributors find their way in the Appwrite source code. Below is a list of currently available tutorials: diff --git a/app/controllers/general.php b/app/controllers/general.php index b08ecb3d12..1d1e4055e4 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -46,11 +46,14 @@ Config::setParam('domainVerification', false); Config::setParam('cookieDomain', 'localhost'); Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) +function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { $utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml'); $host = $request->getHostname() ?? ''; + if (!empty($previewHost)) { + $host = $previewHost; + } $route = Authorization::skip( fn () => $dbForConsole->find('rules', [ @@ -462,15 +465,16 @@ App::init() ->inject('queueForCertificates') ->inject('queueForFunctions') ->inject('isResourceBlocked') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked) { + ->inject('previewHost') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHost) { /* * Appwrite Router */ $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain) { - if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked)) { + if ($host !== $mainDomain || !empty($previewHost)) { + if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost)) { return; } } @@ -681,15 +685,16 @@ App::options() ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) { + ->inject('previewHost') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { /* * Appwrite Router */ $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain) { - if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked)) { + if ($host !== $mainDomain || !empty($previewHost)) { + if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost)) { return; } } @@ -976,15 +981,16 @@ App::get('/robots.txt') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) { + ->inject('previewHost') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); - if ($host === $mainDomain || $host === 'localhost') { + if (($host === $mainDomain || $host === 'localhost') && empty($previewHost)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked); + router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost); } }); @@ -1003,15 +1009,16 @@ App::get('/humans.txt') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) { + ->inject('previewHost') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); - if ($host === $mainDomain || $host === 'localhost') { + if (($host === $mainDomain || $host === 'localhost') && empty($previewHost)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked); + router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost); } }); diff --git a/app/init.php b/app/init.php index d062e218e9..433d2092c5 100644 --- a/app/init.php +++ b/app/init.php @@ -1827,3 +1827,14 @@ App::setResource( 'isResourceBlocked', fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false ); + +App::setResource('previewHost', function (Request $request) { + if (App::isDevelopment()) { + $host = $request->getQuery('preview') ?? ''; + if (!empty($host)) { + return $host; + } + } + + return ''; +}, ['request']); From ca887f018e5343fe8f50d21e98776ce6da17ae54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 6 Nov 2024 11:29:24 +0000 Subject: [PATCH 02/19] fix domain execution response size --- app/controllers/general.php | 42 +++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 1d1e4055e4..467c8b0aca 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -355,26 +355,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo throw $th; } } finally { - $fileSize = 0; - $file = $request->getFiles('file'); - if (!empty($file)) { - $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; - } - - $queueForUsage - ->addMetric(METRIC_NETWORK_REQUESTS, 1) - ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) - ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) - ->addMetric(METRIC_EXECUTIONS, 1) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) - ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function - ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->setProject($project) - ->trigger() - ; - $queueForFunctions ->setType(Func::TYPE_ASYNC_WRITE) ->setExecution($execution) @@ -409,6 +389,28 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo ->setStatusCode($execution['responseStatusCode'] ?? 200) ->send($body); + $fileSize = 0; + $file = $request->getFiles('file'); + if (!empty($file)) { + $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + } + + \var_dump($response->getSize()); + + $queueForUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) + ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) + ->addMetric(METRIC_EXECUTIONS, 1) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) + ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function + ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) + ->setProject($project) + ->trigger() + ; + return true; } elseif ($type === 'api') { $utopia->getRoute()?->label('error', ''); From 87cfd7df3ba527cb26ad624c353bb00e8e6c9f68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 6 Nov 2024 11:39:44 +0000 Subject: [PATCH 03/19] Remove leftover --- app/controllers/general.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 467c8b0aca..e453fa3200 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -395,8 +395,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - \var_dump($response->getSize()); - $queueForUsage ->addMetric(METRIC_NETWORK_REQUESTS, 1) ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) From c9e6ef820211c73506d1129e3887b7f8732384b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 6 Nov 2024 16:05:58 +0000 Subject: [PATCH 04/19] Rename based on PR reviews --- app/controllers/general.php | 38 ++++++++++++++++++------------------- app/init.php | 4 ++-- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 1d1e4055e4..a8ca5985f8 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -46,13 +46,13 @@ Config::setParam('domainVerification', false); Config::setParam('cookieDomain', 'localhost'); Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) +function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml'); $host = $request->getHostname() ?? ''; - if (!empty($previewHost)) { - $host = $previewHost; + if (!empty($previewHostname)) { + $host = $previewHostname; } $route = Authorization::skip( @@ -465,16 +465,16 @@ App::init() ->inject('queueForCertificates') ->inject('queueForFunctions') ->inject('isResourceBlocked') - ->inject('previewHost') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHost) { + ->inject('previewHostname') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHostname) { /* * Appwrite Router */ $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain || !empty($previewHost)) { - if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost)) { + if ($host !== $mainDomain || !empty($previewHostname)) { + if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { return; } } @@ -685,16 +685,16 @@ App::options() ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->inject('previewHost') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { + ->inject('previewHostname') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { /* * Appwrite Router */ $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain || !empty($previewHost)) { - if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost)) { + if ($host !== $mainDomain || !empty($previewHostname)) { + if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { return; } } @@ -981,16 +981,16 @@ App::get('/robots.txt') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->inject('previewHost') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { + ->inject('previewHostname') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); - if (($host === $mainDomain || $host === 'localhost') && empty($previewHost)) { + if (($host === $mainDomain || $host === 'localhost') && empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost); + router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); } }); @@ -1009,16 +1009,16 @@ App::get('/humans.txt') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->inject('previewHost') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHost) { + ->inject('previewHostname') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); - if (($host === $mainDomain || $host === 'localhost') && empty($previewHost)) { + if (($host === $mainDomain || $host === 'localhost') && empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHost); + router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); } }); diff --git a/app/init.php b/app/init.php index 433d2092c5..c7a87c8dc9 100644 --- a/app/init.php +++ b/app/init.php @@ -1828,9 +1828,9 @@ App::setResource( fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false ); -App::setResource('previewHost', function (Request $request) { +App::setResource('previewHostname', function (Request $request) { if (App::isDevelopment()) { - $host = $request->getQuery('preview') ?? ''; + $host = $request->getQuery('appwrite-hostname') ?? ''; if (!empty($host)) { return $host; } From 842e23220ae3250f502be62a3e9c9cb974e2fc6b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 19 Nov 2024 07:51:50 +0545 Subject: [PATCH 05/19] update project last activity on error hook --- app/controllers/general.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/controllers/general.php b/app/controllers/general.php index 0bbfa2b694..f7f15138a6 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -702,6 +702,22 @@ App::error() ->inject('log') ->inject('queueForUsage') ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) { + /** + * Update project last activity + */ + if (!$project->isEmpty() && $project->getId() !== 'console') { + try { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); + } + } catch(Throwable $th) { + Console::error('[Error] updating project\'s last activity'); + Console::error($th->getMessage()); + } + } + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); From e8079cfd295a5423639a0e4babba6773c1136e65 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 19 Nov 2024 06:59:30 +0000 Subject: [PATCH 06/19] lint --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index f7f15138a6..4bf45f3860 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -712,7 +712,7 @@ App::error() $project->setAttribute('accessedAt', DateTime::now()); Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); } - } catch(Throwable $th) { + } catch (Throwable $th) { Console::error('[Error] updating project\'s last activity'); Console::error($th->getMessage()); } From 2da2b5bb7fe2423c6f14639d17496bb46daf0151 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 20 Nov 2024 04:25:37 +0000 Subject: [PATCH 07/19] reset error hook --- app/controllers/general.php | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 4bf45f3860..0bbfa2b694 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -702,22 +702,6 @@ App::error() ->inject('log') ->inject('queueForUsage') ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) { - /** - * Update project last activity - */ - if (!$project->isEmpty() && $project->getId() !== 'console') { - try { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); - } - } catch (Throwable $th) { - Console::error('[Error] updating project\'s last activity'); - Console::error($th->getMessage()); - } - } - $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); From c33c5732e0758a5c9413292dc1b9af925e60c285 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 20 Nov 2024 04:29:57 +0000 Subject: [PATCH 08/19] move accessed at to init --- app/controllers/shared/api.php | 63 ++++++++++++++++------------------ 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 6d87940ff7..a78bf1d88f 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -155,12 +155,13 @@ App::init() ->inject('utopia') ->inject('request') ->inject('dbForConsole') + ->inject('dbForProject') ->inject('project') ->inject('user') ->inject('session') ->inject('servers') ->inject('mode') - ->action(function (App $utopia, Request $request, Database $dbForConsole, Document $project, Document $user, ?Document $session, array $servers, string $mode) { + ->action(function (App $utopia, Request $request, Database $dbForConsole, Database $dbForProject, Document $project, Document $user, ?Document $session, array $servers, string $mode) { $route = $utopia->getRoute(); if ($project->isEmpty()) { @@ -301,6 +302,33 @@ App::init() Authorization::setRole($authRole); } + /** + * Update project last activity + */ + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); + } + } + + /** + * Update user last activity + */ + if (!$user->isEmpty()) { + $accessedAt = $user->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { + $user->setAttribute('accessedAt', DateTime::now()); + + if (APP_MODE_ADMIN !== $mode) { + $dbForProject->updateDocument('users', $user->getId(), $user); + } else { + $dbForConsole->updateDocument('users', $user->getId(), $user); + } + } + } + $service = $route->getLabel('sdk.namespace', ''); if (!empty($service)) { if ( @@ -588,9 +616,7 @@ App::shutdown() ->inject('queueForMessaging') ->inject('dbForProject') ->inject('queueForFunctions') - ->inject('mode') - ->inject('dbForConsole') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Database $dbForProject, Func $queueForFunctions) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -746,8 +772,6 @@ App::shutdown() } } - - if ($project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; @@ -766,33 +790,6 @@ App::shutdown() ->setProject($project) ->trigger(); } - - /** - * Update project last activity - */ - if (!$project->isEmpty() && $project->getId() !== 'console') { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); - } - } - - /** - * Update user last activity - */ - if (!$user->isEmpty()) { - $accessedAt = $user->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { - $user->setAttribute('accessedAt', DateTime::now()); - - if (APP_MODE_ADMIN !== $mode) { - $dbForProject->updateDocument('users', $user->getId(), $user); - } else { - $dbForConsole->updateDocument('users', $user->getId(), $user); - } - } - } }); App::init() From 98d78163a2b8de2c0ccf90ef5131fd28945bc526 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Fri, 22 Nov 2024 22:39:24 +0000 Subject: [PATCH 09/19] Bump console to version 5.2.25 --- app/views/install/compose.phtml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 8d7fecb479..361685d1ed 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -167,7 +167,7 @@ $image = $this->getParam('image', ''); appwrite-console: <<: *x-logging container_name: appwrite-console - image: /console:5.0.12 + image: /console:5.2.25 restart: unless-stopped networks: - appwrite diff --git a/docker-compose.yml b/docker-compose.yml index bae2cc7811..36a642f8f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -199,7 +199,7 @@ services: appwrite-console: <<: *x-logging container_name: appwrite-console - image: appwrite/console:5.0.12 + image: appwrite/console:5.2.25 restart: unless-stopped networks: - appwrite From 07b3cbaec9de9ae586b224a0de2ce401027d8dcf Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Fri, 22 Nov 2024 22:47:02 +0000 Subject: [PATCH 10/19] Bump appwrite version to 1.6.1 --- README-CN.md | 6 +++--- README.md | 6 +++--- app/init.php | 2 +- src/Appwrite/Migration/Migration.php | 1 + 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README-CN.md b/README-CN.md index a5eac1be03..92a9bf9806 100644 --- a/README-CN.md +++ b/README-CN.md @@ -67,7 +67,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` ### Windows @@ -79,7 +79,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` #### PowerShell @@ -89,7 +89,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` 运行后,可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后,服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index a9856a7310..ab57e65c4c 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` ### Windows @@ -87,7 +87,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` #### PowerShell @@ -97,7 +97,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/init.php b/app/init.php index 047e9402b6..d629f1a0b3 100644 --- a/app/init.php +++ b/app/init.php @@ -123,7 +123,7 @@ const APP_USER_ACCESS = 24 * 60 * 60; // 24 hours const APP_PROJECT_ACCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 4318; -const APP_VERSION_STABLE = '1.6.0'; +const APP_VERSION_STABLE = '1.6.1'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index cee1b2d263..19f69b1a4f 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -91,6 +91,7 @@ abstract class Migration '1.5.10' => 'V20', '1.5.11' => 'V20', '1.6.0' => 'V21', + '1.6.1' => 'V21', ]; /** From 9d3bc1f80ae985e295f9e7b0fa62fdd0e901d7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 25 Nov 2024 11:04:04 +0100 Subject: [PATCH 11/19] Fix swoole task warning --- app/http.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/http.php b/app/http.php index c80fccee54..7387b3a43b 100644 --- a/app/http.php +++ b/app/http.php @@ -54,6 +54,7 @@ $http 'http_compression' => false, 'package_max_length' => $payloadSize, 'buffer_output_size' => $payloadSize, + 'task_worker_num' => 1, // required for the task to fetch domains background ]); $http->on(Constant::EVENT_WORKER_START, function ($server, $workerId) { From cb385674a52ea9978243a334dd2c677dc3aacfc1 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Mon, 25 Nov 2024 15:58:22 +0000 Subject: [PATCH 12/19] Add 1.6.1 to CHANGES.md --- CHANGES.md | 114 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 9b6172eeab..62db3d525e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,117 @@ +# Version 1.6.1 + +## What's Changed + +### Notable changes + +* Remove JPEG fallback for webp by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8746 +* Add heic and avif support by @lohanidamodar in https://github.com/appwrite/appwrite/pull/7718 +* Add new runtimes by @Meldiron in https://github.com/appwrite/appwrite/pull/8771 +* Remove audits deletion by @shimonewman in https://github.com/appwrite/appwrite/pull/8766 +* Bump assistant by @loks0n in https://github.com/appwrite/appwrite/pull/8801 +* Change max queries values to 500 by @fogelito in https://github.com/appwrite/appwrite/pull/8802 +* Allow '.wav' as 'audio/x-wav' as well by @basert in https://github.com/appwrite/appwrite/pull/8846 +* Use 1 instead of 0.5 cpu for default function specification by @loks0n in https://github.com/appwrite/appwrite/pull/8848 +* Update function runtimes by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8781 +* Add a realtime heartbeat by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8943 + +### Fixes + +* Trigger functions event only if event is not paused by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8526 +* Update docker-compose to restart usage-dump by @feschaffa in https://github.com/appwrite/appwrite/pull/8642 +* Fix typo in scheduler base by @fogelito in https://github.com/appwrite/appwrite/pull/8691 +* Add domain and force HTTPS env vars to mail worker by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8722 +* Fix webp by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8732 +* Ignore junction tables by @fogelito in https://github.com/appwrite/appwrite/pull/8728 +* Fix logger throwing fatal error by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8724 +* Fix missing protocol for testing SMTP by @byawitz in https://github.com/appwrite/appwrite/pull/8749 +* Make create execution async loose by @loks0n in https://github.com/appwrite/appwrite/pull/8707 +* Fix invalid cursor value by @fogelito in https://github.com/appwrite/appwrite/pull/8109 +* Fix target deletes by @abnegate in https://github.com/appwrite/appwrite/pull/8833 +* Fix translation commas by @loks0n in https://github.com/appwrite/appwrite/pull/8892 +* Fix Migrations having source creds being overwritten and add Migration tests by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8897 +* Fix validator usage for updating string size by @abnegate in https://github.com/appwrite/appwrite/pull/8890 +* Fix create user event not triggering by @loks0n in https://github.com/appwrite/appwrite/pull/8718 +* Improve error handling and logging in the database worker by @fogelito in https://github.com/appwrite/appwrite/pull/8944 +* Remove inaccurate info about leaving the URL parameter empty by @ebenezerdon in https://github.com/appwrite/appwrite/pull/8963 +* Ensure indexes are updated when updating an attribute key by @fogelito in https://github.com/appwrite/appwrite/pull/8971 +* Remove duplicate dart-2.16 runtime template by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8972 +* Fix team invites with existing session by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/9006 +* Improve handling of HTTP requests by dispatching to safe workers by @Meldiron in https://github.com/appwrite/appwrite/pull/9016 +* Fix users create session secret by @stnguyen90 in https://github.com/appwrite/appwrite/pull/9019 +* Fix swoole task warning by @Meldiron in https://github.com/appwrite/appwrite/pull/9025 + +### Miscellaneous + +* Update Init copy by @adityaoberai in https://github.com/appwrite/appwrite/pull/8557 +* Fix security scan permissions and comment by @EVDOG4LIFE in https://github.com/appwrite/appwrite/pull/8525 +* Add Trivy security scans by @btme0011 in https://github.com/appwrite/appwrite/pull/6876 +* Update database stack by @abnegate in https://github.com/appwrite/appwrite/pull/8564 +* Bump database by @abnegate in https://github.com/appwrite/appwrite/pull/8573 +* Sync main with 1.5.x by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8589 +* Add AWS to one-click installs by @byawitz in https://github.com/appwrite/appwrite/pull/8593 +* Update Init copy in readme by @adityaoberai in https://github.com/appwrite/appwrite/pull/8618 +* Sync main into 1.6.x by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8685 +* Sync 1.6.x into main by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8686 +* Feat coroutines by @Meldiron in https://github.com/appwrite/appwrite/pull/7826 +* Sync main into 1.6.x by @Meldiron in https://github.com/appwrite/appwrite/pull/8719 +* Sentence casing endpoint API reference by @choir241 in https://github.com/appwrite/appwrite/pull/8617 +* DB storage metrics by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8404 +* Fix exception thrown when optional array attribute does not exist by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8391 +* Add projects channels to realtime by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8735 +* Base for console roles support by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8565 +* Remove DB disk storage calculation by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8745 +* Messaging adapter default values by @shimonewman in https://github.com/appwrite/appwrite/pull/8742 +* Add payload response type by @loks0n in https://github.com/appwrite/appwrite/pull/8720 +* Fix flaky functions tests by @loks0n in https://github.com/appwrite/appwrite/pull/8682 +* Migrations Backups by @fogelito in https://github.com/appwrite/appwrite/pull/8186 +* Add test for response and request filters by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8697 +* Bump version in SECURITY.md by @EVDOG4LIFE in https://github.com/appwrite/appwrite/pull/8755 +* Add originalId attribute to databases collection by @fogelito in https://github.com/appwrite/appwrite/pull/8764 +* Fix Walter References by @ItzNotABug in https://github.com/appwrite/appwrite/pull/8757 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8769 +* Move new attributes by @abnegate in https://github.com/appwrite/appwrite/pull/8777 +* Add ping endpoint by @loks0n in https://github.com/appwrite/appwrite/pull/8761 +* Fix GitHub action caching by @loks0n in https://github.com/appwrite/appwrite/pull/8772 +* Chore release ruby SDK by @abnegate in https://github.com/appwrite/appwrite/pull/8767 +* Call migration success on success by @abnegate in https://github.com/appwrite/appwrite/pull/8782 +* Update utopia-php/system to 0.9.0 by @basert in https://github.com/appwrite/appwrite/pull/8780 +* Move createDocument from api to worker by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8776 +* Add missing indexes by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8803 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8809 +* Fix typo in BLR region by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8756 +* Add tests for project variables by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8815 +* Replace 'Expires' with 'Cache-Control: private' header to avoid CDN caching by @basert in https://github.com/appwrite/appwrite/pull/8836 +* Allow blocking based on resource attributes by @basert in https://github.com/appwrite/appwrite/pull/8812 +* Check if resource is blocked inside functions worker by @basert in https://github.com/appwrite/appwrite/pull/8855 +* Fix missing allow attribute by @abnegate in https://github.com/appwrite/appwrite/pull/8889 +* Revert function execution order by @basert in https://github.com/appwrite/appwrite/pull/8857 +* Use resource type constants by @basert in https://github.com/appwrite/appwrite/pull/8895 +* Update Database lib by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8680 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8917 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8923 +* Update database for transaction counter fixes with retries by @abnegate in https://github.com/appwrite/appwrite/pull/8927 +* Validate string permissions by @fogelito in https://github.com/appwrite/appwrite/pull/8929 +* Add PubSub adapter support by @basert in https://github.com/appwrite/appwrite/pull/8905 +* List memberships as client by @loks0n in https://github.com/appwrite/appwrite/pull/8913 +* Fix XDebug Extension not being removed by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8891 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8946 +* Use utopia compression by @loks0n in https://github.com/appwrite/appwrite/pull/8938 +* Make compression minimum size configurable by @loks0n in https://github.com/appwrite/appwrite/pull/8947 +* Revert "Update database" by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8949 +* Fix setpaused by @loks0n in https://github.com/appwrite/appwrite/pull/8948 +* Use getDocument instead of find() for rules by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8951 +* Remove double fetch from migrations worker by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8956 +* Fix memberships privacy MFA by @loks0n in https://github.com/appwrite/appwrite/pull/8969 +* Add telemetry by @basert in https://github.com/appwrite/appwrite/pull/8960 +* Send migration errors individually by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8959 +* Add console sdk previews by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8990 +* Unset index length by @fogelito in https://github.com/appwrite/appwrite/pull/8978 +* Update base to 0.9.5 by @basert in https://github.com/appwrite/appwrite/pull/9005 +* Sync main into 1.6.x by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/9011 +* Improved shared tables V2 by @abnegate in https://github.com/appwrite/appwrite/pull/9013 +* Ensure backwards compatibility for 1.6.x by @christyjacob4 in https://github.com/appwrite/appwrite/pull/9018 + # Version 1.6.0 ## What's Changed From 580a9f331fc07b31ae2c5a068f5d6fc370334afb Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 26 Nov 2024 08:59:03 +0000 Subject: [PATCH 13/19] fix user update --- app/controllers/shared/api.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index a78bf1d88f..68c32cc969 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -316,7 +316,7 @@ App::init() /** * Update user last activity */ - if (!$user->isEmpty()) { + if (!empty($user->getId())) { $accessedAt = $user->getAttribute('accessedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { $user->setAttribute('accessedAt', DateTime::now()); From 3a9ba8a6ad6a76454bc9dad586373cd949fd037a Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 22 Oct 2024 13:57:25 +0200 Subject: [PATCH 14/19] feat: move certificate generation to Adapter --- app/init.php | 15 ++ src/Appwrite/Certificates/Adapter.php | 14 ++ .../Certificates/Adapter/LetsEncrypt.php | 126 +++++++++++ src/Appwrite/Certificates/AdapterProvider.php | 19 ++ src/Appwrite/Platform/Tasks/Maintenance.php | 1 + .../Platform/Workers/Certificates.php | 206 ++++-------------- src/Appwrite/Platform/Workers/Deletes.php | 49 ++--- 7 files changed, 234 insertions(+), 196 deletions(-) create mode 100644 src/Appwrite/Certificates/Adapter.php create mode 100644 src/Appwrite/Certificates/Adapter/LetsEncrypt.php create mode 100644 src/Appwrite/Certificates/AdapterProvider.php diff --git a/app/init.php b/app/init.php index 047e9402b6..55f1356d62 100644 --- a/app/init.php +++ b/app/init.php @@ -21,6 +21,8 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; +use Appwrite\Certificates\Adapter\LetsEncrypt; +use Appwrite\Certificates\AdapterProvider; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Certificate; @@ -1531,6 +1533,19 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); +App::setResource('letsEncryptCertificateAdapter', function () { + $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); + if (empty($email)) { + throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.'); + } + + return new LetsEncrypt($email); +}); + +App::setResource('adapterForCertificates', function ($letsEncrypt) { + return new AdapterProvider(fn (string $domain) => $letsEncrypt); +}, ['letsEncryptCertificateAdapter']); + App::setResource('deviceForLocal', function () { return new Local(); }); diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php new file mode 100644 index 0000000000..711e4c09b9 --- /dev/null +++ b/src/Appwrite/Certificates/Adapter.php @@ -0,0 +1,14 @@ +email = $email; + } + + + public function issueCertificate(string $certName, string $domain): ?string + { + $stdout = ''; + $stderr = ''; + + $staging = (App::isProduction()) ? '' : ' --dry-run'; + $exit = Console::execute( + "certbot certonly -v --webroot --noninteractive --agree-tos{$staging}" + . " --email " . $this->email + . " --cert-name " . $certName + . " -w " . APP_STORAGE_CERTIFICATES + . " -d {$domain}", + '', + $stdout, + $stderr + ); + + // Unexpected error, usually 5XX, API limits, ... + if ($exit !== 0) { + throw new Exception('Failed to issue a certificate with message: ' . $stderr); + } + + // Prepare folder in storage for domain + $path = APP_STORAGE_CERTIFICATES . '/' . $domain; + if (!\is_readable($path)) { + if (!\mkdir($path, 0755, true)) { + throw new Exception('Failed to create path for certificate.'); + } + } + + // Move generated files + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { + throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { + throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { + throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { + throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + $config = \implode(PHP_EOL, [ + "tls:", + " certificates:", + " - certFile: /storage/certificates/{$domain}/fullchain.pem", + " keyFile: /storage/certificates/{$domain}/privkey.pem" + ]); + + // Save configuration into Traefik using our new cert files + if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { + throw new Exception('Failed to save Traefik configuration.'); + } + + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? null; + $dt = (new \DateTime())->setTimestamp($validTo); + return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); + } + + public function isRenewRequired(string $domain, Log $log): bool + { + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + if (\file_exists($certPath)) { + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? 0; + + if (empty($validTo)) { + $log->addTag('certificateDomain', $domain); + throw new Exception('Unable to read certificate file (cert.pem).'); + } + + // LetsEncrypt allows renewal 30 days before expiry + $expiryInAdvance = (60 * 60 * 24 * 30); + if ($validTo - $expiryInAdvance > \time()) { + $log->addTag('certificateDomain', $domain); + $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData)); + return false; + } + } + + return true; + } + + public function deleteCertificate(string $domain): void + { + $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; + $checkTraversal = realpath($directory) === $directory; + + if ($checkTraversal && is_dir($directory)) { + // Delete files, so Traefik is aware of change + array_map('unlink', glob($directory . '/*.*')); + rmdir($directory); + Console::info("Deleted certificate files for {$domain}"); + } else { + Console::info("No certificate files found for {$domain}"); + } + } +} diff --git a/src/Appwrite/Certificates/AdapterProvider.php b/src/Appwrite/Certificates/AdapterProvider.php new file mode 100644 index 0000000000..bed73a0164 --- /dev/null +++ b/src/Appwrite/Certificates/AdapterProvider.php @@ -0,0 +1,19 @@ +decisionMaker = $decisionMaker; + } + + public function get(string $domain): Adapter + { + return call_user_func($this->decisionMaker, $domain); + } + +} diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index afb38f35fc..40e72dc683 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -149,6 +149,7 @@ class Maintenance extends Action $certificates = $dbForConsole->find('certificates', [ Query::lessThan('attempts', 5), // Maximum 5 attempts + Query::isNotNull('renewDate'), Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) ]); diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 6f1a710bc0..ecb4cbd127 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Workers; +use Appwrite\Certificates\AdapterProvider; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; @@ -11,7 +12,6 @@ use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; use Exception; use Throwable; -use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -48,7 +48,11 @@ class Certificates extends Action ->inject('queueForEvents') ->inject('queueForFunctions') ->inject('log') - ->callback(fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log)); + ->inject('adapterForCertificates') + ->callback( + fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates) => + $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $adapterForCertificates) + ); } /** @@ -58,11 +62,12 @@ class Certificates extends Action * @param Event $queueForEvents * @param Func $queueForFunctions * @param Log $log + * @param AdapterProvider $adapterForCertificates Retrieve the certificate adapter for the domain * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates): void { $payload = $message->getPayload() ?? []; @@ -76,7 +81,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck); + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $adapterForCertificates, $skipRenewCheck); } /** @@ -85,24 +90,24 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions + * @param AdapterProvider $adapterForCertificates Retrieve the certificate adapter for the domain * @param bool $skipRenewCheck * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain * 2. Get main domain * 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain) - * 4. Validate security email. Cannot be empty, required by LetsEncrypt - * 5. Validate renew date with certificate file, unless requested to skip by parameter - * 6. Issue a certificate using certbot CLI - * 7. Update 'log' attribute on certificate document with Certbot message - * 8. Create storage folder for certificate, if not ready already - * 9. Move certificates from Certbot location to our Storage - * 10. Create/Update our Storage with new Traefik config with new certificate paths + * 4. Validate renew date with certificate file, unless requested to skip by parameter + * 5. Issue a certificate using certbot CLI + * 6. Update 'log' attribute on certificate document with Certbot message + * 7. Create storage folder for certificate, if not ready already + * 8. Move certificates from Certbot location to our Storage + * 9. Create/Update our Storage with new Traefik config with new certificate paths * 11. Read certificate file and update 'renewDate' on certificate document * 12. Update 'issueDate' and 'attempts' on certificate * @@ -119,7 +124,7 @@ class Certificates extends Action * 2. Save document to database * 3. Update all domains documents with current certificate ID * - * Note: Renewals are checked and scheduled from maintenence worker + * Note: Renewals are checked and scheduled from maintenance worker */ // Get current certificate @@ -131,43 +136,32 @@ class Certificates extends Action $certificate->setAttribute('domain', $domain->get()); } + $certificateAdapter = $adapterForCertificates->get($domain->get()); + $success = false; try { - // Email for alerts is required by LetsEncrypt - $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); - if (empty($email)) { - throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue an SSL certificate.'); - } - // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { $mainDomain = $this->getMainDomain(); $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; $this->validateDomain($domain, $isMainDomain, $log); + + // If certificate exists already, double-check expiry date. Skip if job is forced + if (!$certificateAdapter->isRenewRequired($domain->get(), $log)) { + throw new Exception('Renew isn\'t required.'); + } } - // If certificate exists already, double-check expiry date. Skip if job is forced - if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) { - throw new Exception('Renew isn\'t required.'); - } - - // Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate - $folder = ID::unique(); - - // Generate certificate files using Let's Encrypt - $letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email); + // Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates. + $certName = ID::unique(); + $renewDate = $certificateAdapter->issueCertificate($certName, $domain->get()); // Command succeeded, store all data into document - $logs = 'Certificate successfully generated.'; - $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB - - - // Give certificates to Traefik - $this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData); + $certificate->setAttribute('logs', 'Certificate successfully generated.'); // Update certificate info stored in database - $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get())); + $certificate->setAttribute('renewDate', $renewDate); $certificate->setAttribute('attempts', 0); $certificate->setAttribute('issueDate', DateTime::now()); $success = true; @@ -181,7 +175,7 @@ class Certificates extends Action $attempts = $certificate->getAttribute('attempts', 0) + 1; $certificate->setAttribute('attempts', $attempts); - // Store cuttent time as renew date to ensure another attempt in next maintenance cycle + // Store current time as renew date to ensure another attempt in next maintenance cycle. $certificate->setAttribute('renewDate', DateTime::now()); // Send email to security email @@ -245,8 +239,8 @@ class Certificates extends Action } /** - * Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check: - * - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt) + * Internal domain validation functionality to prevent unnecessary attempts. We check: + * - Domain needs to be public and valid (prevents NFT domains that are not supported) * - Domain must have proper DNS record * * @param Domain $domain Domain which we validate @@ -292,136 +286,6 @@ class Certificates extends Action } } - /** - * Reads expiry date of certificate from file and decides if renewal is required or not. - * - * @param string $domain Domain for which we check certificate file - * @return bool True, if certificate needs to be renewed - * @throws Exception - */ - private function isRenewRequired(string $domain, Log $log): bool - { - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; - if (\file_exists($certPath)) { - $validTo = null; - - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? 0; - - if (empty($validTo)) { - $log->addTag('certificateDomain', $domain); - throw new Exception('Unable to read certificate file (cert.pem).'); - } - - // LetsEncrypt allows renewal 30 days before expiry - $expiryInAdvance = (60 * 60 * 24 * 30); - if ($validTo - $expiryInAdvance > \time()) { - $log->addTag('certificateDomain', $domain); - $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData)); - return false; - } - } - - return true; - } - - /** - * LetsEncrypt communication to issue certificate (using certbot CLI) - * - * @param string $folder Folder into which certificates should be generated - * @param string $domain Domain to generate certificate for - * @return array Named array with keys 'stdout' and 'stderr', both string - * @throws Exception - */ - private function issueCertificate(string $folder, string $domain, string $email): array - { - $stdout = ''; - $stderr = ''; - - $staging = (App::isProduction()) ? '' : ' --dry-run'; - $exit = Console::execute("certbot certonly -v --webroot --noninteractive --agree-tos{$staging}" - . " --email " . $email - . " --cert-name " . $folder - . " -w " . APP_STORAGE_CERTIFICATES - . " -d {$domain}", '', $stdout, $stderr); - - // Unexpected error, usually 5XX, API limits, ... - if ($exit !== 0) { - throw new Exception('Failed to issue a certificate with message: ' . $stderr); - } - - return [ - 'stdout' => $stdout, - 'stderr' => $stderr - ]; - } - - /** - * Read new renew date from certificate file generated by Let's Encrypt - * - * @param string $domain Domain which certificate was generated for - * @return string - * @throws \Utopia\Database\Exception - */ - private function getRenewDate(string $domain): string - { - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? null; - $dt = (new \DateTime())->setTimestamp($validTo); - return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days - } - - /** - * Method to take files from Let's Encrypt, and put it into Traefik. - * - * @param string $domain Domain which certificate was generated for - * @param string $folder Folder in which certificates were generated - * @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error - * @return void - * @throws Exception - */ - private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void - { - - // Prepare folder in storage for domain - $path = APP_STORAGE_CERTIFICATES . '/' . $domain; - if (!\is_readable($path)) { - if (!\mkdir($path, 0755, true)) { - throw new Exception('Failed to create path for certificate.'); - } - } - - // Move generated files - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - $config = \implode(PHP_EOL, [ - "tls:", - " certificates:", - " - certFile: /storage/certificates/{$domain}/fullchain.pem", - " keyFile: /storage/certificates/{$domain}/privkey.pem" - ]); - - // Save configuration into Traefik using our new cert files - if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { - throw new Exception('Failed to save Traefik configuration.'); - } - } - /** * Method to make sure information about error is delivered to admnistrator. * @@ -500,6 +364,10 @@ class Certificates extends Action $project = $dbForConsole->getDocument('projects', $projectId); + if ($project->isEmpty()) { + return; + } + /** Trigger Webhook */ $ruleModel = new Rule(); $queueForEvents diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 6c11116a4c..b4570e3929 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Auth\Auth; +use Appwrite\Certificates\AdapterProvider; use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; @@ -50,18 +51,22 @@ class Deletes extends Action ->inject('deviceForFunctions') ->inject('deviceForBuilds') ->inject('deviceForCache') + ->inject('adapterForCertificates') ->inject('abuseRetention') ->inject('executionRetention') ->inject('auditRetention') ->inject('log') - ->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log)); + ->callback( + fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $abuseRetention, $executionRetention, $auditRetention, $log) + ); } /** * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -84,10 +89,10 @@ class Deletes extends Action case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { case DELETE_TYPE_PROJECTS: - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); + $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $adapterProvider, $document, $project); break; case DELETE_TYPE_DEPLOYMENTS: $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); @@ -102,7 +107,7 @@ class Deletes extends Action $this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project); break; case DELETE_TYPE_RULES: - $this->deleteRule($dbForConsole, $document); + $this->deleteRule($dbForConsole, $document, $adapterProvider); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -110,7 +115,7 @@ class Deletes extends Action } break; case DELETE_TYPE_TEAM_PROJECTS: - $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $document); + $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $adapterProvider, $document); break; case DELETE_TYPE_EXECUTIONS: $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); @@ -442,7 +447,7 @@ class Deletes extends Action * @throws Structure * @throws Exception */ - private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Document $document): void + private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, AdapterProvider $adapterProvider, Document $document): void { $projects = $dbForConsole->find('projects', [ @@ -455,7 +460,7 @@ class Deletes extends Action $deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); $deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $project); $dbForConsole->deleteDocument('projects', $project->getId()); } } @@ -473,7 +478,7 @@ class Deletes extends Action * @throws Authorization * @throws DatabaseException */ - private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void + private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, Document $document): void { $projectInternalId = $document->getInternalId(); $projectId = $document->getId(); @@ -537,8 +542,8 @@ class Deletes extends Action // Delete project and function rules $this->deleteByGroup('rules', [ Query::equal('projectInternalId', [$projectInternalId]) - ], $dbForConsole, function (Document $document) use ($dbForConsole) { - $this->deleteRule($dbForConsole, $document); + ], $dbForConsole, function (Document $document) use ($dbForConsole, $adapterProvider) { + $this->deleteRule($dbForConsole, $document, $adapterProvider); }); // Delete Keys @@ -746,7 +751,7 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void + private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, AdapterProvider $adapterProvider, Document $document, Document $project): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -761,8 +766,8 @@ class Deletes extends Action Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('projectInternalId', [$project->getInternalId()]) - ], $dbForConsole, function (Document $document) use ($project, $dbForConsole) { - $this->deleteRule($dbForConsole, $document); + ], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $adapterProvider) { + $this->deleteRule($dbForConsole, $document, $adapterProvider); }); /** @@ -1048,21 +1053,11 @@ class Deletes extends Action * @param Document $document rule document * @return void */ - private function deleteRule(Database $dbForConsole, Document $document): void + private function deleteRule(Database $dbForConsole, Document $document, AdapterProvider $adapterForCertificate): void { - $domain = $document->getAttribute('domain'); - $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; - $checkTraversal = realpath($directory) === $directory; - - if ($checkTraversal && is_dir($directory)) { - // Delete files, so Traefik is aware of change - array_map('unlink', glob($directory . '/*.*')); - rmdir($directory); - Console::info("Deleted certificate files for {$domain}"); - } else { - Console::info("No certificate files found for {$domain}"); - } + $adapter = $adapterForCertificate->get($domain); + $adapter->deleteCertificate($domain); // Delete certificate document, so Appwrite is aware of change if (isset($document['certificateId'])) { From 98984a808500264fb2b3ce14801a7fb062e33d39 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Wed, 20 Nov 2024 11:10:57 +0100 Subject: [PATCH 15/19] refactor: use single adapter --- app/init.php | 15 ------- app/worker.php | 11 ++++- docker-compose.yml | 1 + src/Appwrite/Certificates/AdapterProvider.php | 19 --------- .../{Adapter => }/LetsEncrypt.php | 3 +- .../Platform/Workers/Certificates.php | 24 +++++------ src/Appwrite/Platform/Workers/Deletes.php | 41 +++++++++---------- 7 files changed, 43 insertions(+), 71 deletions(-) delete mode 100644 src/Appwrite/Certificates/AdapterProvider.php rename src/Appwrite/Certificates/{Adapter => }/LetsEncrypt.php (98%) diff --git a/app/init.php b/app/init.php index 55f1356d62..047e9402b6 100644 --- a/app/init.php +++ b/app/init.php @@ -21,8 +21,6 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; -use Appwrite\Certificates\Adapter\LetsEncrypt; -use Appwrite\Certificates\AdapterProvider; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Certificate; @@ -1533,19 +1531,6 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); -App::setResource('letsEncryptCertificateAdapter', function () { - $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); - if (empty($email)) { - throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.'); - } - - return new LetsEncrypt($email); -}); - -App::setResource('adapterForCertificates', function ($letsEncrypt) { - return new AdapterProvider(fn (string $domain) => $letsEncrypt); -}, ['letsEncryptCertificateAdapter']); - App::setResource('deviceForLocal', function () { return new Local(); }); diff --git a/app/worker.php b/app/worker.php index 7899c1d5bc..280e170028 100644 --- a/app/worker.php +++ b/app/worker.php @@ -2,6 +2,7 @@ require_once __DIR__ . '/init.php'; +use Appwrite\Certificates\LetsEncrypt; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Certificate; @@ -16,7 +17,6 @@ use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; -use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -283,6 +283,15 @@ Server::setResource( fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false ); +Server::setResource('certificates', function () { + $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); + if (empty($email)) { + throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.'); + } + + return new LetsEncrypt($email); +}); + Server::setResource('logError', function (Registry $register, Document $project) { return function (Throwable $error, string $namespace, string $action, ?array $extras) use ($register, $project) { $logger = $register->get('logger'); diff --git a/docker-compose.yml b/docker-compose.yml index 36a642f8f6..ef8d898250 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -385,6 +385,7 @@ services: - _APP_EXECUTOR_HOST - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 + - _APP_EMAIL_CERTIFICATES appwrite-worker-databases: entrypoint: worker-databases diff --git a/src/Appwrite/Certificates/AdapterProvider.php b/src/Appwrite/Certificates/AdapterProvider.php deleted file mode 100644 index bed73a0164..0000000000 --- a/src/Appwrite/Certificates/AdapterProvider.php +++ /dev/null @@ -1,19 +0,0 @@ -decisionMaker = $decisionMaker; - } - - public function get(string $domain): Adapter - { - return call_user_func($this->decisionMaker, $domain); - } - -} diff --git a/src/Appwrite/Certificates/Adapter/LetsEncrypt.php b/src/Appwrite/Certificates/LetsEncrypt.php similarity index 98% rename from src/Appwrite/Certificates/Adapter/LetsEncrypt.php rename to src/Appwrite/Certificates/LetsEncrypt.php index b2237189e9..3896eab022 100644 --- a/src/Appwrite/Certificates/Adapter/LetsEncrypt.php +++ b/src/Appwrite/Certificates/LetsEncrypt.php @@ -1,8 +1,7 @@ inject('queueForEvents') ->inject('queueForFunctions') ->inject('log') - ->inject('adapterForCertificates') + ->inject('certificates') ->callback( - fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates) => - $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $adapterForCertificates) + fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates) => + $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates) ); } @@ -62,12 +62,12 @@ class Certificates extends Action * @param Event $queueForEvents * @param Func $queueForFunctions * @param Log $log - * @param AdapterProvider $adapterForCertificates Retrieve the certificate adapter for the domain + * @param CertificatesAdapter $certificates * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates): void + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates): void { $payload = $message->getPayload() ?? []; @@ -81,7 +81,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $adapterForCertificates, $skipRenewCheck); + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates, $skipRenewCheck); } /** @@ -90,13 +90,13 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions - * @param AdapterProvider $adapterForCertificates Retrieve the certificate adapter for the domain + * @param CertificatesAdapter $certificates * @param bool $skipRenewCheck * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, AdapterProvider $adapterForCertificates, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain @@ -136,8 +136,6 @@ class Certificates extends Action $certificate->setAttribute('domain', $domain->get()); } - $certificateAdapter = $adapterForCertificates->get($domain->get()); - $success = false; try { @@ -148,14 +146,14 @@ class Certificates extends Action $this->validateDomain($domain, $isMainDomain, $log); // If certificate exists already, double-check expiry date. Skip if job is forced - if (!$certificateAdapter->isRenewRequired($domain->get(), $log)) { + if (!$certificates->isRenewRequired($domain->get(), $log)) { throw new Exception('Renew isn\'t required.'); } } // Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates. $certName = ID::unique(); - $renewDate = $certificateAdapter->issueCertificate($certName, $domain->get()); + $renewDate = $certificates->issueCertificate($certName, $domain->get()); // Command succeeded, store all data into document $certificate->setAttribute('logs', 'Certificate successfully generated.'); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index b4570e3929..e6002bf75f 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -3,7 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Auth\Auth; -use Appwrite\Certificates\AdapterProvider; +use Appwrite\Certificates\Adapter as CertificatesAdapter; use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; @@ -51,14 +51,14 @@ class Deletes extends Action ->inject('deviceForFunctions') ->inject('deviceForBuilds') ->inject('deviceForCache') - ->inject('adapterForCertificates') + ->inject('certificates') ->inject('abuseRetention') ->inject('executionRetention') ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $abuseRetention, $executionRetention, $auditRetention, $log) + fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -89,10 +89,10 @@ class Deletes extends Action case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { case DELETE_TYPE_PROJECTS: - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $document); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $adapterProvider, $document, $project); + $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project); break; case DELETE_TYPE_DEPLOYMENTS: $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); @@ -107,7 +107,7 @@ class Deletes extends Action $this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project); break; case DELETE_TYPE_RULES: - $this->deleteRule($dbForConsole, $document, $adapterProvider); + $this->deleteRule($dbForConsole, $document, $certificates); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -115,7 +115,7 @@ class Deletes extends Action } break; case DELETE_TYPE_TEAM_PROJECTS: - $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $adapterProvider, $document); + $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $certificates, $document); break; case DELETE_TYPE_EXECUTIONS: $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); @@ -269,7 +269,7 @@ class Deletes extends Action MESSAGE_TYPE_EMAIL => 'emailTotal', MESSAGE_TYPE_SMS => 'smsTotal', MESSAGE_TYPE_PUSH => 'pushTotal', - default => throw new Exception('Invalid target provider type'), + default => throw new Exception('Invalid target CertificatesAdapter type'), }; $dbForProject->decreaseDocumentAttribute( 'topics', @@ -447,7 +447,7 @@ class Deletes extends Action * @throws Structure * @throws Exception */ - private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, AdapterProvider $adapterProvider, Document $document): void + private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void { $projects = $dbForConsole->find('projects', [ @@ -460,7 +460,7 @@ class Deletes extends Action $deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); $deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $adapterProvider, $project); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $project); $dbForConsole->deleteDocument('projects', $project->getId()); } } @@ -478,7 +478,7 @@ class Deletes extends Action * @throws Authorization * @throws DatabaseException */ - private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, AdapterProvider $adapterProvider, Document $document): void + private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Document $document): void { $projectInternalId = $document->getInternalId(); $projectId = $document->getId(); @@ -542,8 +542,8 @@ class Deletes extends Action // Delete project and function rules $this->deleteByGroup('rules', [ Query::equal('projectInternalId', [$projectInternalId]) - ], $dbForConsole, function (Document $document) use ($dbForConsole, $adapterProvider) { - $this->deleteRule($dbForConsole, $document, $adapterProvider); + ], $dbForConsole, function (Document $document) use ($dbForConsole, $certificates) { + $this->deleteRule($dbForConsole, $document, $certificates); }); // Delete Keys @@ -751,7 +751,7 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, AdapterProvider $adapterProvider, Document $document, Document $project): void + private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -766,8 +766,8 @@ class Deletes extends Action Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('projectInternalId', [$project->getInternalId()]) - ], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $adapterProvider) { - $this->deleteRule($dbForConsole, $document, $adapterProvider); + ], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $certificates) { + $this->deleteRule($dbForConsole, $document, $certificates); }); /** @@ -1053,11 +1053,10 @@ class Deletes extends Action * @param Document $document rule document * @return void */ - private function deleteRule(Database $dbForConsole, Document $document, AdapterProvider $adapterForCertificate): void + private function deleteRule(Database $dbForConsole, Document $document, CertificatesAdapter $certificates): void { $domain = $document->getAttribute('domain'); - $adapter = $adapterForCertificate->get($domain); - $adapter->deleteCertificate($domain); + $certificates->deleteCertificate($domain); // Delete certificate document, so Appwrite is aware of change if (isset($document['certificateId'])) { From 442459f6a482121f573168458cafb767875467ec Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 26 Nov 2024 16:14:38 +0200 Subject: [PATCH 16/19] fix restoration --- composer.json | 2 +- composer.lock | 39 ++++++++++++++++++++++++--------------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/composer.json b/composer.json index e3fbcabc83..5fb781083c 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.12.*", - "utopia-php/migration": "0.6.*", + "utopia-php/migration": "dev-0.6.x-fix-structore-validation as 0.6.13", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.5.*", diff --git a/composer.lock b/composer.lock index 87d8b47583..c742097e13 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ae3b9a491c9870a4897cdf712caba1e3", + "content-hash": "8e458c412c8a8ce3ce018b7a2c690832", "packages": [ { "name": "adhocore/jwt", @@ -157,16 +157,16 @@ }, { "name": "appwrite/php-runtimes", - "version": "0.16.4", + "version": "0.16.5", "source": { "type": "git", "url": "https://github.com/appwrite/runtimes.git", - "reference": "7e4741337b9373f77210396e68eca539018cabd1" + "reference": "1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/runtimes/zipball/7e4741337b9373f77210396e68eca539018cabd1", - "reference": "7e4741337b9373f77210396e68eca539018cabd1", + "url": "https://api.github.com/repos/appwrite/runtimes/zipball/1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9", + "reference": "1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9", "shasum": "" }, "require": { @@ -206,9 +206,9 @@ ], "support": { "issues": "https://github.com/appwrite/runtimes/issues", - "source": "https://github.com/appwrite/runtimes/tree/0.16.4" + "source": "https://github.com/appwrite/runtimes/tree/0.16.5" }, - "time": "2024-10-26T10:39:59+00:00" + "time": "2024-11-25T15:17:06+00:00" }, { "name": "beberlei/assert", @@ -3928,16 +3928,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.12", + "version": "dev-0.6.x-fix-structore-validation", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "9a8c905af4cece5c5ec9542a5b534befce067260" + "reference": "ed735136dd10a88ab8601a9831e9cbcf3b6d6560" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/9a8c905af4cece5c5ec9542a5b534befce067260", - "reference": "9a8c905af4cece5c5ec9542a5b534befce067260", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/ed735136dd10a88ab8601a9831e9cbcf3b6d6560", + "reference": "ed735136dd10a88ab8601a9831e9cbcf3b6d6560", "shasum": "" }, "require": { @@ -3978,9 +3978,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.12" + "source": "https://github.com/utopia-php/migration/tree/0.6.x-fix-structore-validation" }, - "time": "2024-11-12T00:31:53+00:00" + "time": "2024-11-26T11:04:12+00:00" }, { "name": "utopia-php/mongo", @@ -8555,9 +8555,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/migration", + "version": "dev-0.6.x-fix-structore-validation", + "alias": "0.6.13", + "alias_normalized": "0.6.13.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/migration": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From 135ff1a72400487c827033e3943a96dd322bea4e Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 26 Nov 2024 17:05:17 +0200 Subject: [PATCH 17/19] bomb migration 0.6.13 --- composer.json | 2 +- composer.lock | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/composer.json b/composer.json index 5fb781083c..e3fbcabc83 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.12.*", - "utopia-php/migration": "dev-0.6.x-fix-structore-validation as 0.6.13", + "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.5.*", diff --git a/composer.lock b/composer.lock index c742097e13..bdb41c2478 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8e458c412c8a8ce3ce018b7a2c690832", + "content-hash": "ae3b9a491c9870a4897cdf712caba1e3", "packages": [ { "name": "adhocore/jwt", @@ -3928,16 +3928,16 @@ }, { "name": "utopia-php/migration", - "version": "dev-0.6.x-fix-structore-validation", + "version": "0.6.13", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "ed735136dd10a88ab8601a9831e9cbcf3b6d6560" + "reference": "68d9b0a9477755afcda607e7e8109785cae17a13" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/ed735136dd10a88ab8601a9831e9cbcf3b6d6560", - "reference": "ed735136dd10a88ab8601a9831e9cbcf3b6d6560", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/68d9b0a9477755afcda607e7e8109785cae17a13", + "reference": "68d9b0a9477755afcda607e7e8109785cae17a13", "shasum": "" }, "require": { @@ -3978,9 +3978,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.x-fix-structore-validation" + "source": "https://github.com/utopia-php/migration/tree/0.6.13" }, - "time": "2024-11-26T11:04:12+00:00" + "time": "2024-11-26T13:57:53+00:00" }, { "name": "utopia-php/mongo", @@ -8555,18 +8555,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/migration", - "version": "dev-0.6.x-fix-structore-validation", - "alias": "0.6.13", - "alias_normalized": "0.6.13.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/migration": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From f0b6c729c925efc5148153ab2f3ad2d6b0d20a69 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 26 Nov 2024 14:54:27 +0100 Subject: [PATCH 18/19] feat: add more tags to sentry --- app/cli.php | 2 +- app/controllers/general.php | 5 ++++- app/http.php | 5 ++++- app/realtime.php | 2 +- app/worker.php | 4 ++-- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/app/cli.php b/app/cli.php index b0f053d3c6..fd49d2fed4 100644 --- a/app/cli.php +++ b/app/cli.php @@ -181,7 +181,7 @@ CLI::setResource('logError', function (Registry $register) { $log = new Log(); $log->setNamespace($namespace); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); diff --git a/app/controllers/general.php b/app/controllers/general.php index 4c4761bba9..9e589452ec 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -861,6 +861,8 @@ App::error() if (isset($user) && !$user->isEmpty()) { $log->setUser(new User($user->getId())); + } else { + $log->setUser(new User('guest-' . hash('sha256', $request->getIP()))); } try { @@ -871,7 +873,7 @@ App::error() } $log->setNamespace("http"); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); @@ -892,6 +894,7 @@ App::error() $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD"); $log->setAction($action); + $log->addTag('service', $action); $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); diff --git a/app/http.php b/app/http.php index 7387b3a43b..c4e48a7f69 100644 --- a/app/http.php +++ b/app/http.php @@ -370,10 +370,12 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool if (isset($user) && !$user->isEmpty()) { $log->setUser(new User($user->getId())); + } else { + $log->setUser(new User('guest-' . hash('sha256', $request->getIP()))); } $log->setNamespace("http"); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($th->getMessage()); @@ -393,6 +395,7 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD"); $log->setAction($action); + $log->addTag('service', $action); $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); diff --git a/app/realtime.php b/app/realtime.php index cfb9bf9a16..03eaa53bcc 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -184,7 +184,7 @@ $logError = function (Throwable $error, string $action) use ($register) { $log = new Log(); $log->setNamespace("realtime"); - $log->setServer(gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); diff --git a/app/worker.php b/app/worker.php index 280e170028..7e4eafeea2 100644 --- a/app/worker.php +++ b/app/worker.php @@ -301,7 +301,7 @@ Server::setResource('logError', function (Registry $register, Document $project) $log = new Log(); $log->setNamespace($namespace); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); @@ -394,7 +394,7 @@ $worker if ($logger) { $log->setNamespace("appwrite-worker"); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); From c062a8f1299d0980ae39130cbb8401aa62933072 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Wed, 27 Nov 2024 16:47:32 +0000 Subject: [PATCH 19/19] Bump console to version 5.2.27 --- app/views/install/compose.phtml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 361685d1ed..ad6d883c4c 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -167,7 +167,7 @@ $image = $this->getParam('image', ''); appwrite-console: <<: *x-logging container_name: appwrite-console - image: /console:5.2.25 + image: /console:5.2.27 restart: unless-stopped networks: - appwrite diff --git a/docker-compose.yml b/docker-compose.yml index ef8d898250..f9a79ee84d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -199,7 +199,7 @@ services: appwrite-console: <<: *x-logging container_name: appwrite-console - image: appwrite/console:5.2.25 + image: appwrite/console:5.2.27 restart: unless-stopped networks: - appwrite