diff --git a/CHANGES.md b/CHANGES.md index 9b6172eeab..62db3d525e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,117 @@ +# Version 1.6.1 + +## What's Changed + +### Notable changes + +* Remove JPEG fallback for webp by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8746 +* Add heic and avif support by @lohanidamodar in https://github.com/appwrite/appwrite/pull/7718 +* Add new runtimes by @Meldiron in https://github.com/appwrite/appwrite/pull/8771 +* Remove audits deletion by @shimonewman in https://github.com/appwrite/appwrite/pull/8766 +* Bump assistant by @loks0n in https://github.com/appwrite/appwrite/pull/8801 +* Change max queries values to 500 by @fogelito in https://github.com/appwrite/appwrite/pull/8802 +* Allow '.wav' as 'audio/x-wav' as well by @basert in https://github.com/appwrite/appwrite/pull/8846 +* Use 1 instead of 0.5 cpu for default function specification by @loks0n in https://github.com/appwrite/appwrite/pull/8848 +* Update function runtimes by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8781 +* Add a realtime heartbeat by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8943 + +### Fixes + +* Trigger functions event only if event is not paused by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8526 +* Update docker-compose to restart usage-dump by @feschaffa in https://github.com/appwrite/appwrite/pull/8642 +* Fix typo in scheduler base by @fogelito in https://github.com/appwrite/appwrite/pull/8691 +* Add domain and force HTTPS env vars to mail worker by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8722 +* Fix webp by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8732 +* Ignore junction tables by @fogelito in https://github.com/appwrite/appwrite/pull/8728 +* Fix logger throwing fatal error by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8724 +* Fix missing protocol for testing SMTP by @byawitz in https://github.com/appwrite/appwrite/pull/8749 +* Make create execution async loose by @loks0n in https://github.com/appwrite/appwrite/pull/8707 +* Fix invalid cursor value by @fogelito in https://github.com/appwrite/appwrite/pull/8109 +* Fix target deletes by @abnegate in https://github.com/appwrite/appwrite/pull/8833 +* Fix translation commas by @loks0n in https://github.com/appwrite/appwrite/pull/8892 +* Fix Migrations having source creds being overwritten and add Migration tests by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8897 +* Fix validator usage for updating string size by @abnegate in https://github.com/appwrite/appwrite/pull/8890 +* Fix create user event not triggering by @loks0n in https://github.com/appwrite/appwrite/pull/8718 +* Improve error handling and logging in the database worker by @fogelito in https://github.com/appwrite/appwrite/pull/8944 +* Remove inaccurate info about leaving the URL parameter empty by @ebenezerdon in https://github.com/appwrite/appwrite/pull/8963 +* Ensure indexes are updated when updating an attribute key by @fogelito in https://github.com/appwrite/appwrite/pull/8971 +* Remove duplicate dart-2.16 runtime template by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8972 +* Fix team invites with existing session by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/9006 +* Improve handling of HTTP requests by dispatching to safe workers by @Meldiron in https://github.com/appwrite/appwrite/pull/9016 +* Fix users create session secret by @stnguyen90 in https://github.com/appwrite/appwrite/pull/9019 +* Fix swoole task warning by @Meldiron in https://github.com/appwrite/appwrite/pull/9025 + +### Miscellaneous + +* Update Init copy by @adityaoberai in https://github.com/appwrite/appwrite/pull/8557 +* Fix security scan permissions and comment by @EVDOG4LIFE in https://github.com/appwrite/appwrite/pull/8525 +* Add Trivy security scans by @btme0011 in https://github.com/appwrite/appwrite/pull/6876 +* Update database stack by @abnegate in https://github.com/appwrite/appwrite/pull/8564 +* Bump database by @abnegate in https://github.com/appwrite/appwrite/pull/8573 +* Sync main with 1.5.x by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8589 +* Add AWS to one-click installs by @byawitz in https://github.com/appwrite/appwrite/pull/8593 +* Update Init copy in readme by @adityaoberai in https://github.com/appwrite/appwrite/pull/8618 +* Sync main into 1.6.x by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8685 +* Sync 1.6.x into main by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8686 +* Feat coroutines by @Meldiron in https://github.com/appwrite/appwrite/pull/7826 +* Sync main into 1.6.x by @Meldiron in https://github.com/appwrite/appwrite/pull/8719 +* Sentence casing endpoint API reference by @choir241 in https://github.com/appwrite/appwrite/pull/8617 +* DB storage metrics by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8404 +* Fix exception thrown when optional array attribute does not exist by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8391 +* Add projects channels to realtime by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8735 +* Base for console roles support by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8565 +* Remove DB disk storage calculation by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8745 +* Messaging adapter default values by @shimonewman in https://github.com/appwrite/appwrite/pull/8742 +* Add payload response type by @loks0n in https://github.com/appwrite/appwrite/pull/8720 +* Fix flaky functions tests by @loks0n in https://github.com/appwrite/appwrite/pull/8682 +* Migrations Backups by @fogelito in https://github.com/appwrite/appwrite/pull/8186 +* Add test for response and request filters by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8697 +* Bump version in SECURITY.md by @EVDOG4LIFE in https://github.com/appwrite/appwrite/pull/8755 +* Add originalId attribute to databases collection by @fogelito in https://github.com/appwrite/appwrite/pull/8764 +* Fix Walter References by @ItzNotABug in https://github.com/appwrite/appwrite/pull/8757 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8769 +* Move new attributes by @abnegate in https://github.com/appwrite/appwrite/pull/8777 +* Add ping endpoint by @loks0n in https://github.com/appwrite/appwrite/pull/8761 +* Fix GitHub action caching by @loks0n in https://github.com/appwrite/appwrite/pull/8772 +* Chore release ruby SDK by @abnegate in https://github.com/appwrite/appwrite/pull/8767 +* Call migration success on success by @abnegate in https://github.com/appwrite/appwrite/pull/8782 +* Update utopia-php/system to 0.9.0 by @basert in https://github.com/appwrite/appwrite/pull/8780 +* Move createDocument from api to worker by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8776 +* Add missing indexes by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8803 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8809 +* Fix typo in BLR region by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8756 +* Add tests for project variables by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8815 +* Replace 'Expires' with 'Cache-Control: private' header to avoid CDN caching by @basert in https://github.com/appwrite/appwrite/pull/8836 +* Allow blocking based on resource attributes by @basert in https://github.com/appwrite/appwrite/pull/8812 +* Check if resource is blocked inside functions worker by @basert in https://github.com/appwrite/appwrite/pull/8855 +* Fix missing allow attribute by @abnegate in https://github.com/appwrite/appwrite/pull/8889 +* Revert function execution order by @basert in https://github.com/appwrite/appwrite/pull/8857 +* Use resource type constants by @basert in https://github.com/appwrite/appwrite/pull/8895 +* Update Database lib by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8680 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8917 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8923 +* Update database for transaction counter fixes with retries by @abnegate in https://github.com/appwrite/appwrite/pull/8927 +* Validate string permissions by @fogelito in https://github.com/appwrite/appwrite/pull/8929 +* Add PubSub adapter support by @basert in https://github.com/appwrite/appwrite/pull/8905 +* List memberships as client by @loks0n in https://github.com/appwrite/appwrite/pull/8913 +* Fix XDebug Extension not being removed by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8891 +* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8946 +* Use utopia compression by @loks0n in https://github.com/appwrite/appwrite/pull/8938 +* Make compression minimum size configurable by @loks0n in https://github.com/appwrite/appwrite/pull/8947 +* Revert "Update database" by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8949 +* Fix setpaused by @loks0n in https://github.com/appwrite/appwrite/pull/8948 +* Use getDocument instead of find() for rules by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8951 +* Remove double fetch from migrations worker by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8956 +* Fix memberships privacy MFA by @loks0n in https://github.com/appwrite/appwrite/pull/8969 +* Add telemetry by @basert in https://github.com/appwrite/appwrite/pull/8960 +* Send migration errors individually by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8959 +* Add console sdk previews by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8990 +* Unset index length by @fogelito in https://github.com/appwrite/appwrite/pull/8978 +* Update base to 0.9.5 by @basert in https://github.com/appwrite/appwrite/pull/9005 +* Sync main into 1.6.x by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/9011 +* Improved shared tables V2 by @abnegate in https://github.com/appwrite/appwrite/pull/9013 +* Ensure backwards compatibility for 1.6.x by @christyjacob4 in https://github.com/appwrite/appwrite/pull/9018 + # Version 1.6.0 ## What's Changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b92361e51a..e89aa369cf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -613,6 +613,18 @@ If you need to clear the cache, you can do so by running the following command: docker compose exec redis redis-cli FLUSHALL ``` +## Using preview domains locally + +Appwrite Functions are automatically given a domain you can visit to execute the function. This domain has format `[SOMETHING].functions.localhost` unless you changed `_APP_DOMAIN_FUNCTIONS` environment variable. This default value works great when running Appwrite locally, but it can be impossible to use preview domains with Cloud woekspaces such as Gitpod or GitHub Codespaces. + +To use preview domains on Cloud workspaces, you can visit hostname provided by them, and supply function's preview domain as URL parameter: + +``` +https://8080-appwrite-appwrite-mjeb3ebilwv.ws-eu116.gitpod.io/ping?preview=672b3c7eab1ac523ccf5.functions.localhost +``` + +The path was set to `/ping` intentionally. Visiting `/` for preview domains might trigger Console background worker, and trigger redirect to Console without our preview URL param. Visiting different path ensures this doesnt happen. + ## Tutorials From time to time, our team will add tutorials that will help contributors find their way in the Appwrite source code. Below is a list of currently available tutorials: diff --git a/README-CN.md b/README-CN.md index a5eac1be03..92a9bf9806 100644 --- a/README-CN.md +++ b/README-CN.md @@ -67,7 +67,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` ### Windows @@ -79,7 +79,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` #### PowerShell @@ -89,7 +89,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` 运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index a9856a7310..ab57e65c4c 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` ### Windows @@ -87,7 +87,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` #### PowerShell @@ -97,7 +97,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.6.0 + appwrite/appwrite:1.6.1 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/cli.php b/app/cli.php index b0f053d3c6..fd49d2fed4 100644 --- a/app/cli.php +++ b/app/cli.php @@ -181,7 +181,7 @@ CLI::setResource('logError', function (Registry $register) { $log = new Log(); $log->setNamespace($namespace); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); diff --git a/app/controllers/general.php b/app/controllers/general.php index 1c1c790300..b0e350c765 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -48,11 +48,14 @@ Config::setParam('domainVerification', false); Config::setParam('cookieDomain', 'localhost'); Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) +function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml'); $host = $request->getHostname() ?? ''; + if (!empty($previewHostname)) { + $host = $previewHostname; + } // TODO: @christyjacob remove once we migrate the rules in 1.7.x if (version_compare(APP_VERSION_STABLE, '1.7.0', '<')) { @@ -359,26 +362,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo throw $th; } } finally { - $fileSize = 0; - $file = $request->getFiles('file'); - if (!empty($file)) { - $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; - } - - $queueForUsage - ->addMetric(METRIC_NETWORK_REQUESTS, 1) - ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) - ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) - ->addMetric(METRIC_EXECUTIONS, 1) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) - ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function - ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->setProject($project) - ->trigger() - ; - $queueForFunctions ->setType(Func::TYPE_ASYNC_WRITE) ->setExecution($execution) @@ -413,6 +396,26 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo ->setStatusCode($execution['responseStatusCode'] ?? 200) ->send($body); + $fileSize = 0; + $file = $request->getFiles('file'); + if (!empty($file)) { + $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + } + + $queueForUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) + ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) + ->addMetric(METRIC_EXECUTIONS, 1) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) + ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function + ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) + ->setProject($project) + ->trigger() + ; + return true; } elseif ($type === 'api') { $utopia->getRoute()?->label('error', ''); @@ -469,16 +472,17 @@ App::init() ->inject('queueForCertificates') ->inject('queueForFunctions') ->inject('isResourceBlocked') + ->inject('previewHostname') ->inject('devKey') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, Document $devKey) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHostname, Document $devKey) { /* * Appwrite Router */ $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain) { - if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked)) { + if ($host !== $mainDomain || !empty($previewHostname)) { + if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { return; } } @@ -702,16 +706,17 @@ App::options() ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') + ->inject('previewHostname') ->inject('devKey') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, Document $devKey) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname, Document $devKey) { /* * Appwrite Router */ $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain - if ($host !== $mainDomain) { - if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked)) { + if ($host !== $mainDomain || !empty($previewHostname)) { + if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { return; } } @@ -867,6 +872,8 @@ App::error() if (isset($user) && !$user->isEmpty()) { $log->setUser(new User($user->getId())); + } else { + $log->setUser(new User('guest-' . hash('sha256', $request->getIP()))); } try { @@ -877,7 +884,7 @@ App::error() } $log->setNamespace("http"); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); @@ -898,6 +905,7 @@ App::error() $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD"); $log->setAction($action); + $log->addTag('service', $action); $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); @@ -999,15 +1007,16 @@ App::get('/robots.txt') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) { + ->inject('previewHostname') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); - if ($host === $mainDomain || $host === 'localhost') { + if (($host === $mainDomain || $host === 'localhost') && empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked); + router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); } }); @@ -1026,15 +1035,16 @@ App::get('/humans.txt') ->inject('queueForFunctions') ->inject('geodb') ->inject('isResourceBlocked') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) { + ->inject('previewHostname') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); - if ($host === $mainDomain || $host === 'localhost') { + if (($host === $mainDomain || $host === 'localhost') && empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked); + router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); } }); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 6b2abd228f..bd18f42993 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -185,13 +185,14 @@ App::init() ->inject('utopia') ->inject('request') ->inject('dbForConsole') + ->inject('dbForProject') ->inject('project') ->inject('user') ->inject('session') ->inject('servers') ->inject('mode') ->inject('team') - ->action(function (App $utopia, Request $request, Database $dbForConsole, Document $project, Document $user, ?Document $session, array $servers, string $mode, Document $team) { + ->action(function (App $utopia, Request $request, Database $dbForConsole, Database $dbForProject, Document $project, Document $user, ?Document $session, array $servers, string $mode, Document $team) { $route = $utopia->getRoute(); if ($project->isEmpty()) { @@ -335,6 +336,33 @@ App::init() Authorization::setRole($authRole); } + /** + * Update project last activity + */ + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); + } + } + + /** + * Update user last activity + */ + if (!empty($user->getId())) { + $accessedAt = $user->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { + $user->setAttribute('accessedAt', DateTime::now()); + + if (APP_MODE_ADMIN !== $mode) { + $dbForProject->updateDocument('users', $user->getId(), $user); + } else { + $dbForConsole->updateDocument('users', $user->getId(), $user); + } + } + } + /** Do not allow access to disabled services */ $service = $route->getLabel('sdk.namespace', ''); if (!empty($service)) { @@ -643,9 +671,7 @@ App::shutdown() ->inject('queueForWebhooks') ->inject('queueForRealtime') ->inject('dbForProject') - ->inject('mode') - ->inject('dbForConsole') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, string $mode, Database $dbForConsole) use ($parseLabel) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -765,8 +791,6 @@ App::shutdown() } } - - if ($project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; @@ -785,33 +809,6 @@ App::shutdown() ->setProject($project) ->trigger(); } - - /** - * Update project last activity - */ - if (!$project->isEmpty() && $project->getId() !== 'console') { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); - } - } - - /** - * Update user last activity - */ - if (!$user->isEmpty()) { - $accessedAt = $user->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { - $user->setAttribute('accessedAt', DateTime::now()); - - if (APP_MODE_ADMIN !== $mode) { - $dbForProject->updateDocument('users', $user->getId(), $user); - } else { - $dbForConsole->updateDocument('users', $user->getId(), $user); - } - } - } }); App::init() diff --git a/app/http.php b/app/http.php index c80fccee54..c4e48a7f69 100644 --- a/app/http.php +++ b/app/http.php @@ -54,6 +54,7 @@ $http 'http_compression' => false, 'package_max_length' => $payloadSize, 'buffer_output_size' => $payloadSize, + 'task_worker_num' => 1, // required for the task to fetch domains background ]); $http->on(Constant::EVENT_WORKER_START, function ($server, $workerId) { @@ -369,10 +370,12 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool if (isset($user) && !$user->isEmpty()) { $log->setUser(new User($user->getId())); + } else { + $log->setUser(new User('guest-' . hash('sha256', $request->getIP()))); } $log->setNamespace("http"); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($th->getMessage()); @@ -392,6 +395,7 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD"); $log->setAction($action); + $log->addTag('service', $action); $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); diff --git a/app/init.php b/app/init.php index 22e69ebd18..9dc5e28e80 100644 --- a/app/init.php +++ b/app/init.php @@ -124,7 +124,7 @@ const APP_USER_ACCESS = 24 * 60 * 60; // 24 hours const APP_PROJECT_ACCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 4318; -const APP_VERSION_STABLE = '1.6.0'; +const APP_VERSION_STABLE = '1.6.1'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; @@ -1864,3 +1864,14 @@ App::setResource( 'isResourceBlocked', fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false ); + +App::setResource('previewHostname', function (Request $request) { + if (App::isDevelopment()) { + $host = $request->getQuery('appwrite-hostname') ?? ''; + if (!empty($host)) { + return $host; + } + } + + return ''; +}, ['request']); diff --git a/app/realtime.php b/app/realtime.php index cfb9bf9a16..03eaa53bcc 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -184,7 +184,7 @@ $logError = function (Throwable $error, string $action) use ($register) { $log = new Log(); $log->setNamespace("realtime"); - $log->setServer(gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 8d7fecb479..ad6d883c4c 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -167,7 +167,7 @@ $image = $this->getParam('image', ''); appwrite-console: <<: *x-logging container_name: appwrite-console - image: /console:5.0.12 + image: /console:5.2.27 restart: unless-stopped networks: - appwrite diff --git a/app/worker.php b/app/worker.php index 7899c1d5bc..7e4eafeea2 100644 --- a/app/worker.php +++ b/app/worker.php @@ -2,6 +2,7 @@ require_once __DIR__ . '/init.php'; +use Appwrite\Certificates\LetsEncrypt; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Certificate; @@ -16,7 +17,6 @@ use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; -use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -283,6 +283,15 @@ Server::setResource( fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false ); +Server::setResource('certificates', function () { + $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); + if (empty($email)) { + throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.'); + } + + return new LetsEncrypt($email); +}); + Server::setResource('logError', function (Registry $register, Document $project) { return function (Throwable $error, string $namespace, string $action, ?array $extras) use ($register, $project) { $logger = $register->get('logger'); @@ -292,7 +301,7 @@ Server::setResource('logError', function (Registry $register, Document $project) $log = new Log(); $log->setNamespace($namespace); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); @@ -385,7 +394,7 @@ $worker if ($logger) { $log->setNamespace("appwrite-worker"); - $log->setServer(\gethostname()); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); diff --git a/composer.lock b/composer.lock index 87d8b47583..bdb41c2478 100644 --- a/composer.lock +++ b/composer.lock @@ -157,16 +157,16 @@ }, { "name": "appwrite/php-runtimes", - "version": "0.16.4", + "version": "0.16.5", "source": { "type": "git", "url": "https://github.com/appwrite/runtimes.git", - "reference": "7e4741337b9373f77210396e68eca539018cabd1" + "reference": "1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/runtimes/zipball/7e4741337b9373f77210396e68eca539018cabd1", - "reference": "7e4741337b9373f77210396e68eca539018cabd1", + "url": "https://api.github.com/repos/appwrite/runtimes/zipball/1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9", + "reference": "1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9", "shasum": "" }, "require": { @@ -206,9 +206,9 @@ ], "support": { "issues": "https://github.com/appwrite/runtimes/issues", - "source": "https://github.com/appwrite/runtimes/tree/0.16.4" + "source": "https://github.com/appwrite/runtimes/tree/0.16.5" }, - "time": "2024-10-26T10:39:59+00:00" + "time": "2024-11-25T15:17:06+00:00" }, { "name": "beberlei/assert", @@ -3928,16 +3928,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.12", + "version": "0.6.13", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "9a8c905af4cece5c5ec9542a5b534befce067260" + "reference": "68d9b0a9477755afcda607e7e8109785cae17a13" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/9a8c905af4cece5c5ec9542a5b534befce067260", - "reference": "9a8c905af4cece5c5ec9542a5b534befce067260", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/68d9b0a9477755afcda607e7e8109785cae17a13", + "reference": "68d9b0a9477755afcda607e7e8109785cae17a13", "shasum": "" }, "require": { @@ -3978,9 +3978,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.12" + "source": "https://github.com/utopia-php/migration/tree/0.6.13" }, - "time": "2024-11-12T00:31:53+00:00" + "time": "2024-11-26T13:57:53+00:00" }, { "name": "utopia-php/mongo", @@ -8557,7 +8557,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index bae2cc7811..f9a79ee84d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -199,7 +199,7 @@ services: appwrite-console: <<: *x-logging container_name: appwrite-console - image: appwrite/console:5.0.12 + image: appwrite/console:5.2.27 restart: unless-stopped networks: - appwrite @@ -385,6 +385,7 @@ services: - _APP_EXECUTOR_HOST - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 + - _APP_EMAIL_CERTIFICATES appwrite-worker-databases: entrypoint: worker-databases diff --git a/src/Appwrite/Certificates/Adapter.php b/src/Appwrite/Certificates/Adapter.php new file mode 100644 index 0000000000..711e4c09b9 --- /dev/null +++ b/src/Appwrite/Certificates/Adapter.php @@ -0,0 +1,14 @@ +email = $email; + } + + + public function issueCertificate(string $certName, string $domain): ?string + { + $stdout = ''; + $stderr = ''; + + $staging = (App::isProduction()) ? '' : ' --dry-run'; + $exit = Console::execute( + "certbot certonly -v --webroot --noninteractive --agree-tos{$staging}" + . " --email " . $this->email + . " --cert-name " . $certName + . " -w " . APP_STORAGE_CERTIFICATES + . " -d {$domain}", + '', + $stdout, + $stderr + ); + + // Unexpected error, usually 5XX, API limits, ... + if ($exit !== 0) { + throw new Exception('Failed to issue a certificate with message: ' . $stderr); + } + + // Prepare folder in storage for domain + $path = APP_STORAGE_CERTIFICATES . '/' . $domain; + if (!\is_readable($path)) { + if (!\mkdir($path, 0755, true)) { + throw new Exception('Failed to create path for certificate.'); + } + } + + // Move generated files + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { + throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { + throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { + throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + if (!@\rename('/etc/letsencrypt/live/' . $certName . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { + throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout); + } + + $config = \implode(PHP_EOL, [ + "tls:", + " certificates:", + " - certFile: /storage/certificates/{$domain}/fullchain.pem", + " keyFile: /storage/certificates/{$domain}/privkey.pem" + ]); + + // Save configuration into Traefik using our new cert files + if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { + throw new Exception('Failed to save Traefik configuration.'); + } + + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? null; + $dt = (new \DateTime())->setTimestamp($validTo); + return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); + } + + public function isRenewRequired(string $domain, Log $log): bool + { + $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; + if (\file_exists($certPath)) { + $certData = openssl_x509_parse(file_get_contents($certPath)); + $validTo = $certData['validTo_time_t'] ?? 0; + + if (empty($validTo)) { + $log->addTag('certificateDomain', $domain); + throw new Exception('Unable to read certificate file (cert.pem).'); + } + + // LetsEncrypt allows renewal 30 days before expiry + $expiryInAdvance = (60 * 60 * 24 * 30); + if ($validTo - $expiryInAdvance > \time()) { + $log->addTag('certificateDomain', $domain); + $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData)); + return false; + } + } + + return true; + } + + public function deleteCertificate(string $domain): void + { + $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; + $checkTraversal = realpath($directory) === $directory; + + if ($checkTraversal && is_dir($directory)) { + // Delete files, so Traefik is aware of change + array_map('unlink', glob($directory . '/*.*')); + rmdir($directory); + Console::info("Deleted certificate files for {$domain}"); + } else { + Console::info("No certificate files found for {$domain}"); + } + } +} diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index cee1b2d263..19f69b1a4f 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -91,6 +91,7 @@ abstract class Migration '1.5.10' => 'V20', '1.5.11' => 'V20', '1.6.0' => 'V21', + '1.6.1' => 'V21', ]; /** diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index afb38f35fc..40e72dc683 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -149,6 +149,7 @@ class Maintenance extends Action $certificates = $dbForConsole->find('certificates', [ Query::lessThan('attempts', 5), // Maximum 5 attempts + Query::isNotNull('renewDate'), Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew) Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains) ]); diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 6f1a710bc0..db7c73dd0c 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Workers; +use Appwrite\Certificates\Adapter as CertificatesAdapter; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; @@ -11,7 +12,6 @@ use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; use Exception; use Throwable; -use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -48,7 +48,11 @@ class Certificates extends Action ->inject('queueForEvents') ->inject('queueForFunctions') ->inject('log') - ->callback(fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log)); + ->inject('certificates') + ->callback( + fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates) => + $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates) + ); } /** @@ -58,11 +62,12 @@ class Certificates extends Action * @param Event $queueForEvents * @param Func $queueForFunctions * @param Log $log + * @param CertificatesAdapter $certificates * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates): void { $payload = $message->getPayload() ?? []; @@ -76,7 +81,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck); + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates, $skipRenewCheck); } /** @@ -85,24 +90,24 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions + * @param CertificatesAdapter $certificates * @param bool $skipRenewCheck * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain * 2. Get main domain * 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain) - * 4. Validate security email. Cannot be empty, required by LetsEncrypt - * 5. Validate renew date with certificate file, unless requested to skip by parameter - * 6. Issue a certificate using certbot CLI - * 7. Update 'log' attribute on certificate document with Certbot message - * 8. Create storage folder for certificate, if not ready already - * 9. Move certificates from Certbot location to our Storage - * 10. Create/Update our Storage with new Traefik config with new certificate paths + * 4. Validate renew date with certificate file, unless requested to skip by parameter + * 5. Issue a certificate using certbot CLI + * 6. Update 'log' attribute on certificate document with Certbot message + * 7. Create storage folder for certificate, if not ready already + * 8. Move certificates from Certbot location to our Storage + * 9. Create/Update our Storage with new Traefik config with new certificate paths * 11. Read certificate file and update 'renewDate' on certificate document * 12. Update 'issueDate' and 'attempts' on certificate * @@ -119,7 +124,7 @@ class Certificates extends Action * 2. Save document to database * 3. Update all domains documents with current certificate ID * - * Note: Renewals are checked and scheduled from maintenence worker + * Note: Renewals are checked and scheduled from maintenance worker */ // Get current certificate @@ -134,40 +139,27 @@ class Certificates extends Action $success = false; try { - // Email for alerts is required by LetsEncrypt - $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); - if (empty($email)) { - throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue an SSL certificate.'); - } - // Validate domain and DNS records. Skip if job is forced if (!$skipRenewCheck) { $mainDomain = $this->getMainDomain(); $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain; $this->validateDomain($domain, $isMainDomain, $log); + + // If certificate exists already, double-check expiry date. Skip if job is forced + if (!$certificates->isRenewRequired($domain->get(), $log)) { + throw new Exception('Renew isn\'t required.'); + } } - // If certificate exists already, double-check expiry date. Skip if job is forced - if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) { - throw new Exception('Renew isn\'t required.'); - } - - // Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate - $folder = ID::unique(); - - // Generate certificate files using Let's Encrypt - $letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email); + // Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates. + $certName = ID::unique(); + $renewDate = $certificates->issueCertificate($certName, $domain->get()); // Command succeeded, store all data into document - $logs = 'Certificate successfully generated.'; - $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB - - - // Give certificates to Traefik - $this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData); + $certificate->setAttribute('logs', 'Certificate successfully generated.'); // Update certificate info stored in database - $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get())); + $certificate->setAttribute('renewDate', $renewDate); $certificate->setAttribute('attempts', 0); $certificate->setAttribute('issueDate', DateTime::now()); $success = true; @@ -181,7 +173,7 @@ class Certificates extends Action $attempts = $certificate->getAttribute('attempts', 0) + 1; $certificate->setAttribute('attempts', $attempts); - // Store cuttent time as renew date to ensure another attempt in next maintenance cycle + // Store current time as renew date to ensure another attempt in next maintenance cycle. $certificate->setAttribute('renewDate', DateTime::now()); // Send email to security email @@ -245,8 +237,8 @@ class Certificates extends Action } /** - * Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check: - * - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt) + * Internal domain validation functionality to prevent unnecessary attempts. We check: + * - Domain needs to be public and valid (prevents NFT domains that are not supported) * - Domain must have proper DNS record * * @param Domain $domain Domain which we validate @@ -292,136 +284,6 @@ class Certificates extends Action } } - /** - * Reads expiry date of certificate from file and decides if renewal is required or not. - * - * @param string $domain Domain for which we check certificate file - * @return bool True, if certificate needs to be renewed - * @throws Exception - */ - private function isRenewRequired(string $domain, Log $log): bool - { - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; - if (\file_exists($certPath)) { - $validTo = null; - - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? 0; - - if (empty($validTo)) { - $log->addTag('certificateDomain', $domain); - throw new Exception('Unable to read certificate file (cert.pem).'); - } - - // LetsEncrypt allows renewal 30 days before expiry - $expiryInAdvance = (60 * 60 * 24 * 30); - if ($validTo - $expiryInAdvance > \time()) { - $log->addTag('certificateDomain', $domain); - $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData)); - return false; - } - } - - return true; - } - - /** - * LetsEncrypt communication to issue certificate (using certbot CLI) - * - * @param string $folder Folder into which certificates should be generated - * @param string $domain Domain to generate certificate for - * @return array Named array with keys 'stdout' and 'stderr', both string - * @throws Exception - */ - private function issueCertificate(string $folder, string $domain, string $email): array - { - $stdout = ''; - $stderr = ''; - - $staging = (App::isProduction()) ? '' : ' --dry-run'; - $exit = Console::execute("certbot certonly -v --webroot --noninteractive --agree-tos{$staging}" - . " --email " . $email - . " --cert-name " . $folder - . " -w " . APP_STORAGE_CERTIFICATES - . " -d {$domain}", '', $stdout, $stderr); - - // Unexpected error, usually 5XX, API limits, ... - if ($exit !== 0) { - throw new Exception('Failed to issue a certificate with message: ' . $stderr); - } - - return [ - 'stdout' => $stdout, - 'stderr' => $stderr - ]; - } - - /** - * Read new renew date from certificate file generated by Let's Encrypt - * - * @param string $domain Domain which certificate was generated for - * @return string - * @throws \Utopia\Database\Exception - */ - private function getRenewDate(string $domain): string - { - $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; - $certData = openssl_x509_parse(file_get_contents($certPath)); - $validTo = $certData['validTo_time_t'] ?? null; - $dt = (new \DateTime())->setTimestamp($validTo); - return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days - } - - /** - * Method to take files from Let's Encrypt, and put it into Traefik. - * - * @param string $domain Domain which certificate was generated for - * @param string $folder Folder in which certificates were generated - * @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error - * @return void - * @throws Exception - */ - private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void - { - - // Prepare folder in storage for domain - $path = APP_STORAGE_CERTIFICATES . '/' . $domain; - if (!\is_readable($path)) { - if (!\mkdir($path, 0755, true)) { - throw new Exception('Failed to create path for certificate.'); - } - } - - // Move generated files - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) { - throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) { - throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) { - throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) { - throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']); - } - - $config = \implode(PHP_EOL, [ - "tls:", - " certificates:", - " - certFile: /storage/certificates/{$domain}/fullchain.pem", - " keyFile: /storage/certificates/{$domain}/privkey.pem" - ]); - - // Save configuration into Traefik using our new cert files - if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) { - throw new Exception('Failed to save Traefik configuration.'); - } - } - /** * Method to make sure information about error is delivered to admnistrator. * @@ -500,6 +362,10 @@ class Certificates extends Action $project = $dbForConsole->getDocument('projects', $projectId); + if ($project->isEmpty()) { + return; + } + /** Trigger Webhook */ $ruleModel = new Rule(); $queueForEvents diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 6c11116a4c..e6002bf75f 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Auth\Auth; +use Appwrite\Certificates\Adapter as CertificatesAdapter; use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; @@ -50,18 +51,22 @@ class Deletes extends Action ->inject('deviceForFunctions') ->inject('deviceForBuilds') ->inject('deviceForCache') + ->inject('certificates') ->inject('abuseRetention') ->inject('executionRetention') ->inject('auditRetention') ->inject('log') - ->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log)); + ->callback( + fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) + ); } /** * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -84,10 +89,10 @@ class Deletes extends Action case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { case DELETE_TYPE_PROJECTS: - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); + $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project); break; case DELETE_TYPE_DEPLOYMENTS: $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); @@ -102,7 +107,7 @@ class Deletes extends Action $this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project); break; case DELETE_TYPE_RULES: - $this->deleteRule($dbForConsole, $document); + $this->deleteRule($dbForConsole, $document, $certificates); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -110,7 +115,7 @@ class Deletes extends Action } break; case DELETE_TYPE_TEAM_PROJECTS: - $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $document); + $this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $certificates, $document); break; case DELETE_TYPE_EXECUTIONS: $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); @@ -264,7 +269,7 @@ class Deletes extends Action MESSAGE_TYPE_EMAIL => 'emailTotal', MESSAGE_TYPE_SMS => 'smsTotal', MESSAGE_TYPE_PUSH => 'pushTotal', - default => throw new Exception('Invalid target provider type'), + default => throw new Exception('Invalid target CertificatesAdapter type'), }; $dbForProject->decreaseDocumentAttribute( 'topics', @@ -442,7 +447,7 @@ class Deletes extends Action * @throws Structure * @throws Exception */ - private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Document $document): void + private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void { $projects = $dbForConsole->find('projects', [ @@ -455,7 +460,7 @@ class Deletes extends Action $deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); $deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $project); $dbForConsole->deleteDocument('projects', $project->getId()); } } @@ -473,7 +478,7 @@ class Deletes extends Action * @throws Authorization * @throws DatabaseException */ - private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void + private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Document $document): void { $projectInternalId = $document->getInternalId(); $projectId = $document->getId(); @@ -537,8 +542,8 @@ class Deletes extends Action // Delete project and function rules $this->deleteByGroup('rules', [ Query::equal('projectInternalId', [$projectInternalId]) - ], $dbForConsole, function (Document $document) use ($dbForConsole) { - $this->deleteRule($dbForConsole, $document); + ], $dbForConsole, function (Document $document) use ($dbForConsole, $certificates) { + $this->deleteRule($dbForConsole, $document, $certificates); }); // Delete Keys @@ -746,7 +751,7 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void + private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -761,8 +766,8 @@ class Deletes extends Action Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('projectInternalId', [$project->getInternalId()]) - ], $dbForConsole, function (Document $document) use ($project, $dbForConsole) { - $this->deleteRule($dbForConsole, $document); + ], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $certificates) { + $this->deleteRule($dbForConsole, $document, $certificates); }); /** @@ -1048,21 +1053,10 @@ class Deletes extends Action * @param Document $document rule document * @return void */ - private function deleteRule(Database $dbForConsole, Document $document): void + private function deleteRule(Database $dbForConsole, Document $document, CertificatesAdapter $certificates): void { - $domain = $document->getAttribute('domain'); - $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; - $checkTraversal = realpath($directory) === $directory; - - if ($checkTraversal && is_dir($directory)) { - // Delete files, so Traefik is aware of change - array_map('unlink', glob($directory . '/*.*')); - rmdir($directory); - Console::info("Deleted certificate files for {$domain}"); - } else { - Console::info("No certificate files found for {$domain}"); - } + $certificates->deleteCertificate($domain); // Delete certificate document, so Appwrite is aware of change if (isset($document['certificateId'])) {