Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/1.6.x' into feat-development-keys

Browse Source
This commit is contained in:
Damodar Lohani
2024-11-28 01:37:25 +00:00
parent 27905c2a90 91be70babf
commit fe7851b97f
20 changed files with 451 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGES.md
+114
View File
@@ -1,3 +1,117 @@
# Version 1.6.1
## What's Changed
### Notable changes
* Remove JPEG fallback for webp by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8746
* Add heic and avif support by @lohanidamodar in https://github.com/appwrite/appwrite/pull/7718
* Add new runtimes by @Meldiron in https://github.com/appwrite/appwrite/pull/8771
* Remove audits deletion by @shimonewman in https://github.com/appwrite/appwrite/pull/8766
* Bump assistant by @loks0n in https://github.com/appwrite/appwrite/pull/8801
* Change max queries values to 500 by @fogelito in https://github.com/appwrite/appwrite/pull/8802
* Allow '.wav' as 'audio/x-wav' as well by @basert in https://github.com/appwrite/appwrite/pull/8846
* Use 1 instead of 0.5 cpu for default function specification by @loks0n in https://github.com/appwrite/appwrite/pull/8848
* Update function runtimes by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8781
* Add a realtime heartbeat by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8943
### Fixes
* Trigger functions event only if event is not paused by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8526
* Update docker-compose to restart usage-dump by @feschaffa in https://github.com/appwrite/appwrite/pull/8642
* Fix typo in scheduler base by @fogelito in https://github.com/appwrite/appwrite/pull/8691
* Add domain and force HTTPS env vars to mail worker by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8722
* Fix webp by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8732
* Ignore junction tables by @fogelito in https://github.com/appwrite/appwrite/pull/8728
* Fix logger throwing fatal error by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8724
* Fix missing protocol for testing SMTP by @byawitz in https://github.com/appwrite/appwrite/pull/8749
* Make create execution async loose by @loks0n in https://github.com/appwrite/appwrite/pull/8707
* Fix invalid cursor value by @fogelito in https://github.com/appwrite/appwrite/pull/8109
* Fix target deletes by @abnegate in https://github.com/appwrite/appwrite/pull/8833
* Fix translation commas by @loks0n in https://github.com/appwrite/appwrite/pull/8892
* Fix Migrations having source creds being overwritten and add Migration tests by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8897
* Fix validator usage for updating string size by @abnegate in https://github.com/appwrite/appwrite/pull/8890
* Fix create user event not triggering by @loks0n in https://github.com/appwrite/appwrite/pull/8718
* Improve error handling and logging in the database worker by @fogelito in https://github.com/appwrite/appwrite/pull/8944
* Remove inaccurate info about leaving the URL parameter empty by @ebenezerdon in https://github.com/appwrite/appwrite/pull/8963
* Ensure indexes are updated when updating an attribute key by @fogelito in https://github.com/appwrite/appwrite/pull/8971
* Remove duplicate dart-2.16 runtime template by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8972
* Fix team invites with existing session by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/9006
* Improve handling of HTTP requests by dispatching to safe workers by @Meldiron in https://github.com/appwrite/appwrite/pull/9016
* Fix users create session secret by @stnguyen90 in https://github.com/appwrite/appwrite/pull/9019
* Fix swoole task warning by @Meldiron in https://github.com/appwrite/appwrite/pull/9025
### Miscellaneous
* Update Init copy by @adityaoberai in https://github.com/appwrite/appwrite/pull/8557
* Fix security scan permissions and comment by @EVDOG4LIFE in https://github.com/appwrite/appwrite/pull/8525
* Add Trivy security scans by @btme0011 in https://github.com/appwrite/appwrite/pull/6876
* Update database stack by @abnegate in https://github.com/appwrite/appwrite/pull/8564
* Bump database by @abnegate in https://github.com/appwrite/appwrite/pull/8573
* Sync main with 1.5.x by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8589
* Add AWS to one-click installs by @byawitz in https://github.com/appwrite/appwrite/pull/8593
* Update Init copy in readme by @adityaoberai in https://github.com/appwrite/appwrite/pull/8618
* Sync main into 1.6.x by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8685
* Sync 1.6.x into main by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8686
* Feat coroutines by @Meldiron in https://github.com/appwrite/appwrite/pull/7826
* Sync main into 1.6.x by @Meldiron in https://github.com/appwrite/appwrite/pull/8719
* Sentence casing endpoint API reference by @choir241 in https://github.com/appwrite/appwrite/pull/8617
* DB storage metrics by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8404
* Fix exception thrown when optional array attribute does not exist by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8391
* Add projects channels to realtime by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8735
* Base for console roles support by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8565
* Remove DB disk storage calculation by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8745
* Messaging adapter default values by @shimonewman in https://github.com/appwrite/appwrite/pull/8742
* Add payload response type by @loks0n in https://github.com/appwrite/appwrite/pull/8720
* Fix flaky functions tests by @loks0n in https://github.com/appwrite/appwrite/pull/8682
* Migrations Backups by @fogelito in https://github.com/appwrite/appwrite/pull/8186
* Add test for response and request filters by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8697
* Bump version in SECURITY.md by @EVDOG4LIFE in https://github.com/appwrite/appwrite/pull/8755
* Add originalId attribute to databases collection by @fogelito in https://github.com/appwrite/appwrite/pull/8764
* Fix Walter References by @ItzNotABug in https://github.com/appwrite/appwrite/pull/8757
* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8769
* Move new attributes by @abnegate in https://github.com/appwrite/appwrite/pull/8777
* Add ping endpoint by @loks0n in https://github.com/appwrite/appwrite/pull/8761
* Fix GitHub action caching by @loks0n in https://github.com/appwrite/appwrite/pull/8772
* Chore release ruby SDK by @abnegate in https://github.com/appwrite/appwrite/pull/8767
* Call migration success on success by @abnegate in https://github.com/appwrite/appwrite/pull/8782
* Update utopia-php/system to 0.9.0 by @basert in https://github.com/appwrite/appwrite/pull/8780
* Move createDocument from api to worker by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8776
* Add missing indexes by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8803
* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8809
* Fix typo in BLR region by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8756
* Add tests for project variables by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8815
* Replace 'Expires' with 'Cache-Control: private' header to avoid CDN caching by @basert in https://github.com/appwrite/appwrite/pull/8836
* Allow blocking based on resource attributes by @basert in https://github.com/appwrite/appwrite/pull/8812
* Check if resource is blocked inside functions worker by @basert in https://github.com/appwrite/appwrite/pull/8855
* Fix missing allow attribute by @abnegate in https://github.com/appwrite/appwrite/pull/8889
* Revert function execution order by @basert in https://github.com/appwrite/appwrite/pull/8857
* Use resource type constants by @basert in https://github.com/appwrite/appwrite/pull/8895
* Update Database lib by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8680
* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8917
* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8923
* Update database for transaction counter fixes with retries by @abnegate in https://github.com/appwrite/appwrite/pull/8927
* Validate string permissions by @fogelito in https://github.com/appwrite/appwrite/pull/8929
* Add PubSub adapter support by @basert in https://github.com/appwrite/appwrite/pull/8905
* List memberships as client by @loks0n in https://github.com/appwrite/appwrite/pull/8913
* Fix XDebug Extension not being removed by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8891
* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8946
* Use utopia compression by @loks0n in https://github.com/appwrite/appwrite/pull/8938
* Make compression minimum size configurable by @loks0n in https://github.com/appwrite/appwrite/pull/8947
* Revert "Update database" by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8949
* Fix setpaused by @loks0n in https://github.com/appwrite/appwrite/pull/8948
* Use getDocument instead of find() for rules by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8951
* Remove double fetch from migrations worker by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8956
* Fix memberships privacy MFA by @loks0n in https://github.com/appwrite/appwrite/pull/8969
* Add telemetry by @basert in https://github.com/appwrite/appwrite/pull/8960
* Send migration errors individually by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8959
* Add console sdk previews by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8990
* Unset index length by @fogelito in https://github.com/appwrite/appwrite/pull/8978
* Update base to 0.9.5 by @basert in https://github.com/appwrite/appwrite/pull/9005
* Sync main into 1.6.x by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/9011
* Improved shared tables V2 by @abnegate in https://github.com/appwrite/appwrite/pull/9013
* Ensure backwards compatibility for 1.6.x by @christyjacob4 in https://github.com/appwrite/appwrite/pull/9018
# Version 1.6.0
## What's Changed
CONTRIBUTING.md
+12
View File
@@ -613,6 +613,18 @@ If you need to clear the cache, you can do so by running the following command:
docker compose exec redis redis-cli FLUSHALL
```
## Using preview domains locally
Appwrite Functions are automatically given a domain you can visit to execute the function. This domain has format `[SOMETHING].functions.localhost` unless you changed `_APP_DOMAIN_FUNCTIONS` environment variable. This default value works great when running Appwrite locally, but it can be impossible to use preview domains with Cloud woekspaces such as Gitpod or GitHub Codespaces.
To use preview domains on Cloud workspaces, you can visit hostname provided by them, and supply function's preview domain as URL parameter:
```
https://8080-appwrite-appwrite-mjeb3ebilwv.ws-eu116.gitpod.io/ping?preview=672b3c7eab1ac523ccf5.functions.localhost
```
The path was set to `/ping` intentionally. Visiting `/` for preview domains might trigger Console background worker, and trigger redirect to Console without our preview URL param. Visiting different path ensures this doesnt happen.
## Tutorials
From time to time, our team will add tutorials that will help contributors find their way in the Appwrite source code. Below is a list of currently available tutorials:
README-CN.md
+3 -3
View File
@@ -67,7 +67,7 @@ docker run -it --rm \
--volume /var/run/docker.sock:/var/run/docker.sock \
--volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
--entrypoint="install" \
appwrite/appwrite:1.6.0
appwrite/appwrite:1.6.1
```
### Windows
@@ -79,7 +79,7 @@ docker run -it --rm ^
--volume //var/run/docker.sock:/var/run/docker.sock ^
--volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
--entrypoint="install" ^
appwrite/appwrite:1.6.0
appwrite/appwrite:1.6.1
```
#### PowerShell
@@ -89,7 +89,7 @@ docker run -it --rm `
--volume /var/run/docker.sock:/var/run/docker.sock `
--volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
--entrypoint="install" `
appwrite/appwrite:1.6.0
appwrite/appwrite:1.6.1
```
运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。
README.md
+3 -3
View File
@@ -75,7 +75,7 @@ docker run -it --rm \
--volume /var/run/docker.sock:/var/run/docker.sock \
--volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
--entrypoint="install" \
appwrite/appwrite:1.6.0
appwrite/appwrite:1.6.1
```
### Windows
@@ -87,7 +87,7 @@ docker run -it --rm ^
--volume //var/run/docker.sock:/var/run/docker.sock ^
--volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
--entrypoint="install" ^
appwrite/appwrite:1.6.0
appwrite/appwrite:1.6.1
```
#### PowerShell
@@ -97,7 +97,7 @@ docker run -it --rm `
--volume /var/run/docker.sock:/var/run/docker.sock `
--volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
--entrypoint="install" `
appwrite/appwrite:1.6.0
appwrite/appwrite:1.6.1
```
Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation.
app/cli.php
+1 -1
View File
@@ -181,7 +181,7 @@ CLI::setResource('logError', function (Registry $register) {
$log = new Log();
$log->setNamespace($namespace);
$log->setServer(\gethostname());
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
app/controllers/general.php
+44 -34
View File
@@ -48,11 +48,14 @@ Config::setParam('domainVerification', false);
Config::setParam('cookieDomain', 'localhost');
Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE);
function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked)
function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname)
{
$utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml');
$host = $request->getHostname() ?? '';
if (!empty($previewHostname)) {
$host = $previewHostname;
}
// TODO: @christyjacob remove once we migrate the rules in 1.7.x
if (version_compare(APP_VERSION_STABLE, '1.7.0', '<')) {
@@ -359,26 +362,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
throw $th;
}
} finally {
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$queueForUsage
->addMetric(METRIC_NETWORK_REQUESTS, 1)
->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize())
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->setProject($project)
->trigger()
;
$queueForFunctions
->setType(Func::TYPE_ASYNC_WRITE)
->setExecution($execution)
@@ -413,6 +396,26 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
->setStatusCode($execution['responseStatusCode'] ?? 200)
->send($body);
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$queueForUsage
->addMetric(METRIC_NETWORK_REQUESTS, 1)
->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize())
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->setProject($project)
->trigger()
;
return true;
} elseif ($type === 'api') {
$utopia->getRoute()?->label('error', '');
@@ -469,16 +472,17 @@ App::init()
->inject('queueForCertificates')
->inject('queueForFunctions')
->inject('isResourceBlocked')
->inject('previewHostname')
->inject('devKey')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, Document $devKey) {
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHostname, Document $devKey) {
/*
* Appwrite Router
*/
$host = $request->getHostname() ?? '';
$mainDomain = System::getEnv('_APP_DOMAIN', '');
// Only run Router when external domain
if ($host !== $mainDomain) {
if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked)) {
if ($host !== $mainDomain || !empty($previewHostname)) {
if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) {
return;
}
}
@@ -702,16 +706,17 @@ App::options()
->inject('queueForFunctions')
->inject('geodb')
->inject('isResourceBlocked')
->inject('previewHostname')
->inject('devKey')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, Document $devKey) {
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname, Document $devKey) {
/*
* Appwrite Router
*/
$host = $request->getHostname() ?? '';
$mainDomain = System::getEnv('_APP_DOMAIN', '');
// Only run Router when external domain
if ($host !== $mainDomain) {
if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked)) {
if ($host !== $mainDomain || !empty($previewHostname)) {
if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) {
return;
}
}
@@ -867,6 +872,8 @@ App::error()
if (isset($user) && !$user->isEmpty()) {
$log->setUser(new User($user->getId()));
} else {
$log->setUser(new User('guest-' . hash('sha256', $request->getIP())));
}
try {
@@ -877,7 +884,7 @@ App::error()
}
$log->setNamespace("http");
$log->setServer(\gethostname());
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
@@ -898,6 +905,7 @@ App::error()
$action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
$log->setAction($action);
$log->addTag('service', $action);
$isProduction = System::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
@@ -999,15 +1007,16 @@ App::get('/robots.txt')
->inject('queueForFunctions')
->inject('geodb')
->inject('isResourceBlocked')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) {
->inject('previewHostname')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) {
$host = $request->getHostname() ?? '';
$mainDomain = System::getEnv('_APP_DOMAIN', '');
if ($host === $mainDomain || $host === 'localhost') {
if (($host === $mainDomain || $host === 'localhost') && empty($previewHostname)) {
$template = new View(__DIR__ . '/../views/general/robots.phtml');
$response->text($template->render(false));
} else {
router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked);
router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname);
}
});
@@ -1026,15 +1035,16 @@ App::get('/humans.txt')
->inject('queueForFunctions')
->inject('geodb')
->inject('isResourceBlocked')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) {
->inject('previewHostname')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) {
$host = $request->getHostname() ?? '';
$mainDomain = System::getEnv('_APP_DOMAIN', '');
if ($host === $mainDomain || $host === 'localhost') {
if (($host === $mainDomain || $host === 'localhost') && empty($previewHostname)) {
$template = new View(__DIR__ . '/../views/general/humans.phtml');
$response->text($template->render(false));
} else {
router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked);
router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname);
}
});
app/controllers/shared/api.php
+30 -33
View File
@@ -185,13 +185,14 @@ App::init()
->inject('utopia')
->inject('request')
->inject('dbForConsole')
->inject('dbForProject')
->inject('project')
->inject('user')
->inject('session')
->inject('servers')
->inject('mode')
->inject('team')
->action(function (App $utopia, Request $request, Database $dbForConsole, Document $project, Document $user, ?Document $session, array $servers, string $mode, Document $team) {
->action(function (App $utopia, Request $request, Database $dbForConsole, Database $dbForProject, Document $project, Document $user, ?Document $session, array $servers, string $mode, Document $team) {
$route = $utopia->getRoute();
if ($project->isEmpty()) {
@@ -335,6 +336,33 @@ App::init()
Authorization::setRole($authRole);
}
/**
* Update project last activity
*/
if (!$project->isEmpty() && $project->getId() !== 'console') {
$accessedAt = $project->getAttribute('accessedAt', '');
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) {
$project->setAttribute('accessedAt', DateTime::now());
Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project));
}
}
/**
* Update user last activity
*/
if (!empty($user->getId())) {
$accessedAt = $user->getAttribute('accessedAt', '');
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) {
$user->setAttribute('accessedAt', DateTime::now());
if (APP_MODE_ADMIN !== $mode) {
$dbForProject->updateDocument('users', $user->getId(), $user);
} else {
$dbForConsole->updateDocument('users', $user->getId(), $user);
}
}
}
/** Do not allow access to disabled services */
$service = $route->getLabel('sdk.namespace', '');
if (!empty($service)) {
@@ -643,9 +671,7 @@ App::shutdown()
->inject('queueForWebhooks')
->inject('queueForRealtime')
->inject('dbForProject')
->inject('mode')
->inject('dbForConsole')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, string $mode, Database $dbForConsole) use ($parseLabel) {
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject) use ($parseLabel) {
$responsePayload = $response->getPayload();
@@ -765,8 +791,6 @@ App::shutdown()
}
}
if ($project->getId() !== 'console') {
if (!Auth::isPrivilegedUser(Authorization::getRoles())) {
$fileSize = 0;
@@ -785,33 +809,6 @@ App::shutdown()
->setProject($project)
->trigger();
}
/**
* Update project last activity
*/
if (!$project->isEmpty() && $project->getId() !== 'console') {
$accessedAt = $project->getAttribute('accessedAt', '');
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) {
$project->setAttribute('accessedAt', DateTime::now());
Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project));
}
}
/**
* Update user last activity
*/
if (!$user->isEmpty()) {
$accessedAt = $user->getAttribute('accessedAt', '');
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) {
$user->setAttribute('accessedAt', DateTime::now());
if (APP_MODE_ADMIN !== $mode) {
$dbForProject->updateDocument('users', $user->getId(), $user);
} else {
$dbForConsole->updateDocument('users', $user->getId(), $user);
}
}
}
});
App::init()
app/http.php
+5 -1
View File
@@ -54,6 +54,7 @@ $http
'http_compression' => false,
'package_max_length' => $payloadSize,
'buffer_output_size' => $payloadSize,
'task_worker_num' => 1, // required for the task to fetch domains background
]);
$http->on(Constant::EVENT_WORKER_START, function ($server, $workerId) {
@@ -369,10 +370,12 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool
if (isset($user) && !$user->isEmpty()) {
$log->setUser(new User($user->getId()));
} else {
$log->setUser(new User('guest-' . hash('sha256', $request->getIP())));
}
$log->setNamespace("http");
$log->setServer(\gethostname());
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($th->getMessage());
@@ -392,6 +395,7 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool
$action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
$log->setAction($action);
$log->addTag('service', $action);
$isProduction = System::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
app/init.php
+12 -1
View File
@@ -124,7 +124,7 @@ const APP_USER_ACCESS = 24 * 60 * 60; // 24 hours
const APP_PROJECT_ACCESS = 24 * 60 * 60; // 24 hours
const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
const APP_CACHE_BUSTER = 4318;
const APP_VERSION_STABLE = '1.6.0';
const APP_VERSION_STABLE = '1.6.1';
const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
const APP_DATABASE_ATTRIBUTE_IP = 'ip';
@@ -1864,3 +1864,14 @@ App::setResource(
'isResourceBlocked',
fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false
);
App::setResource('previewHostname', function (Request $request) {
if (App::isDevelopment()) {
$host = $request->getQuery('appwrite-hostname') ?? '';
if (!empty($host)) {
return $host;
}
}
return '';
}, ['request']);
app/realtime.php
+1 -1
View File
@@ -184,7 +184,7 @@ $logError = function (Throwable $error, string $action) use ($register) {
$log = new Log();
$log->setNamespace("realtime");
$log->setServer(gethostname());
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
app/views/install/compose.phtml
+1 -1
View File
@@ -167,7 +167,7 @@ $image = $this->getParam('image', '');
appwrite-console:
<<: *x-logging
container_name: appwrite-console
image: <?php echo $organization; ?>/console:5.0.12
image: <?php echo $organization; ?>/console:5.2.27
restart: unless-stopped
networks:
- appwrite
app/worker.php
+12 -3
View File
@@ -2,6 +2,7 @@
require_once __DIR__ . '/init.php';
use Appwrite\Certificates\LetsEncrypt;
use Appwrite\Event\Audit;
use Appwrite\Event\Build;
use Appwrite\Event\Certificate;
@@ -16,7 +17,6 @@ use Appwrite\Event\Usage;
use Appwrite\Event\UsageDump;
use Appwrite\Platform\Appwrite;
use Swoole\Runtime;
use Utopia\App;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
@@ -283,6 +283,15 @@ Server::setResource(
fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false
);
Server::setResource('certificates', function () {
$email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'));
if (empty($email)) {
throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.');
}
return new LetsEncrypt($email);
});
Server::setResource('logError', function (Registry $register, Document $project) {
return function (Throwable $error, string $namespace, string $action, ?array $extras) use ($register, $project) {
$logger = $register->get('logger');
@@ -292,7 +301,7 @@ Server::setResource('logError', function (Registry $register, Document $project)
$log = new Log();
$log->setNamespace($namespace);
$log->setServer(\gethostname());
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
@@ -385,7 +394,7 @@ $worker
if ($logger) {
$log->setNamespace("appwrite-worker");
$log->setServer(\gethostname());
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
composer.lock
Generated
+13 -13
View File
@@ -157,16 +157,16 @@
},
{
"name": "appwrite/php-runtimes",
"version": "0.16.4",
"version": "0.16.5",
"source": {
"type": "git",
"url": "https://github.com/appwrite/runtimes.git",
"reference": "7e4741337b9373f77210396e68eca539018cabd1"
"reference": "1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/appwrite/runtimes/zipball/7e4741337b9373f77210396e68eca539018cabd1",
"reference": "7e4741337b9373f77210396e68eca539018cabd1",
"url": "https://api.github.com/repos/appwrite/runtimes/zipball/1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9",
"reference": "1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9",
"shasum": ""
},
"require": {
@@ -206,9 +206,9 @@
],
"support": {
"issues": "https://github.com/appwrite/runtimes/issues",
"source": "https://github.com/appwrite/runtimes/tree/0.16.4"
"source": "https://github.com/appwrite/runtimes/tree/0.16.5"
},
"time": "2024-10-26T10:39:59+00:00"
"time": "2024-11-25T15:17:06+00:00"
},
{
"name": "beberlei/assert",
@@ -3928,16 +3928,16 @@
},
{
"name": "utopia-php/migration",
"version": "0.6.12",
"version": "0.6.13",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/migration.git",
"reference": "9a8c905af4cece5c5ec9542a5b534befce067260"
"reference": "68d9b0a9477755afcda607e7e8109785cae17a13"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/9a8c905af4cece5c5ec9542a5b534befce067260",
"reference": "9a8c905af4cece5c5ec9542a5b534befce067260",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/68d9b0a9477755afcda607e7e8109785cae17a13",
"reference": "68d9b0a9477755afcda607e7e8109785cae17a13",
"shasum": ""
},
"require": {
@@ -3978,9 +3978,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/migration/issues",
"source": "https://github.com/utopia-php/migration/tree/0.6.12"
"source": "https://github.com/utopia-php/migration/tree/0.6.13"
},
"time": "2024-11-12T00:31:53+00:00"
"time": "2024-11-26T13:57:53+00:00"
},
{
"name": "utopia-php/mongo",
@@ -8557,7 +8557,7 @@
],
"aliases": [],
"minimum-stability": "stable",
"stability-flags": {},
"stability-flags": [],
"prefer-stable": false,
"prefer-lowest": false,
"platform": {
docker-compose.yml
+2 -1
View File
@@ -199,7 +199,7 @@ services:
appwrite-console:
<<: *x-logging
container_name: appwrite-console
image: appwrite/console:5.0.12
image: appwrite/console:5.2.27
restart: unless-stopped
networks:
- appwrite
@@ -385,6 +385,7 @@ services:
- _APP_EXECUTOR_HOST
- _APP_DATABASE_SHARED_TABLES
- _APP_DATABASE_SHARED_TABLES_V1
- _APP_EMAIL_CERTIFICATES
appwrite-worker-databases:
entrypoint: worker-databases
src/Appwrite/Certificates/Adapter.php
+14
View File
@@ -0,0 +1,14 @@
<?php
namespace Appwrite\Certificates;
use Utopia\Logger\Log;
interface Adapter
{
public function issueCertificate(string $certName, string $domain): ?string;
public function isRenewRequired(string $domain, Log $log): bool;
public function deleteCertificate(string $domain): void;
}
src/Appwrite/Certificates/LetsEncrypt.php
+125
View File
@@ -0,0 +1,125 @@
<?php
namespace Appwrite\Certificates;
use Exception;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Logger\Log;
class LetsEncrypt implements Adapter
{
private string $email;
public function __construct(string $email)
{
$this->email = $email;
}
public function issueCertificate(string $certName, string $domain): ?string
{
$stdout = '';
$stderr = '';
$staging = (App::isProduction()) ? '' : ' --dry-run';
$exit = Console::execute(
"certbot certonly -v --webroot --noninteractive --agree-tos{$staging}"
. " --email " . $this->email
. " --cert-name " . $certName
. " -w " . APP_STORAGE_CERTIFICATES
. " -d {$domain}",
'',
$stdout,
$stderr
);
// Unexpected error, usually 5XX, API limits, ...
if ($exit !== 0) {
throw new Exception('Failed to issue a certificate with message: ' . $stderr);
}
// Prepare folder in storage for domain
$path = APP_STORAGE_CERTIFICATES . '/' . $domain;
if (!\is_readable($path)) {
if (!\mkdir($path, 0755, true)) {
throw new Exception('Failed to create path for certificate.');
}
}
// Move generated files
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) {
throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) {
throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) {
throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) {
throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
$config = \implode(PHP_EOL, [
"tls:",
" certificates:",
" - certFile: /storage/certificates/{$domain}/fullchain.pem",
" keyFile: /storage/certificates/{$domain}/privkey.pem"
]);
// Save configuration into Traefik using our new cert files
if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) {
throw new Exception('Failed to save Traefik configuration.');
}
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? null;
$dt = (new \DateTime())->setTimestamp($validTo);
return DateTime::addSeconds($dt, -60 * 60 * 24 * 30);
}
public function isRenewRequired(string $domain, Log $log): bool
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
if (\file_exists($certPath)) {
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? 0;
if (empty($validTo)) {
$log->addTag('certificateDomain', $domain);
throw new Exception('Unable to read certificate file (cert.pem).');
}
// LetsEncrypt allows renewal 30 days before expiry
$expiryInAdvance = (60 * 60 * 24 * 30);
if ($validTo - $expiryInAdvance > \time()) {
$log->addTag('certificateDomain', $domain);
$log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
return false;
}
}
return true;
}
public function deleteCertificate(string $domain): void
{
$directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
$checkTraversal = realpath($directory) === $directory;
if ($checkTraversal && is_dir($directory)) {
// Delete files, so Traefik is aware of change
array_map('unlink', glob($directory . '/*.*'));
rmdir($directory);
Console::info("Deleted certificate files for {$domain}");
} else {
Console::info("No certificate files found for {$domain}");
}
}
}
src/Appwrite/Migration/Migration.php
+1
View File
@@ -91,6 +91,7 @@ abstract class Migration
'1.5.10' => 'V20',
'1.5.11' => 'V20',
'1.6.0' => 'V21',
'1.6.1' => 'V21',
];
/**
src/Appwrite/Platform/Tasks/Maintenance.php
+1
View File
@@ -149,6 +149,7 @@ class Maintenance extends Action
$certificates = $dbForConsole->find('certificates', [
Query::lessThan('attempts', 5), // Maximum 5 attempts
Query::isNotNull('renewDate'),
Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew)
Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains)
]);
src/Appwrite/Platform/Workers/Certificates.php
+35 -169
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Certificates\Adapter as CertificatesAdapter;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Mail;
@@ -11,7 +12,6 @@ use Appwrite\Template\Template;
use Appwrite\Utopia\Response\Model\Rule;
use Exception;
use Throwable;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
@@ -48,7 +48,11 @@ class Certificates extends Action
->inject('queueForEvents')
->inject('queueForFunctions')
->inject('log')
->callback(fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log));
->inject('certificates')
->callback(
fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates) =>
$this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates)
);
}
/**
@@ -58,11 +62,12 @@ class Certificates extends Action
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param Log $log
* @param CertificatesAdapter $certificates
* @return void
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void
public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates): void
{
$payload = $message->getPayload() ?? [];
@@ -76,7 +81,7 @@ class Certificates extends Action
$log->addTag('domain', $domain->get());
$this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck);
$this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates, $skipRenewCheck);
}
/**
@@ -85,24 +90,24 @@ class Certificates extends Action
* @param Mail $queueForMails
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param CertificatesAdapter $certificates
* @param bool $skipRenewCheck
* @return void
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void
private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void
{
/**
* 1. Read arguments and validate domain
* 2. Get main domain
* 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain)
* 4. Validate security email. Cannot be empty, required by LetsEncrypt
* 5. Validate renew date with certificate file, unless requested to skip by parameter
* 6. Issue a certificate using certbot CLI
* 7. Update 'log' attribute on certificate document with Certbot message
* 8. Create storage folder for certificate, if not ready already
* 9. Move certificates from Certbot location to our Storage
* 10. Create/Update our Storage with new Traefik config with new certificate paths
* 4. Validate renew date with certificate file, unless requested to skip by parameter
* 5. Issue a certificate using certbot CLI
* 6. Update 'log' attribute on certificate document with Certbot message
* 7. Create storage folder for certificate, if not ready already
* 8. Move certificates from Certbot location to our Storage
* 9. Create/Update our Storage with new Traefik config with new certificate paths
* 11. Read certificate file and update 'renewDate' on certificate document
* 12. Update 'issueDate' and 'attempts' on certificate
*
@@ -119,7 +124,7 @@ class Certificates extends Action
* 2. Save document to database
* 3. Update all domains documents with current certificate ID
*
* Note: Renewals are checked and scheduled from maintenence worker
* Note: Renewals are checked and scheduled from maintenance worker
*/
// Get current certificate
@@ -134,40 +139,27 @@ class Certificates extends Action
$success = false;
try {
// Email for alerts is required by LetsEncrypt
$email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'));
if (empty($email)) {
throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue an SSL certificate.');
}
// Validate domain and DNS records. Skip if job is forced
if (!$skipRenewCheck) {
$mainDomain = $this->getMainDomain();
$isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain;
$this->validateDomain($domain, $isMainDomain, $log);
// If certificate exists already, double-check expiry date. Skip if job is forced
if (!$certificates->isRenewRequired($domain->get(), $log)) {
throw new Exception('Renew isn\'t required.');
}
}
// If certificate exists already, double-check expiry date. Skip if job is forced
if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) {
throw new Exception('Renew isn\'t required.');
}
// Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate
$folder = ID::unique();
// Generate certificate files using Let's Encrypt
$letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email);
// Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates.
$certName = ID::unique();
$renewDate = $certificates->issueCertificate($certName, $domain->get());
// Command succeeded, store all data into document
$logs = 'Certificate successfully generated.';
$certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB
// Give certificates to Traefik
$this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData);
$certificate->setAttribute('logs', 'Certificate successfully generated.');
// Update certificate info stored in database
$certificate->setAttribute('renewDate', $this->getRenewDate($domain->get()));
$certificate->setAttribute('renewDate', $renewDate);
$certificate->setAttribute('attempts', 0);
$certificate->setAttribute('issueDate', DateTime::now());
$success = true;
@@ -181,7 +173,7 @@ class Certificates extends Action
$attempts = $certificate->getAttribute('attempts', 0) + 1;
$certificate->setAttribute('attempts', $attempts);
// Store cuttent time as renew date to ensure another attempt in next maintenance cycle
// Store current time as renew date to ensure another attempt in next maintenance cycle.
$certificate->setAttribute('renewDate', DateTime::now());
// Send email to security email
@@ -245,8 +237,8 @@ class Certificates extends Action
}
/**
* Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check:
* - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt)
* Internal domain validation functionality to prevent unnecessary attempts. We check:
* - Domain needs to be public and valid (prevents NFT domains that are not supported)
* - Domain must have proper DNS record
*
* @param Domain $domain Domain which we validate
@@ -292,136 +284,6 @@ class Certificates extends Action
}
}
/**
* Reads expiry date of certificate from file and decides if renewal is required or not.
*
* @param string $domain Domain for which we check certificate file
* @return bool True, if certificate needs to be renewed
* @throws Exception
*/
private function isRenewRequired(string $domain, Log $log): bool
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
if (\file_exists($certPath)) {
$validTo = null;
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? 0;
if (empty($validTo)) {
$log->addTag('certificateDomain', $domain);
throw new Exception('Unable to read certificate file (cert.pem).');
}
// LetsEncrypt allows renewal 30 days before expiry
$expiryInAdvance = (60 * 60 * 24 * 30);
if ($validTo - $expiryInAdvance > \time()) {
$log->addTag('certificateDomain', $domain);
$log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
return false;
}
}
return true;
}
/**
* LetsEncrypt communication to issue certificate (using certbot CLI)
*
* @param string $folder Folder into which certificates should be generated
* @param string $domain Domain to generate certificate for
* @return array Named array with keys 'stdout' and 'stderr', both string
* @throws Exception
*/
private function issueCertificate(string $folder, string $domain, string $email): array
{
$stdout = '';
$stderr = '';
$staging = (App::isProduction()) ? '' : ' --dry-run';
$exit = Console::execute("certbot certonly -v --webroot --noninteractive --agree-tos{$staging}"
. " --email " . $email
. " --cert-name " . $folder
. " -w " . APP_STORAGE_CERTIFICATES
. " -d {$domain}", '', $stdout, $stderr);
// Unexpected error, usually 5XX, API limits, ...
if ($exit !== 0) {
throw new Exception('Failed to issue a certificate with message: ' . $stderr);
}
return [
'stdout' => $stdout,
'stderr' => $stderr
];
}
/**
* Read new renew date from certificate file generated by Let's Encrypt
*
* @param string $domain Domain which certificate was generated for
* @return string
* @throws \Utopia\Database\Exception
*/
private function getRenewDate(string $domain): string
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? null;
$dt = (new \DateTime())->setTimestamp($validTo);
return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days
}
/**
* Method to take files from Let's Encrypt, and put it into Traefik.
*
* @param string $domain Domain which certificate was generated for
* @param string $folder Folder in which certificates were generated
* @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error
* @return void
* @throws Exception
*/
private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void
{
// Prepare folder in storage for domain
$path = APP_STORAGE_CERTIFICATES . '/' . $domain;
if (!\is_readable($path)) {
if (!\mkdir($path, 0755, true)) {
throw new Exception('Failed to create path for certificate.');
}
}
// Move generated files
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) {
throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) {
throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) {
throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) {
throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
$config = \implode(PHP_EOL, [
"tls:",
" certificates:",
" - certFile: /storage/certificates/{$domain}/fullchain.pem",
" keyFile: /storage/certificates/{$domain}/privkey.pem"
]);
// Save configuration into Traefik using our new cert files
if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) {
throw new Exception('Failed to save Traefik configuration.');
}
}
/**
* Method to make sure information about error is delivered to admnistrator.
*
@@ -500,6 +362,10 @@ class Certificates extends Action
$project = $dbForConsole->getDocument('projects', $projectId);
if ($project->isEmpty()) {
return;
}
/** Trigger Webhook */
$ruleModel = new Rule();
$queueForEvents
src/Appwrite/Platform/Workers/Deletes.php
+22 -28
View File
@@ -3,6 +3,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Auth\Auth;
use Appwrite\Certificates\Adapter as CertificatesAdapter;
use Appwrite\Extend\Exception;
use Executor\Executor;
use Throwable;
@@ -50,18 +51,22 @@ class Deletes extends Action
->inject('deviceForFunctions')
->inject('deviceForBuilds')
->inject('deviceForCache')
->inject('certificates')
->inject('abuseRetention')
->inject('executionRetention')
->inject('auditRetention')
->inject('log')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log));
->callback(
fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) =>
$this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log)
);
}
/**
* @throws Exception
* @throws Throwable
*/
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void
{
$payload = $message->getPayload() ?? [];
@@ -84,10 +89,10 @@ class Deletes extends Action
case DELETE_TYPE_DOCUMENT:
switch ($document->getCollection()) {
case DELETE_TYPE_PROJECTS:
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document);
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document);
break;
case DELETE_TYPE_FUNCTIONS:
$this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project);
$this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project);
break;
case DELETE_TYPE_DEPLOYMENTS:
$this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project);
@@ -102,7 +107,7 @@ class Deletes extends Action
$this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project);
break;
case DELETE_TYPE_RULES:
$this->deleteRule($dbForConsole, $document);
$this->deleteRule($dbForConsole, $document, $certificates);
break;
default:
Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
@@ -110,7 +115,7 @@ class Deletes extends Action
}
break;
case DELETE_TYPE_TEAM_PROJECTS:
$this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $document);
$this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $certificates, $document);
break;
case DELETE_TYPE_EXECUTIONS:
$this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
@@ -264,7 +269,7 @@ class Deletes extends Action
MESSAGE_TYPE_EMAIL => 'emailTotal',
MESSAGE_TYPE_SMS => 'smsTotal',
MESSAGE_TYPE_PUSH => 'pushTotal',
default => throw new Exception('Invalid target provider type'),
default => throw new Exception('Invalid target CertificatesAdapter type'),
};
$dbForProject->decreaseDocumentAttribute(
'topics',
@@ -442,7 +447,7 @@ class Deletes extends Action
* @throws Structure
* @throws Exception
*/
private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Document $document): void
private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void
{
$projects = $dbForConsole->find('projects', [
@@ -455,7 +460,7 @@ class Deletes extends Action
$deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId());
$deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId());
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project);
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $project);
$dbForConsole->deleteDocument('projects', $project->getId());
}
}
@@ -473,7 +478,7 @@ class Deletes extends Action
* @throws Authorization
* @throws DatabaseException
*/
private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void
private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Document $document): void
{
$projectInternalId = $document->getInternalId();
$projectId = $document->getId();
@@ -537,8 +542,8 @@ class Deletes extends Action
// Delete project and function rules
$this->deleteByGroup('rules', [
Query::equal('projectInternalId', [$projectInternalId])
], $dbForConsole, function (Document $document) use ($dbForConsole) {
$this->deleteRule($dbForConsole, $document);
], $dbForConsole, function (Document $document) use ($dbForConsole, $certificates) {
$this->deleteRule($dbForConsole, $document, $certificates);
});
// Delete Keys
@@ -746,7 +751,7 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void
private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
@@ -761,8 +766,8 @@ class Deletes extends Action
Query::equal('resourceType', ['function']),
Query::equal('resourceInternalId', [$functionInternalId]),
Query::equal('projectInternalId', [$project->getInternalId()])
], $dbForConsole, function (Document $document) use ($project, $dbForConsole) {
$this->deleteRule($dbForConsole, $document);
], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $certificates) {
$this->deleteRule($dbForConsole, $document, $certificates);
});
/**
@@ -1048,21 +1053,10 @@ class Deletes extends Action
* @param Document $document rule document
* @return void
*/
private function deleteRule(Database $dbForConsole, Document $document): void
private function deleteRule(Database $dbForConsole, Document $document, CertificatesAdapter $certificates): void
{
$domain = $document->getAttribute('domain');
$directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
$checkTraversal = realpath($directory) === $directory;
if ($checkTraversal && is_dir($directory)) {
// Delete files, so Traefik is aware of change
array_map('unlink', glob($directory . '/*.*'));
rmdir($directory);
Console::info("Deleted certificate files for {$domain}");
} else {
Console::info("No certificate files found for {$domain}");
}
$certificates->deleteCertificate($domain);
// Delete certificate document, so Appwrite is aware of change
if (isset($document['certificateId'])) {