Compare commits

...
Author SHA1 Message Date
shimon 28bbb83435 handling map size 2024-12-10 15:17:39 +02:00
shimon fcd6894fcb Storing udage history in local copy 2024-12-09 10:24:18 +02:00
Jake BarnbyandGitHub dfb5ff1feb Merge pull request #9066 from appwrite/chore-upgrade-utopia-storage
Chore: upgrade utopia storage
2024-12-05 21:00:07 +13:00
Damodar Lohani 968b16d8bf upgrade utopia storage 2024-12-04 08:38:26 +00:00
Jake BarnbyandGitHub 3ce63b3d7d Merge pull request #9038 from appwrite/1.6.x-fix-delete-attribute
Fix delete notFound attribute
2024-12-02 13:49:42 +13:00
Steven NguyenandGitHub 3954560e30 Merge pull request #9050 from appwrite/1.6.x-db-0.53.20.a
Bomb db 0.53.200
2024-12-01 00:43:27 -08:00
fogelito 8349a7e8f7 lock 2024-12-01 10:17:28 +02:00
fogelito b923893699 0.53.20.hotfix1 2024-12-01 10:11:19 +02:00
Christy JacobandGitHub 16b5a8507e Merge pull request #9037 from appwrite/lohanidamodar-patch-4
Update storage.php
2024-11-28 12:10:41 +01:00
Christy JacobandGitHub 992808f640 Merge pull request #9039 from appwrite/fix-rules-collection
feat: use environment variable to check rules format
2024-11-28 11:49:50 +01:00
Christy Jacob 4dbf11e236 feat: review comments and linter 2024-11-28 14:33:02 +04:00
Christy Jacob 48d20b2c67 feat: update proxy and certificates files 2024-11-28 14:18:37 +04:00
Christy Jacob 7e98493324 feat: use environment variable to check rules format 2024-11-28 14:08:26 +04:00
fogelito 1054f877a1 Merge branch '1.6.x' of https://github.com/appwrite/appwrite into 1.6.x-fix-delete-attribute 2024-11-28 12:07:40 +02:00
Damodar LohaniandGitHub a4ff1bf3be Update storage.php 2024-11-28 15:47:09 +05:45
fogelito 6be15eee0c try catch not found 2024-11-28 11:59:43 +02:00
Jake BarnbyandGitHub 3d44757e37 Merge pull request #9032 from appwrite/1.6.x-fix-delete-attribute
Remove failed attribute
2024-11-28 16:32:32 +13:00
Jake Barnby 856be8ce0f Merge remote-tracking branch 'origin/1.6.x' into 1.6.x-fix-delete-attribute 2024-11-28 16:14:23 +13:00
Steven NguyenandGitHub 91be70babf Merge pull request #9034 from appwrite/chore-bump-console-5.2.27
Bump console to version 5.2.27
2024-11-27 09:48:39 -08:00
Steven NguyenandGitHub c062a8f129 Bump console to version 5.2.27 2024-11-27 16:47:32 +00:00
Steven NguyenandGitHub 35005657b9 Merge pull request #9026 from appwrite/feat-1.6.1-release-prep
Prepare 1.6.1 release
2024-11-27 08:44:36 -08:00
fogelito 98864b04e1 fix $options 2024-11-27 18:02:37 +02:00
fogelito e39c96e6d1 Merge branch '1.6.x' of https://github.com/appwrite/appwrite into 1.6.x-fix-delete-attribute 2024-11-27 14:37:36 +02:00
fogelito ca6cd349fb Remove failed attribute 2024-11-27 14:35:58 +02:00
Christy JacobandGitHub 441e56f445 Merge pull request #8932 from appwrite/fix-preview-domains-development
Fix: Preview domains locally
2024-11-27 13:13:33 +01:00
Christy JacobandGitHub 92faa24d38 Merge pull request #9028 from appwrite/PLA-1913
feat: add more tags to logging
2024-11-26 16:43:46 +01:00
Christy JacobandGitHub 399261bfb1 Merge pull request #9029 from appwrite/1.6.x-fix-restoration
fix restoration
2024-11-26 16:40:38 +01:00
Fabian Gruber f0b6c729c9 feat: add more tags to sentry 2024-11-26 16:13:55 +01:00
fogelito 135ff1a724 bomb migration 0.6.13 2024-11-26 17:05:17 +02:00
fogelito 442459f6a4 fix restoration 2024-11-26 16:14:38 +02:00
Christy JacobandGitHub 74c8a664a8 Merge pull request #8838 from appwrite/PLA-1791
feat: move certificate generation to Adapter
2024-11-26 14:24:12 +01:00
Fabian Gruber 98984a8085 refactor: use single adapter 2024-11-26 13:54:52 +01:00
Fabian Gruber 3a9ba8a6ad feat: move certificate generation to Adapter 2024-11-26 13:54:52 +01:00
Christy JacobandGitHub ce35567c05 Merge pull request #9002 from appwrite/fix-project-accessed-at
Fix: update project last activity on init
2024-11-26 13:18:55 +01:00
Damodar Lohani 3a21ed6333 Merge remote-tracking branch 'origin/1.6.x' into fix-project-accessed-at 2024-11-26 11:51:50 +00:00
Damodar Lohani 580a9f331f fix user update 2024-11-26 08:59:03 +00:00
Christy JacobandGitHub 33fad2cb23 Merge pull request #9020 from appwrite/chore-bump-console-5.2.25
Bump console to version 5.2.25
2024-11-25 17:24:25 +01:00
Christy JacobandGitHub 533009c1da Merge branch '1.6.x' into chore-bump-console-5.2.25 2024-11-25 17:21:52 +01:00
Steven NguyenandGitHub cb385674a5 Add 1.6.1 to CHANGES.md 2024-11-25 15:58:22 +00:00
Steven NguyenandGitHub fb4c71aebb Merge pull request #9025 from appwrite/fix-tasks-warning
Fix: swoole task warning
2024-11-25 07:07:09 -08:00
Matej Bačo 9d3bc1f80a Fix swoole task warning 2024-11-25 11:04:04 +01:00
Steven NguyenandGitHub 8da2b249b7 Merge pull request #9019 from appwrite/fix-8569-users-create-session-secret
Fix users create session secret
2024-11-22 21:55:28 -08:00
Steven NguyenandGitHub 07b3cbaec9 Bump appwrite version to 1.6.1 2024-11-22 22:47:02 +00:00
Steven NguyenandGitHub 98d78163a2 Bump console to version 5.2.25 2024-11-22 22:39:24 +00:00
Steven NguyenandGitHub 014c613c93 fix: update secret returned from users.createSession()
1. Include at least 1 factor because the minumum number of factors
   required when mfa is disabled is 1.
2. Purge the cached user document to ensure the new session is included
   in subsequent requests for the user.
3. Fix the encoding of the secret to match other parts of the codebase.
2024-11-22 21:52:27 +00:00
Steven NguyenandGitHub 54ec39f513 Merge pull request #9016 from appwrite/feat-safe-workers
Feat: Safe workers
2024-11-22 11:07:18 -08:00
Steven NguyenandGitHub ad235610da Merge pull request #9018 from appwrite/rules-backwards-compatibility
chore: backwards compatibility for 1.6.x
2024-11-22 09:53:09 -08:00
Christy Jacob ae7cb36240 chore: linter 2024-11-22 17:18:23 +00:00
Christy Jacob 33dd4df694 chore: backwards compatibility for 1.6.x 2024-11-22 17:16:51 +00:00
Matej Bačo 2fcfd1d7c9 Add safe workers 2024-11-22 13:57:42 +00:00
Damodar Lohani c33c5732e0 move accessed at to init 2024-11-20 04:29:57 +00:00
Damodar Lohani 2da2b5bb7f reset error hook 2024-11-20 04:25:37 +00:00
Damodar Lohani e8079cfd29 lint 2024-11-19 06:59:30 +00:00
Damodar LohaniandGitHub 842e23220a update project last activity on error hook 2024-11-19 07:51:50 +05:45
Matej BačoandGitHub b2cd20bf29 Merge pull request #8934 from appwrite/fix-bandwidth-calculation
Fix: bandwidth calculation
2024-11-18 11:16:51 +01:00
Matej Bačo c9e6ef8202 Rename based on PR reviews 2024-11-06 16:05:58 +00:00
Matej Bačo 87cfd7df3b Remove leftover 2024-11-06 11:39:44 +00:00
Matej Bačo ca887f018e fix domain execution response size 2024-11-06 11:29:24 +00:00
Matej Bačo 54c24b24e6 Improve preview domains local development flow 2024-11-06 10:52:43 +00:00
30 changed files with 861 additions and 434 deletions
+1
View File
@@ -23,6 +23,7 @@ _APP_OPENSSL_KEY_V1=your-secret-key
_APP_DOMAIN=traefik
_APP_DOMAIN_FUNCTIONS=functions.localhost
_APP_DOMAIN_TARGET=localhost
_APP_RULES_FORMAT=md5
_APP_REDIS_HOST=redis
_APP_REDIS_PORT=6379
_APP_REDIS_PASS=
+114
View File
@@ -1,3 +1,117 @@
# Version 1.6.1
## What's Changed
### Notable changes
* Remove JPEG fallback for webp by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8746
* Add heic and avif support by @lohanidamodar in https://github.com/appwrite/appwrite/pull/7718
* Add new runtimes by @Meldiron in https://github.com/appwrite/appwrite/pull/8771
* Remove audits deletion by @shimonewman in https://github.com/appwrite/appwrite/pull/8766
* Bump assistant by @loks0n in https://github.com/appwrite/appwrite/pull/8801
* Change max queries values to 500 by @fogelito in https://github.com/appwrite/appwrite/pull/8802
* Allow '.wav' as 'audio/x-wav' as well by @basert in https://github.com/appwrite/appwrite/pull/8846
* Use 1 instead of 0.5 cpu for default function specification by @loks0n in https://github.com/appwrite/appwrite/pull/8848
* Update function runtimes by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8781
* Add a realtime heartbeat by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8943
### Fixes
* Trigger functions event only if event is not paused by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8526
* Update docker-compose to restart usage-dump by @feschaffa in https://github.com/appwrite/appwrite/pull/8642
* Fix typo in scheduler base by @fogelito in https://github.com/appwrite/appwrite/pull/8691
* Add domain and force HTTPS env vars to mail worker by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8722
* Fix webp by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8732
* Ignore junction tables by @fogelito in https://github.com/appwrite/appwrite/pull/8728
* Fix logger throwing fatal error by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8724
* Fix missing protocol for testing SMTP by @byawitz in https://github.com/appwrite/appwrite/pull/8749
* Make create execution async loose by @loks0n in https://github.com/appwrite/appwrite/pull/8707
* Fix invalid cursor value by @fogelito in https://github.com/appwrite/appwrite/pull/8109
* Fix target deletes by @abnegate in https://github.com/appwrite/appwrite/pull/8833
* Fix translation commas by @loks0n in https://github.com/appwrite/appwrite/pull/8892
* Fix Migrations having source creds being overwritten and add Migration tests by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8897
* Fix validator usage for updating string size by @abnegate in https://github.com/appwrite/appwrite/pull/8890
* Fix create user event not triggering by @loks0n in https://github.com/appwrite/appwrite/pull/8718
* Improve error handling and logging in the database worker by @fogelito in https://github.com/appwrite/appwrite/pull/8944
* Remove inaccurate info about leaving the URL parameter empty by @ebenezerdon in https://github.com/appwrite/appwrite/pull/8963
* Ensure indexes are updated when updating an attribute key by @fogelito in https://github.com/appwrite/appwrite/pull/8971
* Remove duplicate dart-2.16 runtime template by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8972
* Fix team invites with existing session by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/9006
* Improve handling of HTTP requests by dispatching to safe workers by @Meldiron in https://github.com/appwrite/appwrite/pull/9016
* Fix users create session secret by @stnguyen90 in https://github.com/appwrite/appwrite/pull/9019
* Fix swoole task warning by @Meldiron in https://github.com/appwrite/appwrite/pull/9025
### Miscellaneous
* Update Init copy by @adityaoberai in https://github.com/appwrite/appwrite/pull/8557
* Fix security scan permissions and comment by @EVDOG4LIFE in https://github.com/appwrite/appwrite/pull/8525
* Add Trivy security scans by @btme0011 in https://github.com/appwrite/appwrite/pull/6876
* Update database stack by @abnegate in https://github.com/appwrite/appwrite/pull/8564
* Bump database by @abnegate in https://github.com/appwrite/appwrite/pull/8573
* Sync main with 1.5.x by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8589
* Add AWS to one-click installs by @byawitz in https://github.com/appwrite/appwrite/pull/8593
* Update Init copy in readme by @adityaoberai in https://github.com/appwrite/appwrite/pull/8618
* Sync main into 1.6.x by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8685
* Sync 1.6.x into main by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8686
* Feat coroutines by @Meldiron in https://github.com/appwrite/appwrite/pull/7826
* Sync main into 1.6.x by @Meldiron in https://github.com/appwrite/appwrite/pull/8719
* Sentence casing endpoint API reference by @choir241 in https://github.com/appwrite/appwrite/pull/8617
* DB storage metrics by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8404
* Fix exception thrown when optional array attribute does not exist by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8391
* Add projects channels to realtime by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8735
* Base for console roles support by @lohanidamodar in https://github.com/appwrite/appwrite/pull/8565
* Remove DB disk storage calculation by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8745
* Messaging adapter default values by @shimonewman in https://github.com/appwrite/appwrite/pull/8742
* Add payload response type by @loks0n in https://github.com/appwrite/appwrite/pull/8720
* Fix flaky functions tests by @loks0n in https://github.com/appwrite/appwrite/pull/8682
* Migrations Backups by @fogelito in https://github.com/appwrite/appwrite/pull/8186
* Add test for response and request filters by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8697
* Bump version in SECURITY.md by @EVDOG4LIFE in https://github.com/appwrite/appwrite/pull/8755
* Add originalId attribute to databases collection by @fogelito in https://github.com/appwrite/appwrite/pull/8764
* Fix Walter References by @ItzNotABug in https://github.com/appwrite/appwrite/pull/8757
* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8769
* Move new attributes by @abnegate in https://github.com/appwrite/appwrite/pull/8777
* Add ping endpoint by @loks0n in https://github.com/appwrite/appwrite/pull/8761
* Fix GitHub action caching by @loks0n in https://github.com/appwrite/appwrite/pull/8772
* Chore release ruby SDK by @abnegate in https://github.com/appwrite/appwrite/pull/8767
* Call migration success on success by @abnegate in https://github.com/appwrite/appwrite/pull/8782
* Update utopia-php/system to 0.9.0 by @basert in https://github.com/appwrite/appwrite/pull/8780
* Move createDocument from api to worker by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8776
* Add missing indexes by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8803
* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8809
* Fix typo in BLR region by @stnguyen90 in https://github.com/appwrite/appwrite/pull/8756
* Add tests for project variables by @vermakhushboo in https://github.com/appwrite/appwrite/pull/8815
* Replace 'Expires' with 'Cache-Control: private' header to avoid CDN caching by @basert in https://github.com/appwrite/appwrite/pull/8836
* Allow blocking based on resource attributes by @basert in https://github.com/appwrite/appwrite/pull/8812
* Check if resource is blocked inside functions worker by @basert in https://github.com/appwrite/appwrite/pull/8855
* Fix missing allow attribute by @abnegate in https://github.com/appwrite/appwrite/pull/8889
* Revert function execution order by @basert in https://github.com/appwrite/appwrite/pull/8857
* Use resource type constants by @basert in https://github.com/appwrite/appwrite/pull/8895
* Update Database lib by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8680
* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8917
* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8923
* Update database for transaction counter fixes with retries by @abnegate in https://github.com/appwrite/appwrite/pull/8927
* Validate string permissions by @fogelito in https://github.com/appwrite/appwrite/pull/8929
* Add PubSub adapter support by @basert in https://github.com/appwrite/appwrite/pull/8905
* List memberships as client by @loks0n in https://github.com/appwrite/appwrite/pull/8913
* Fix XDebug Extension not being removed by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8891
* Update database by @abnegate in https://github.com/appwrite/appwrite/pull/8946
* Use utopia compression by @loks0n in https://github.com/appwrite/appwrite/pull/8938
* Make compression minimum size configurable by @loks0n in https://github.com/appwrite/appwrite/pull/8947
* Revert "Update database" by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8949
* Fix setpaused by @loks0n in https://github.com/appwrite/appwrite/pull/8948
* Use getDocument instead of find() for rules by @christyjacob4 in https://github.com/appwrite/appwrite/pull/8951
* Remove double fetch from migrations worker by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8956
* Fix memberships privacy MFA by @loks0n in https://github.com/appwrite/appwrite/pull/8969
* Add telemetry by @basert in https://github.com/appwrite/appwrite/pull/8960
* Send migration errors individually by @PineappleIOnic in https://github.com/appwrite/appwrite/pull/8959
* Add console sdk previews by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/8990
* Unset index length by @fogelito in https://github.com/appwrite/appwrite/pull/8978
* Update base to 0.9.5 by @basert in https://github.com/appwrite/appwrite/pull/9005
* Sync main into 1.6.x by @TorstenDittmann in https://github.com/appwrite/appwrite/pull/9011
* Improved shared tables V2 by @abnegate in https://github.com/appwrite/appwrite/pull/9013
* Ensure backwards compatibility for 1.6.x by @christyjacob4 in https://github.com/appwrite/appwrite/pull/9018
# Version 1.6.0
## What's Changed
+12
View File
@@ -613,6 +613,18 @@ If you need to clear the cache, you can do so by running the following command:
docker compose exec redis redis-cli FLUSHALL
```
## Using preview domains locally
Appwrite Functions are automatically given a domain you can visit to execute the function. This domain has format `[SOMETHING].functions.localhost` unless you changed `_APP_DOMAIN_FUNCTIONS` environment variable. This default value works great when running Appwrite locally, but it can be impossible to use preview domains with Cloud woekspaces such as Gitpod or GitHub Codespaces.
To use preview domains on Cloud workspaces, you can visit hostname provided by them, and supply function's preview domain as URL parameter:
```
https://8080-appwrite-appwrite-mjeb3ebilwv.ws-eu116.gitpod.io/ping?preview=672b3c7eab1ac523ccf5.functions.localhost
```
The path was set to `/ping` intentionally. Visiting `/` for preview domains might trigger Console background worker, and trigger redirect to Console without our preview URL param. Visiting different path ensures this doesnt happen.
## Tutorials
From time to time, our team will add tutorials that will help contributors find their way in the Appwrite source code. Below is a list of currently available tutorials:
+3 -3
View File
@@ -67,7 +67,7 @@ docker run -it --rm \
--volume /var/run/docker.sock:/var/run/docker.sock \
--volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
--entrypoint="install" \
appwrite/appwrite:1.6.0
appwrite/appwrite:1.6.1
```
### Windows
@@ -79,7 +79,7 @@ docker run -it --rm ^
--volume //var/run/docker.sock:/var/run/docker.sock ^
--volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
--entrypoint="install" ^
appwrite/appwrite:1.6.0
appwrite/appwrite:1.6.1
```
#### PowerShell
@@ -89,7 +89,7 @@ docker run -it --rm `
--volume /var/run/docker.sock:/var/run/docker.sock `
--volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
--entrypoint="install" `
appwrite/appwrite:1.6.0
appwrite/appwrite:1.6.1
```
运行后,可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后,服务器可能需要几分钟才能启动。
+3 -3
View File
@@ -75,7 +75,7 @@ docker run -it --rm \
--volume /var/run/docker.sock:/var/run/docker.sock \
--volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
--entrypoint="install" \
appwrite/appwrite:1.6.0
appwrite/appwrite:1.6.1
```
### Windows
@@ -87,7 +87,7 @@ docker run -it --rm ^
--volume //var/run/docker.sock:/var/run/docker.sock ^
--volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
--entrypoint="install" ^
appwrite/appwrite:1.6.0
appwrite/appwrite:1.6.1
```
#### PowerShell
@@ -97,7 +97,7 @@ docker run -it --rm `
--volume /var/run/docker.sock:/var/run/docker.sock `
--volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
--entrypoint="install" `
appwrite/appwrite:1.6.0
appwrite/appwrite:1.6.1
```
Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation.
+1 -1
View File
@@ -181,7 +181,7 @@ CLI::setResource('logError', function (Registry $register) {
$log = new Log();
$log->setNamespace($namespace);
$log->setServer(\gethostname());
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
+1 -7
View File
@@ -290,15 +290,9 @@ function updateAttribute(
$attribute->setAttribute('size', $size);
}
$formatOptions = $attribute->getAttribute('formatOptions');
switch ($attribute->getAttribute('format')) {
case APP_DATABASE_ATTRIBUTE_INT_RANGE:
case APP_DATABASE_ATTRIBUTE_FLOAT_RANGE:
if ($min === $formatOptions['min'] && $max === $formatOptions['max']) {
break;
}
if ($min > $max) {
throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Minimum value must be lesser than maximum value');
}
@@ -385,7 +379,7 @@ function updateAttribute(
size: $size,
required: $required,
default: $default,
formatOptions: $options ?? null,
formatOptions: $options,
newKey: $newKey ?? null
);
} catch (TruncateException) {
+2 -1
View File
@@ -328,7 +328,8 @@ App::post('/v1/functions')
if (!empty($functionsDomain)) {
$routeSubdomain = ID::unique();
$domain = "{$routeSubdomain}.{$functionsDomain}";
$ruleId = md5($domain);
// TODO: @christyjacob remove once we migrate the rules in 1.7.x
$ruleId = System::getEnv('_APP_RULES_FORMAT') === 'md5' ? md5($domain) : ID::unique();
$rule = Authorization::skip(
fn () => $dbForConsole->createDocument('rules', new Document([
+13 -3
View File
@@ -11,6 +11,7 @@ use Utopia\App;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception\Query as QueryException;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Database\Validator\Query\Cursor;
use Utopia\Database\Validator\UID;
@@ -59,8 +60,15 @@ App::post('/v1/proxy/rules')
throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'This domain name is not allowed. Please pick another one.');
}
$ruleId = md5($domain);
$document = $dbForConsole->getDocument('rules', $ruleId);
// TODO: @christyjacob remove once we migrate the rules in 1.7.x
if (System::getEnv('_APP_RULES_FORMAT') === 'md5') {
$document = $dbForConsole->getDocument('rules', md5($domain));
} else {
$document = $dbForConsole->findOne('rules', [
Query::equal('domain', [$domain]),
]);
}
if (!$document->isEmpty()) {
if ($document->getAttribute('projectId') === $project->getId()) {
@@ -101,7 +109,9 @@ App::post('/v1/proxy/rules')
throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Domain may not start with http:// or https://.');
}
$ruleId = md5($domain->get());
// TODO: @christyjacob remove once we migrate the rules in 1.7.x
$ruleId = System::getEnv('_APP_RULES_FORMAT') === 'md5' ? md5($domain->get()) : ID::unique();
$rule = new Document([
'$id' => $ruleId,
'projectId' => $project->getId(),
+3 -2
View File
@@ -74,13 +74,14 @@ App::post('/v1/storage/buckets')
->inject('response')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, int $maximumFileSize, array $allowedFileExtensions, ?string $compression, bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $queueForEvents) {
->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, int $maximumFileSize, array $allowedFileExtensions, ?string $compression, ?bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $queueForEvents) {
$bucketId = $bucketId === 'unique()' ? ID::unique() : $bucketId;
// Map aggregate permissions into the multiple permissions they represent.
$permissions = Permission::aggregate($permissions);
$compression ??= Compression::NONE;
$encryption ??= true;
try {
$files = (Config::getParam('collections', [])['buckets'] ?? [])['files'] ?? [];
if (empty($files)) {
@@ -260,7 +261,7 @@ App::put('/v1/storage/buckets/:bucketId')
->inject('response')
->inject('dbForProject')
->inject('queueForEvents')
->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, ?int $maximumFileSize, array $allowedFileExtensions, ?string $compression, bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $queueForEvents) {
->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, ?int $maximumFileSize, array $allowedFileExtensions, ?string $compression, ?bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $queueForEvents) {
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if ($bucket->isEmpty()) {
+5 -1
View File
@@ -1804,6 +1804,7 @@ App::post('/v1/users/:userId/sessions')
'provider' => Auth::SESSION_PROVIDER_SERVER,
'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak
'userAgent' => $request->getUserAgent('UNKNOWN'),
'factors' => ['server'],
'ip' => $request->getIP(),
'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--',
'expire' => $expire,
@@ -1816,8 +1817,11 @@ App::post('/v1/users/:userId/sessions')
$countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown'));
$session = $dbForProject->createDocument('sessions', $session);
$dbForProject->purgeCachedDocument('users', $user->getId());
$session
->setAttribute('secret', $secret)
->setAttribute('secret', Auth::encodeSession($user->getId(), $secret))
->setAttribute('countryName', $countryName);
$queueForEvents
+72 -38
View File
@@ -29,6 +29,7 @@ use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Domains\Domain;
use Utopia\DSN\DSN;
@@ -45,13 +46,26 @@ Config::setParam('domainVerification', false);
Config::setParam('cookieDomain', 'localhost');
Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE);
function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked)
function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname)
{
$utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml');
$host = $request->getHostname() ?? '';
if (!empty($previewHostname)) {
$host = $previewHostname;
}
$route = Authorization::skip(fn () => $dbForConsole->getDocument('rules', md5($host)));
// TODO: @christyjacob remove once we migrate the rules in 1.7.x
if (System::getEnv('_APP_RULES_FORMAT') === 'md5') {
$route = Authorization::skip(fn () => $dbForConsole->getDocument('rules', md5($host)));
} else {
$route = Authorization::skip(
fn () => $dbForConsole->find('rules', [
Query::equal('domain', [$host]),
Query::limit(1)
])
)[0] ?? new Document();
}
if ($route->isEmpty()) {
if ($host === System::getEnv('_APP_DOMAIN_FUNCTIONS', '')) {
@@ -346,26 +360,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
throw $th;
}
} finally {
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$queueForUsage
->addMetric(METRIC_NETWORK_REQUESTS, 1)
->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize())
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->setProject($project)
->trigger()
;
$queueForFunctions
->setType(Func::TYPE_ASYNC_WRITE)
->setExecution($execution)
@@ -400,6 +394,26 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
->setStatusCode($execution['responseStatusCode'] ?? 200)
->send($body);
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$queueForUsage
->addMetric(METRIC_NETWORK_REQUESTS, 1)
->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize())
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->setProject($project)
->trigger()
;
return true;
} elseif ($type === 'api') {
$utopia->getRoute()?->label('error', '');
@@ -456,15 +470,16 @@ App::init()
->inject('queueForCertificates')
->inject('queueForFunctions')
->inject('isResourceBlocked')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked) {
->inject('previewHostname')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHostname) {
/*
* Appwrite Router
*/
$host = $request->getHostname() ?? '';
$mainDomain = System::getEnv('_APP_DOMAIN', '');
// Only run Router when external domain
if ($host !== $mainDomain) {
if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked)) {
if ($host !== $mainDomain || !empty($previewHostname)) {
if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) {
return;
}
}
@@ -512,18 +527,31 @@ App::init()
if (!empty($envDomain) && $envDomain !== 'localhost') {
$mainDomain = $envDomain;
} else {
$domainDocument = $dbForConsole->getDocument('rules', md5($envDomain));
// TODO: @christyjacob remove once we migrate the rules in 1.7.x
if (System::getEnv('_APP_RULES_FORMAT') === 'md5') {
$domainDocument = $dbForConsole->getDocument('rules', md5($envDomain));
} else {
$domainDocument = $dbForConsole->findOne('rules', [Query::orderAsc('$id')]);
}
$mainDomain = !$domainDocument->isEmpty() ? $domainDocument->getAttribute('domain') : $domain->get();
}
if ($mainDomain !== $domain->get()) {
Console::warning($domain->get() . ' is not a main domain. Skipping SSL certificate generation.');
} else {
$domainDocument = $dbForConsole->getDocument('rules', md5($domain->get()));
// TODO: @christyjacob remove once we migrate the rules in 1.7.x
if (System::getEnv('_APP_RULES_FORMAT') === 'md5') {
$domainDocument = $dbForConsole->getDocument('rules', md5($domain->get()));
} else {
$domainDocument = $dbForConsole->findOne('rules', [
Query::equal('domain', [$domain->get()])
]);
}
if ($domainDocument->isEmpty()) {
$domainDocument = new Document([
'$id' => md5($domain->get()),
// TODO: @christyjacob remove once we migrate the rules in 1.7.x
'$id' => System::getEnv('_APP_RULES_FORMAT') === 'md5' ? md5($domain->get()) : ID::unique(),
'domain' => $domain->get(),
'resourceType' => 'api',
'status' => 'verifying',
@@ -674,15 +702,16 @@ App::options()
->inject('queueForFunctions')
->inject('geodb')
->inject('isResourceBlocked')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) {
->inject('previewHostname')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) {
/*
* Appwrite Router
*/
$host = $request->getHostname() ?? '';
$mainDomain = System::getEnv('_APP_DOMAIN', '');
// Only run Router when external domain
if ($host !== $mainDomain) {
if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked)) {
if ($host !== $mainDomain || !empty($previewHostname)) {
if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) {
return;
}
}
@@ -837,6 +866,8 @@ App::error()
if (isset($user) && !$user->isEmpty()) {
$log->setUser(new User($user->getId()));
} else {
$log->setUser(new User('guest-' . hash('sha256', $request->getIP())));
}
try {
@@ -847,7 +878,7 @@ App::error()
}
$log->setNamespace("http");
$log->setServer(\gethostname());
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
@@ -868,6 +899,7 @@ App::error()
$action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
$log->setAction($action);
$log->addTag('service', $action);
$isProduction = System::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
@@ -969,15 +1001,16 @@ App::get('/robots.txt')
->inject('queueForFunctions')
->inject('geodb')
->inject('isResourceBlocked')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) {
->inject('previewHostname')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) {
$host = $request->getHostname() ?? '';
$mainDomain = System::getEnv('_APP_DOMAIN', '');
if ($host === $mainDomain || $host === 'localhost') {
if (($host === $mainDomain || $host === 'localhost') && empty($previewHostname)) {
$template = new View(__DIR__ . '/../views/general/robots.phtml');
$response->text($template->render(false));
} else {
router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked);
router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname);
}
});
@@ -996,15 +1029,16 @@ App::get('/humans.txt')
->inject('queueForFunctions')
->inject('geodb')
->inject('isResourceBlocked')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked) {
->inject('previewHostname')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) {
$host = $request->getHostname() ?? '';
$mainDomain = System::getEnv('_APP_DOMAIN', '');
if ($host === $mainDomain || $host === 'localhost') {
if (($host === $mainDomain || $host === 'localhost') && empty($previewHostname)) {
$template = new View(__DIR__ . '/../views/general/humans.phtml');
$response->text($template->render(false));
} else {
router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked);
router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname);
}
});
+30 -33
View File
@@ -185,13 +185,14 @@ App::init()
->inject('utopia')
->inject('request')
->inject('dbForConsole')
->inject('dbForProject')
->inject('project')
->inject('user')
->inject('session')
->inject('servers')
->inject('mode')
->inject('team')
->action(function (App $utopia, Request $request, Database $dbForConsole, Document $project, Document $user, ?Document $session, array $servers, string $mode, Document $team) {
->action(function (App $utopia, Request $request, Database $dbForConsole, Database $dbForProject, Document $project, Document $user, ?Document $session, array $servers, string $mode, Document $team) {
$route = $utopia->getRoute();
if ($project->isEmpty()) {
@@ -335,6 +336,33 @@ App::init()
Authorization::setRole($authRole);
}
/**
* Update project last activity
*/
if (!$project->isEmpty() && $project->getId() !== 'console') {
$accessedAt = $project->getAttribute('accessedAt', '');
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) {
$project->setAttribute('accessedAt', DateTime::now());
Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project));
}
}
/**
* Update user last activity
*/
if (!empty($user->getId())) {
$accessedAt = $user->getAttribute('accessedAt', '');
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) {
$user->setAttribute('accessedAt', DateTime::now());
if (APP_MODE_ADMIN !== $mode) {
$dbForProject->updateDocument('users', $user->getId(), $user);
} else {
$dbForConsole->updateDocument('users', $user->getId(), $user);
}
}
}
/** Do not allow access to disabled services */
$service = $route->getLabel('sdk.namespace', '');
if (!empty($service)) {
@@ -641,9 +669,7 @@ App::shutdown()
->inject('queueForWebhooks')
->inject('queueForRealtime')
->inject('dbForProject')
->inject('mode')
->inject('dbForConsole')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, string $mode, Database $dbForConsole) use ($parseLabel) {
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject) use ($parseLabel) {
$responsePayload = $response->getPayload();
@@ -763,8 +789,6 @@ App::shutdown()
}
}
if ($project->getId() !== 'console') {
if (!Auth::isPrivilegedUser(Authorization::getRoles())) {
$fileSize = 0;
@@ -783,33 +807,6 @@ App::shutdown()
->setProject($project)
->trigger();
}
/**
* Update project last activity
*/
if (!$project->isEmpty() && $project->getId() !== 'console') {
$accessedAt = $project->getAttribute('accessedAt', '');
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) {
$project->setAttribute('accessedAt', DateTime::now());
Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project));
}
}
/**
* Update user last activity
*/
if (!$user->isEmpty()) {
$accessedAt = $user->getAttribute('accessedAt', '');
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) {
$user->setAttribute('accessedAt', DateTime::now());
if (APP_MODE_ADMIN !== $mode) {
$dbForProject->updateDocument('users', $user->getId(), $user);
} else {
$dbForConsole->updateDocument('users', $user->getId(), $user);
}
}
}
});
App::init()
+162 -3
View File
@@ -9,6 +9,7 @@ use Swoole\Http\Request as SwooleRequest;
use Swoole\Http\Response as SwooleResponse;
use Swoole\Http\Server;
use Swoole\Process;
use Swoole\Table;
use Utopia\Abuse\Adapters\Database\TimeLimit;
use Utopia\App;
use Utopia\Audit\Audit;
@@ -16,11 +17,13 @@ use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Logger\Log;
use Utopia\Logger\Log\User;
@@ -28,6 +31,12 @@ use Utopia\Pools\Group;
use Utopia\Swoole\Files;
use Utopia\System\System;
const DOMAIN_SYNC_TIMER = 30; // 30 seconds
$domains = new Table(1_000_000); // 1 million rows
$domains->column('value', Table::TYPE_INT, 1);
$domains->create();
$http = new Server(
host: "0.0.0.0",
port: System::getEnv('PORT', 80),
@@ -35,15 +44,17 @@ $http = new Server(
);
$payloadSize = 12 * (1024 * 1024); // 12MB - adding slight buffer for headers and other data that might be sent with the payload - update later with valid testing
$workerNumber = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6));
$totalWorkers = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6));
$http
->set([
'worker_num' => $workerNumber,
'worker_num' => $totalWorkers,
'dispatch_func' => 'dispatch',
'open_http2_protocol' => true,
'http_compression' => false,
'package_max_length' => $payloadSize,
'buffer_output_size' => $payloadSize,
'task_worker_num' => 1, // required for the task to fetch domains background
]);
$http->on(Constant::EVENT_WORKER_START, function ($server, $workerId) {
@@ -58,6 +69,93 @@ $http->on(Constant::EVENT_AFTER_RELOAD, function ($server, $workerId) {
Console::success('Reload completed...');
});
/**
* Assigns HTTP requests to worker threads by analyzing its payload/content.
*
* Routes requests as 'safe' or 'risky' based on specific content patterns (like POST actions or certain domains)
* to optimize load distribution between the workers. Utilizes `$safeThreadsPercent` to manage risk by assigning
* riskier tasks to a dedicated worker subset. Prefers idle workers, with fallback to random selection if necessary.
* doc: https://openswoole.com/docs/modules/swoole-server/configuration#dispatch_func
*
* @param Server $server Swoole server instance.
* @param int $fd client ID
* @param int $type the type of data and its current state
* @param string|null $data Request content for categorization.
* @global int $totalThreads Total number of workers.
* @return int Chosen worker ID for the request.
*/
function dispatch(Server $server, int $fd, int $type, $data = null): int
{
global $totalWorkers, $domains;
// If data is not set we can send request to any worker
// first we try to pick idle worker, if not we randomly pick a worker
if ($data === null) {
for ($i = 0; $i < $totalWorkers; $i++) {
if ($server->getWorkerStatus($i) === SWOOLE_WORKER_IDLE) {
return $i;
}
}
return rand(0, $totalWorkers - 1);
}
$riskyWorkersPercent = intval(System::getEnv('_APP_RISKY_WORKERS_PERCENT', 80)) / 100; // Decimal form 0 to 1
// Each worker has numeric ID, starting from 0 and incrementing
// From 0 to riskyWorkers, we consider safe workers
// From riskyWorkers to totalWorkers, we consider risky workers
$riskyWorkers = (int) floor($totalWorkers * $riskyWorkersPercent); // Absolute amount of risky workers
$domain = '';
// max up to 3 as first line has request details and second line has host
$lines = explode("\n", $data, 3);
$request = $lines[0];
if (count($lines) > 1) {
$domain = trim(explode('Host: ', $lines[1])[1]);
}
// Sync executions are considered risky
$risky = false;
if (str_starts_with($request, 'POST') && str_contains($request, '/executions')) {
$risky = true;
} elseif (str_ends_with($domain, System::getEnv('_APP_DOMAIN_FUNCTIONS'))) {
$risky = true;
} elseif ($domains->get(md5($domain), 'value') === 1) {
// executions request coming from custom domain
$risky = true;
}
if ($risky) {
// If risky request, only consider risky workers
for ($j = $riskyWorkers; $j < $totalWorkers; $j++) {
/** Reference https://openswoole.com/docs/modules/swoole-server-getWorkerStatus#description */
if ($server->getWorkerStatus($j) === SWOOLE_WORKER_IDLE) {
// If idle worker found, give to him
return $j;
}
}
// If no idle workers, give to random risky worker
$worker = rand($riskyWorkers, $totalWorkers - 1);
Console::warning("swoole_dispatch: Risky branch: did not find a idle worker, picking random worker {$worker}");
return $worker;
}
// If safe request, give to any idle worker
// Its fine to pick risky worker here, because it's idle. Idle is never actually risky
for ($i = 0; $i < $totalWorkers; $i++) {
if ($server->getWorkerStatus($i) === SWOOLE_WORKER_IDLE) {
return $i;
}
}
// If no idle worker found, give to random safe worker
// We avoid risky workers here, as it could be in work - not idle. Thats exactly when they are risky.
$worker = rand(0, $riskyWorkers - 1);
Console::warning("swoole_dispatch: Non-risky branch: did not find a idle worker, picking random worker {$worker}");
return $worker;
}
include __DIR__ . '/controllers/general.php';
$http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $register) {
@@ -213,6 +311,9 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg
Console::success('Server started successfully (max payload is ' . number_format($payloadSize) . ' bytes)');
Console::info("Master pid {$http->master_pid}, manager pid {$http->manager_pid}");
// Start the task that starts fetching custom domains
$http->task([], 0);
// listen ctrl + c
Process::signal(2, function () use ($http) {
Console::log('Stop by Ctrl+C');
@@ -269,10 +370,12 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool
if (isset($user) && !$user->isEmpty()) {
$log->setUser(new User($user->getId()));
} else {
$log->setUser(new User('guest-' . hash('sha256', $request->getIP())));
}
$log->setNamespace("http");
$log->setServer(\gethostname());
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($th->getMessage());
@@ -292,6 +395,7 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool
$action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
$log->setAction($action);
$log->addTag('service', $action);
$isProduction = System::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
@@ -330,4 +434,59 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool
}
});
// Fetch domains every `DOMAIN_SYNC_TIMER` seconds and update in the memory
$http->on('Task', function () use ($register, $domains) {
$lastSyncUpdate = null;
$pools = $register->get('pools');
App::setResource('pools', fn () => $pools);
$app = new App('UTC');
/** @var Utopia\Database\Database $dbForConsole */
$dbForConsole = $app->getResource('dbForConsole');
Console::loop(function () use ($dbForConsole, $domains, &$lastSyncUpdate) {
try {
$time = DateTime::now();
$limit = 1000;
$sum = $limit;
$latestDocument = null;
while ($sum === $limit) {
$queries = [Query::limit($limit)];
if ($latestDocument !== null) {
$queries[] = Query::cursorAfter($latestDocument);
}
if ($lastSyncUpdate != null) {
$queries[] = Query::greaterThanEqual('$updatedAt', $lastSyncUpdate);
}
$queries[] = Query::equal('resourceType', ['function']);
$results = [];
try {
$results = Authorization::skip(fn () => $dbForConsole->find('rules', $queries));
} catch (Throwable $th) {
Console::error($th->getMessage());
}
$sum = count($results);
foreach ($results as $document) {
$domain = $document->getAttribute('domain');
if (str_ends_with($domain, System::getEnv('_APP_DOMAIN_FUNCTIONS'))) {
continue;
}
$domains->set(md5($domain), ['value' => 1]);
}
$latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null;
}
$lastSyncUpdate = $time;
if ($sum > 0) {
Console::log("Sync domains tick: {$sum} domains were updated");
}
} catch (Throwable $th) {
Console::error($th->getMessage());
}
}, DOMAIN_SYNC_TIMER, 0, function ($error) {
Console::error($error);
});
});
$http->start();
+12 -1
View File
@@ -123,7 +123,7 @@ const APP_USER_ACCESS = 24 * 60 * 60; // 24 hours
const APP_PROJECT_ACCESS = 24 * 60 * 60; // 24 hours
const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
const APP_CACHE_BUSTER = 4318;
const APP_VERSION_STABLE = '1.6.0';
const APP_VERSION_STABLE = '1.6.1';
const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
const APP_DATABASE_ATTRIBUTE_IP = 'ip';
@@ -1828,3 +1828,14 @@ App::setResource(
'isResourceBlocked',
fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false
);
App::setResource('previewHostname', function (Request $request) {
if (App::isDevelopment()) {
$host = $request->getQuery('appwrite-hostname') ?? '';
if (!empty($host)) {
return $host;
}
}
return '';
}, ['request']);
+1 -1
View File
@@ -184,7 +184,7 @@ $logError = function (Throwable $error, string $action) use ($register) {
$log = new Log();
$log->setNamespace("realtime");
$log->setServer(gethostname());
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
+1 -1
View File
@@ -167,7 +167,7 @@ $image = $this->getParam('image', '');
appwrite-console:
<<: *x-logging
container_name: appwrite-console
image: <?php echo $organization; ?>/console:5.0.12
image: <?php echo $organization; ?>/console:5.2.27
restart: unless-stopped
networks:
- appwrite
+12 -3
View File
@@ -2,6 +2,7 @@
require_once __DIR__ . '/init.php';
use Appwrite\Certificates\LetsEncrypt;
use Appwrite\Event\Audit;
use Appwrite\Event\Build;
use Appwrite\Event\Certificate;
@@ -16,7 +17,6 @@ use Appwrite\Event\Usage;
use Appwrite\Event\UsageDump;
use Appwrite\Platform\Appwrite;
use Swoole\Runtime;
use Utopia\App;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
@@ -283,6 +283,15 @@ Server::setResource(
fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false
);
Server::setResource('certificates', function () {
$email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'));
if (empty($email)) {
throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.');
}
return new LetsEncrypt($email);
});
Server::setResource('logError', function (Registry $register, Document $project) {
return function (Throwable $error, string $namespace, string $action, ?array $extras) use ($register, $project) {
$logger = $register->get('logger');
@@ -292,7 +301,7 @@ Server::setResource('logError', function (Registry $register, Document $project)
$log = new Log();
$log->setNamespace($namespace);
$log->setServer(\gethostname());
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
@@ -385,7 +394,7 @@ $worker
if ($logger) {
$log->setNamespace("appwrite-worker");
$log->setServer(\gethostname());
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
+1 -1
View File
@@ -51,7 +51,7 @@
"utopia-php/cache": "0.11.*",
"utopia-php/cli": "0.15.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.53.20",
"utopia-php/database": "0.53.200",
"utopia-php/domains": "0.5.*",
"utopia-php/dsn": "0.2.1",
"utopia-php/framework": "0.33.*",
Generated
+108 -107
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "ae3b9a491c9870a4897cdf712caba1e3",
"content-hash": "fae350df93342992edd8f639948e1570",
"packages": [
{
"name": "adhocore/jwt",
@@ -157,16 +157,16 @@
},
{
"name": "appwrite/php-runtimes",
"version": "0.16.4",
"version": "0.16.5",
"source": {
"type": "git",
"url": "https://github.com/appwrite/runtimes.git",
"reference": "7e4741337b9373f77210396e68eca539018cabd1"
"reference": "1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/appwrite/runtimes/zipball/7e4741337b9373f77210396e68eca539018cabd1",
"reference": "7e4741337b9373f77210396e68eca539018cabd1",
"url": "https://api.github.com/repos/appwrite/runtimes/zipball/1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9",
"reference": "1e430646fdf847a7caf3c611dcf3d6d5a28c3fd9",
"shasum": ""
},
"require": {
@@ -206,9 +206,9 @@
],
"support": {
"issues": "https://github.com/appwrite/runtimes/issues",
"source": "https://github.com/appwrite/runtimes/tree/0.16.4"
"source": "https://github.com/appwrite/runtimes/tree/0.16.5"
},
"time": "2024-10-26T10:39:59+00:00"
"time": "2024-11-25T15:17:06+00:00"
},
{
"name": "beberlei/assert",
@@ -709,16 +709,16 @@
},
{
"name": "google/protobuf",
"version": "v4.28.3",
"version": "v4.29.0",
"source": {
"type": "git",
"url": "https://github.com/protocolbuffers/protobuf-php.git",
"reference": "c5c311e0f3d89928251ac5a2f0e3db283612c100"
"reference": "0ef6b2eb74b782f3f9023276c324d22e440f7587"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/c5c311e0f3d89928251ac5a2f0e3db283612c100",
"reference": "c5c311e0f3d89928251ac5a2f0e3db283612c100",
"url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/0ef6b2eb74b782f3f9023276c324d22e440f7587",
"reference": "0ef6b2eb74b782f3f9023276c324d22e440f7587",
"shasum": ""
},
"require": {
@@ -747,9 +747,9 @@
"proto"
],
"support": {
"source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.28.3"
"source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.0"
},
"time": "2024-10-22T22:27:17+00:00"
"time": "2024-11-27T18:37:40+00:00"
},
{
"name": "jean85/pretty-package-versions",
@@ -2343,9 +2343,9 @@
"type": "library",
"extra": {
"branch-alias": {
"v10.0": "10.0.x-dev",
"v8.3": "8.3.x-dev",
"v9.0": "9.0.x-dev",
"v8.3": "8.3.x-dev"
"v10.0": "10.0.x-dev"
}
},
"autoload": {
@@ -2386,16 +2386,16 @@
},
{
"name": "symfony/deprecation-contracts",
"version": "v3.5.0",
"version": "v3.5.1",
"source": {
"type": "git",
"url": "https://github.com/symfony/deprecation-contracts.git",
"reference": "0e0d29ce1f20deffb4ab1b016a7257c4f1e789a1"
"reference": "74c71c939a79f7d5bf3c1ce9f5ea37ba0114c6f6"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/0e0d29ce1f20deffb4ab1b016a7257c4f1e789a1",
"reference": "0e0d29ce1f20deffb4ab1b016a7257c4f1e789a1",
"url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/74c71c939a79f7d5bf3c1ce9f5ea37ba0114c6f6",
"reference": "74c71c939a79f7d5bf3c1ce9f5ea37ba0114c6f6",
"shasum": ""
},
"require": {
@@ -2433,7 +2433,7 @@
"description": "A generic function and convention to trigger deprecation notices",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/deprecation-contracts/tree/v3.5.0"
"source": "https://github.com/symfony/deprecation-contracts/tree/v3.5.1"
},
"funding": [
{
@@ -2449,30 +2449,31 @@
"type": "tidelift"
}
],
"time": "2024-04-18T09:32:20+00:00"
"time": "2024-09-25T14:20:29+00:00"
},
{
"name": "symfony/http-client",
"version": "v7.1.8",
"version": "v7.2.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/http-client.git",
"reference": "c30d91a1deac0dc3ed5e604683cf2e1dfc635b8a"
"reference": "955e43336aff03df1e8a8e17daefabb0127a313b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/http-client/zipball/c30d91a1deac0dc3ed5e604683cf2e1dfc635b8a",
"reference": "c30d91a1deac0dc3ed5e604683cf2e1dfc635b8a",
"url": "https://api.github.com/repos/symfony/http-client/zipball/955e43336aff03df1e8a8e17daefabb0127a313b",
"reference": "955e43336aff03df1e8a8e17daefabb0127a313b",
"shasum": ""
},
"require": {
"php": ">=8.2",
"psr/log": "^1|^2|^3",
"symfony/deprecation-contracts": "^2.5|^3",
"symfony/http-client-contracts": "^3.4.1",
"symfony/http-client-contracts": "~3.4.3|^3.5.1",
"symfony/service-contracts": "^2.5|^3"
},
"conflict": {
"amphp/amp": "<2.5",
"php-http/discovery": "<1.15",
"symfony/http-foundation": "<6.4"
},
@@ -2483,14 +2484,14 @@
"symfony/http-client-implementation": "3.0"
},
"require-dev": {
"amphp/amp": "^2.5",
"amphp/http-client": "^4.2.1",
"amphp/http-tunnel": "^1.0",
"amphp/http-client": "^4.2.1|^5.0",
"amphp/http-tunnel": "^1.0|^2.0",
"amphp/socket": "^1.1",
"guzzlehttp/promises": "^1.4|^2.0",
"nyholm/psr7": "^1.0",
"php-http/httplug": "^1.0|^2.0",
"psr/http-client": "^1.0",
"symfony/amphp-http-client-meta": "^1.0|^2.0",
"symfony/dependency-injection": "^6.4|^7.0",
"symfony/http-kernel": "^6.4|^7.0",
"symfony/messenger": "^6.4|^7.0",
@@ -2527,7 +2528,7 @@
"http"
],
"support": {
"source": "https://github.com/symfony/http-client/tree/v7.1.8"
"source": "https://github.com/symfony/http-client/tree/v7.2.0"
},
"funding": [
{
@@ -2543,20 +2544,20 @@
"type": "tidelift"
}
],
"time": "2024-11-13T13:40:27+00:00"
"time": "2024-11-29T08:22:02+00:00"
},
{
"name": "symfony/http-client-contracts",
"version": "v3.5.0",
"version": "v3.5.1",
"source": {
"type": "git",
"url": "https://github.com/symfony/http-client-contracts.git",
"reference": "20414d96f391677bf80078aa55baece78b82647d"
"reference": "c2f3ad828596624ca39ea40f83617ef51ca8bbf9"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/20414d96f391677bf80078aa55baece78b82647d",
"reference": "20414d96f391677bf80078aa55baece78b82647d",
"url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/c2f3ad828596624ca39ea40f83617ef51ca8bbf9",
"reference": "c2f3ad828596624ca39ea40f83617ef51ca8bbf9",
"shasum": ""
},
"require": {
@@ -2605,7 +2606,7 @@
"standards"
],
"support": {
"source": "https://github.com/symfony/http-client-contracts/tree/v3.5.0"
"source": "https://github.com/symfony/http-client-contracts/tree/v3.5.1"
},
"funding": [
{
@@ -2621,7 +2622,7 @@
"type": "tidelift"
}
],
"time": "2024-04-18T09:32:20+00:00"
"time": "2024-11-25T12:02:18+00:00"
},
{
"name": "symfony/polyfill-mbstring",
@@ -2861,16 +2862,16 @@
},
{
"name": "symfony/service-contracts",
"version": "v3.5.0",
"version": "v3.5.1",
"source": {
"type": "git",
"url": "https://github.com/symfony/service-contracts.git",
"reference": "bd1d9e59a81d8fa4acdcea3f617c581f7475a80f"
"reference": "e53260aabf78fb3d63f8d79d69ece59f80d5eda0"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/service-contracts/zipball/bd1d9e59a81d8fa4acdcea3f617c581f7475a80f",
"reference": "bd1d9e59a81d8fa4acdcea3f617c581f7475a80f",
"url": "https://api.github.com/repos/symfony/service-contracts/zipball/e53260aabf78fb3d63f8d79d69ece59f80d5eda0",
"reference": "e53260aabf78fb3d63f8d79d69ece59f80d5eda0",
"shasum": ""
},
"require": {
@@ -2924,7 +2925,7 @@
"standards"
],
"support": {
"source": "https://github.com/symfony/service-contracts/tree/v3.5.0"
"source": "https://github.com/symfony/service-contracts/tree/v3.5.1"
},
"funding": [
{
@@ -2940,7 +2941,7 @@
"type": "tidelift"
}
],
"time": "2024-04-18T09:32:20+00:00"
"time": "2024-09-25T14:20:29+00:00"
},
{
"name": "tbachert/spi",
@@ -3475,16 +3476,16 @@
},
{
"name": "utopia-php/database",
"version": "0.53.20",
"version": "0.53.200",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"reference": "e43f8ee26e06ee8812737e63642dbd7ee7c9dc04"
"reference": "570c63a3760d0e1404679ddfacd9484af40bd9fc"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/e43f8ee26e06ee8812737e63642dbd7ee7c9dc04",
"reference": "e43f8ee26e06ee8812737e63642dbd7ee7c9dc04",
"url": "https://api.github.com/repos/utopia-php/database/zipball/570c63a3760d0e1404679ddfacd9484af40bd9fc",
"reference": "570c63a3760d0e1404679ddfacd9484af40bd9fc",
"shasum": ""
},
"require": {
@@ -3525,9 +3526,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.53.20"
"source": "https://github.com/utopia-php/database/tree/0.53.200"
},
"time": "2024-11-12T00:23:36+00:00"
"time": "2024-12-01T07:59:15+00:00"
},
{
"name": "utopia-php/domains",
@@ -3928,16 +3929,16 @@
},
{
"name": "utopia-php/migration",
"version": "0.6.12",
"version": "0.6.13",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/migration.git",
"reference": "9a8c905af4cece5c5ec9542a5b534befce067260"
"reference": "68d9b0a9477755afcda607e7e8109785cae17a13"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/9a8c905af4cece5c5ec9542a5b534befce067260",
"reference": "9a8c905af4cece5c5ec9542a5b534befce067260",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/68d9b0a9477755afcda607e7e8109785cae17a13",
"reference": "68d9b0a9477755afcda607e7e8109785cae17a13",
"shasum": ""
},
"require": {
@@ -3978,9 +3979,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/migration/issues",
"source": "https://github.com/utopia-php/migration/tree/0.6.12"
"source": "https://github.com/utopia-php/migration/tree/0.6.13"
},
"time": "2024-11-12T00:31:53+00:00"
"time": "2024-11-26T13:57:53+00:00"
},
{
"name": "utopia-php/mongo",
@@ -4362,16 +4363,16 @@
},
{
"name": "utopia-php/storage",
"version": "0.18.6",
"version": "0.18.8",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/storage.git",
"reference": "893ccf06e183f8ece2aed8dbf14d64d6ba036071"
"reference": "84737afa634e6a833fc4f8b0c967553234d3f215"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/893ccf06e183f8ece2aed8dbf14d64d6ba036071",
"reference": "893ccf06e183f8ece2aed8dbf14d64d6ba036071",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/84737afa634e6a833fc4f8b0c967553234d3f215",
"reference": "84737afa634e6a833fc4f8b0c967553234d3f215",
"shasum": ""
},
"require": {
@@ -4411,9 +4412,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/storage/issues",
"source": "https://github.com/utopia-php/storage/tree/0.18.6"
"source": "https://github.com/utopia-php/storage/tree/0.18.8"
},
"time": "2024-11-06T09:58:50+00:00"
"time": "2024-12-04T08:30:35+00:00"
},
{
"name": "utopia-php/swoole",
@@ -5127,16 +5128,16 @@
},
{
"name": "laravel/pint",
"version": "v1.18.2",
"version": "v1.18.3",
"source": {
"type": "git",
"url": "https://github.com/laravel/pint.git",
"reference": "f55daaf7eb6c2f49ddf6702fb42e3091c64d8a64"
"reference": "cef51821608239040ab841ad6e1c6ae502ae3026"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/laravel/pint/zipball/f55daaf7eb6c2f49ddf6702fb42e3091c64d8a64",
"reference": "f55daaf7eb6c2f49ddf6702fb42e3091c64d8a64",
"url": "https://api.github.com/repos/laravel/pint/zipball/cef51821608239040ab841ad6e1c6ae502ae3026",
"reference": "cef51821608239040ab841ad6e1c6ae502ae3026",
"shasum": ""
},
"require": {
@@ -5147,13 +5148,13 @@
"php": "^8.1.0"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "^3.64.0",
"illuminate/view": "^10.48.20",
"larastan/larastan": "^2.9.8",
"friendsofphp/php-cs-fixer": "^3.65.0",
"illuminate/view": "^10.48.24",
"larastan/larastan": "^2.9.11",
"laravel-zero/framework": "^10.4.0",
"mockery/mockery": "^1.6.12",
"nunomaduro/termwind": "^1.15.1",
"pestphp/pest": "^2.35.1"
"nunomaduro/termwind": "^1.17.0",
"pestphp/pest": "^2.36.0"
},
"bin": [
"builds/pint"
@@ -5189,7 +5190,7 @@
"issues": "https://github.com/laravel/pint/issues",
"source": "https://github.com/laravel/pint"
},
"time": "2024-11-20T09:33:46+00:00"
"time": "2024-11-26T15:34:00+00:00"
},
{
"name": "matthiasmullie/minify",
@@ -7577,16 +7578,16 @@
},
{
"name": "symfony/console",
"version": "v7.1.8",
"version": "v7.2.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/console.git",
"reference": "ff04e5b5ba043d2badfb308197b9e6b42883fcd5"
"reference": "23c8aae6d764e2bae02d2a99f7532a7f6ed619cf"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/console/zipball/ff04e5b5ba043d2badfb308197b9e6b42883fcd5",
"reference": "ff04e5b5ba043d2badfb308197b9e6b42883fcd5",
"url": "https://api.github.com/repos/symfony/console/zipball/23c8aae6d764e2bae02d2a99f7532a7f6ed619cf",
"reference": "23c8aae6d764e2bae02d2a99f7532a7f6ed619cf",
"shasum": ""
},
"require": {
@@ -7650,7 +7651,7 @@
"terminal"
],
"support": {
"source": "https://github.com/symfony/console/tree/v7.1.8"
"source": "https://github.com/symfony/console/tree/v7.2.0"
},
"funding": [
{
@@ -7666,20 +7667,20 @@
"type": "tidelift"
}
],
"time": "2024-11-06T14:23:19+00:00"
"time": "2024-11-06T14:24:19+00:00"
},
{
"name": "symfony/filesystem",
"version": "v7.1.6",
"version": "v7.2.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/filesystem.git",
"reference": "c835867b3c62bb05c7fe3d637c871c7ae52024d4"
"reference": "b8dce482de9d7c9fe2891155035a7248ab5c7fdb"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/filesystem/zipball/c835867b3c62bb05c7fe3d637c871c7ae52024d4",
"reference": "c835867b3c62bb05c7fe3d637c871c7ae52024d4",
"url": "https://api.github.com/repos/symfony/filesystem/zipball/b8dce482de9d7c9fe2891155035a7248ab5c7fdb",
"reference": "b8dce482de9d7c9fe2891155035a7248ab5c7fdb",
"shasum": ""
},
"require": {
@@ -7716,7 +7717,7 @@
"description": "Provides basic utilities for the filesystem",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/filesystem/tree/v7.1.6"
"source": "https://github.com/symfony/filesystem/tree/v7.2.0"
},
"funding": [
{
@@ -7732,20 +7733,20 @@
"type": "tidelift"
}
],
"time": "2024-10-25T15:11:02+00:00"
"time": "2024-10-25T15:15:23+00:00"
},
{
"name": "symfony/finder",
"version": "v7.1.6",
"version": "v7.2.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/finder.git",
"reference": "2cb89664897be33f78c65d3d2845954c8d7a43b8"
"reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/finder/zipball/2cb89664897be33f78c65d3d2845954c8d7a43b8",
"reference": "2cb89664897be33f78c65d3d2845954c8d7a43b8",
"url": "https://api.github.com/repos/symfony/finder/zipball/6de263e5868b9a137602dd1e33e4d48bfae99c49",
"reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49",
"shasum": ""
},
"require": {
@@ -7780,7 +7781,7 @@
"description": "Finds files and directories via an intuitive fluent interface",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/finder/tree/v7.1.6"
"source": "https://github.com/symfony/finder/tree/v7.2.0"
},
"funding": [
{
@@ -7796,20 +7797,20 @@
"type": "tidelift"
}
],
"time": "2024-10-01T08:31:23+00:00"
"time": "2024-10-23T06:56:12+00:00"
},
{
"name": "symfony/options-resolver",
"version": "v7.1.6",
"version": "v7.2.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/options-resolver.git",
"reference": "85e95eeede2d41cd146146e98c9c81d9214cae85"
"reference": "7da8fbac9dcfef75ffc212235d76b2754ce0cf50"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/options-resolver/zipball/85e95eeede2d41cd146146e98c9c81d9214cae85",
"reference": "85e95eeede2d41cd146146e98c9c81d9214cae85",
"url": "https://api.github.com/repos/symfony/options-resolver/zipball/7da8fbac9dcfef75ffc212235d76b2754ce0cf50",
"reference": "7da8fbac9dcfef75ffc212235d76b2754ce0cf50",
"shasum": ""
},
"require": {
@@ -7847,7 +7848,7 @@
"options"
],
"support": {
"source": "https://github.com/symfony/options-resolver/tree/v7.1.6"
"source": "https://github.com/symfony/options-resolver/tree/v7.2.0"
},
"funding": [
{
@@ -7863,7 +7864,7 @@
"type": "tidelift"
}
],
"time": "2024-09-25T14:20:29+00:00"
"time": "2024-11-20T11:17:29+00:00"
},
{
"name": "symfony/polyfill-ctype",
@@ -8181,16 +8182,16 @@
},
{
"name": "symfony/process",
"version": "v7.1.8",
"version": "v7.2.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/process.git",
"reference": "42783370fda6e538771f7c7a36e9fa2ee3a84892"
"reference": "d34b22ba9390ec19d2dd966c40aa9e8462f27a7e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/process/zipball/42783370fda6e538771f7c7a36e9fa2ee3a84892",
"reference": "42783370fda6e538771f7c7a36e9fa2ee3a84892",
"url": "https://api.github.com/repos/symfony/process/zipball/d34b22ba9390ec19d2dd966c40aa9e8462f27a7e",
"reference": "d34b22ba9390ec19d2dd966c40aa9e8462f27a7e",
"shasum": ""
},
"require": {
@@ -8222,7 +8223,7 @@
"description": "Executes commands in sub-processes",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/process/tree/v7.1.8"
"source": "https://github.com/symfony/process/tree/v7.2.0"
},
"funding": [
{
@@ -8238,20 +8239,20 @@
"type": "tidelift"
}
],
"time": "2024-11-06T14:23:19+00:00"
"time": "2024-11-06T14:24:19+00:00"
},
{
"name": "symfony/string",
"version": "v7.1.8",
"version": "v7.2.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/string.git",
"reference": "591ebd41565f356fcd8b090fe64dbb5878f50281"
"reference": "446e0d146f991dde3e73f45f2c97a9faad773c82"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/string/zipball/591ebd41565f356fcd8b090fe64dbb5878f50281",
"reference": "591ebd41565f356fcd8b090fe64dbb5878f50281",
"url": "https://api.github.com/repos/symfony/string/zipball/446e0d146f991dde3e73f45f2c97a9faad773c82",
"reference": "446e0d146f991dde3e73f45f2c97a9faad773c82",
"shasum": ""
},
"require": {
@@ -8309,7 +8310,7 @@
"utf8"
],
"support": {
"source": "https://github.com/symfony/string/tree/v7.1.8"
"source": "https://github.com/symfony/string/tree/v7.2.0"
},
"funding": [
{
@@ -8325,7 +8326,7 @@
"type": "tidelift"
}
],
"time": "2024-11-13T13:31:21+00:00"
"time": "2024-11-13T13:31:26+00:00"
},
{
"name": "textalk/websocket",
+2 -1
View File
@@ -199,7 +199,7 @@ services:
appwrite-console:
<<: *x-logging
container_name: appwrite-console
image: appwrite/console:5.0.12
image: appwrite/console:5.2.27
restart: unless-stopped
networks:
- appwrite
@@ -385,6 +385,7 @@ services:
- _APP_EXECUTOR_HOST
- _APP_DATABASE_SHARED_TABLES
- _APP_DATABASE_SHARED_TABLES_V1
- _APP_EMAIL_CERTIFICATES
appwrite-worker-databases:
entrypoint: worker-databases
+14
View File
@@ -0,0 +1,14 @@
<?php
namespace Appwrite\Certificates;
use Utopia\Logger\Log;
interface Adapter
{
public function issueCertificate(string $certName, string $domain): ?string;
public function isRenewRequired(string $domain, Log $log): bool;
public function deleteCertificate(string $domain): void;
}
+125
View File
@@ -0,0 +1,125 @@
<?php
namespace Appwrite\Certificates;
use Exception;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Logger\Log;
class LetsEncrypt implements Adapter
{
private string $email;
public function __construct(string $email)
{
$this->email = $email;
}
public function issueCertificate(string $certName, string $domain): ?string
{
$stdout = '';
$stderr = '';
$staging = (App::isProduction()) ? '' : ' --dry-run';
$exit = Console::execute(
"certbot certonly -v --webroot --noninteractive --agree-tos{$staging}"
. " --email " . $this->email
. " --cert-name " . $certName
. " -w " . APP_STORAGE_CERTIFICATES
. " -d {$domain}",
'',
$stdout,
$stderr
);
// Unexpected error, usually 5XX, API limits, ...
if ($exit !== 0) {
throw new Exception('Failed to issue a certificate with message: ' . $stderr);
}
// Prepare folder in storage for domain
$path = APP_STORAGE_CERTIFICATES . '/' . $domain;
if (!\is_readable($path)) {
if (!\mkdir($path, 0755, true)) {
throw new Exception('Failed to create path for certificate.');
}
}
// Move generated files
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) {
throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) {
throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) {
throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) {
throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
$config = \implode(PHP_EOL, [
"tls:",
" certificates:",
" - certFile: /storage/certificates/{$domain}/fullchain.pem",
" keyFile: /storage/certificates/{$domain}/privkey.pem"
]);
// Save configuration into Traefik using our new cert files
if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) {
throw new Exception('Failed to save Traefik configuration.');
}
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? null;
$dt = (new \DateTime())->setTimestamp($validTo);
return DateTime::addSeconds($dt, -60 * 60 * 24 * 30);
}
public function isRenewRequired(string $domain, Log $log): bool
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
if (\file_exists($certPath)) {
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? 0;
if (empty($validTo)) {
$log->addTag('certificateDomain', $domain);
throw new Exception('Unable to read certificate file (cert.pem).');
}
// LetsEncrypt allows renewal 30 days before expiry
$expiryInAdvance = (60 * 60 * 24 * 30);
if ($validTo - $expiryInAdvance > \time()) {
$log->addTag('certificateDomain', $domain);
$log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
return false;
}
}
return true;
}
public function deleteCertificate(string $domain): void
{
$directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
$checkTraversal = realpath($directory) === $directory;
if ($checkTraversal && is_dir($directory)) {
// Delete files, so Traefik is aware of change
array_map('unlink', glob($directory . '/*.*'));
rmdir($directory);
Console::info("Deleted certificate files for {$domain}");
} else {
Console::info("No certificate files found for {$domain}");
}
}
}
+1
View File
@@ -91,6 +91,7 @@ abstract class Migration
'1.5.10' => 'V20',
'1.5.11' => 'V20',
'1.6.0' => 'V21',
'1.6.1' => 'V21',
];
/**
@@ -149,6 +149,7 @@ class Maintenance extends Action
$certificates = $dbForConsole->find('certificates', [
Query::lessThan('attempts', 5), // Maximum 5 attempts
Query::isNotNull('renewDate'),
Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew)
Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains)
]);
+43 -171
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Certificates\Adapter as CertificatesAdapter;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Mail;
@@ -11,7 +12,6 @@ use Appwrite\Template\Template;
use Appwrite\Utopia\Response\Model\Rule;
use Exception;
use Throwable;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
@@ -48,7 +48,11 @@ class Certificates extends Action
->inject('queueForEvents')
->inject('queueForFunctions')
->inject('log')
->callback(fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log));
->inject('certificates')
->callback(
fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates) =>
$this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates)
);
}
/**
@@ -58,11 +62,12 @@ class Certificates extends Action
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param Log $log
* @param CertificatesAdapter $certificates
* @return void
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void
public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates): void
{
$payload = $message->getPayload() ?? [];
@@ -76,7 +81,7 @@ class Certificates extends Action
$log->addTag('domain', $domain->get());
$this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck);
$this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates, $skipRenewCheck);
}
/**
@@ -85,24 +90,24 @@ class Certificates extends Action
* @param Mail $queueForMails
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param CertificatesAdapter $certificates
* @param bool $skipRenewCheck
* @return void
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void
private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void
{
/**
* 1. Read arguments and validate domain
* 2. Get main domain
* 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain)
* 4. Validate security email. Cannot be empty, required by LetsEncrypt
* 5. Validate renew date with certificate file, unless requested to skip by parameter
* 6. Issue a certificate using certbot CLI
* 7. Update 'log' attribute on certificate document with Certbot message
* 8. Create storage folder for certificate, if not ready already
* 9. Move certificates from Certbot location to our Storage
* 10. Create/Update our Storage with new Traefik config with new certificate paths
* 4. Validate renew date with certificate file, unless requested to skip by parameter
* 5. Issue a certificate using certbot CLI
* 6. Update 'log' attribute on certificate document with Certbot message
* 7. Create storage folder for certificate, if not ready already
* 8. Move certificates from Certbot location to our Storage
* 9. Create/Update our Storage with new Traefik config with new certificate paths
* 11. Read certificate file and update 'renewDate' on certificate document
* 12. Update 'issueDate' and 'attempts' on certificate
*
@@ -119,7 +124,7 @@ class Certificates extends Action
* 2. Save document to database
* 3. Update all domains documents with current certificate ID
*
* Note: Renewals are checked and scheduled from maintenence worker
* Note: Renewals are checked and scheduled from maintenance worker
*/
// Get current certificate
@@ -134,40 +139,27 @@ class Certificates extends Action
$success = false;
try {
// Email for alerts is required by LetsEncrypt
$email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'));
if (empty($email)) {
throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue an SSL certificate.');
}
// Validate domain and DNS records. Skip if job is forced
if (!$skipRenewCheck) {
$mainDomain = $this->getMainDomain();
$isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain;
$this->validateDomain($domain, $isMainDomain, $log);
// If certificate exists already, double-check expiry date. Skip if job is forced
if (!$certificates->isRenewRequired($domain->get(), $log)) {
throw new Exception('Renew isn\'t required.');
}
}
// If certificate exists already, double-check expiry date. Skip if job is forced
if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) {
throw new Exception('Renew isn\'t required.');
}
// Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate
$folder = ID::unique();
// Generate certificate files using Let's Encrypt
$letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email);
// Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates.
$certName = ID::unique();
$renewDate = $certificates->issueCertificate($certName, $domain->get());
// Command succeeded, store all data into document
$logs = 'Certificate successfully generated.';
$certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB
// Give certificates to Traefik
$this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData);
$certificate->setAttribute('logs', 'Certificate successfully generated.');
// Update certificate info stored in database
$certificate->setAttribute('renewDate', $this->getRenewDate($domain->get()));
$certificate->setAttribute('renewDate', $renewDate);
$certificate->setAttribute('attempts', 0);
$certificate->setAttribute('issueDate', DateTime::now());
$success = true;
@@ -181,7 +173,7 @@ class Certificates extends Action
$attempts = $certificate->getAttribute('attempts', 0) + 1;
$certificate->setAttribute('attempts', $attempts);
// Store cuttent time as renew date to ensure another attempt in next maintenance cycle
// Store current time as renew date to ensure another attempt in next maintenance cycle.
$certificate->setAttribute('renewDate', DateTime::now());
// Send email to security email
@@ -245,8 +237,8 @@ class Certificates extends Action
}
/**
* Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check:
* - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt)
* Internal domain validation functionality to prevent unnecessary attempts. We check:
* - Domain needs to be public and valid (prevents NFT domains that are not supported)
* - Domain must have proper DNS record
*
* @param Domain $domain Domain which we validate
@@ -292,136 +284,6 @@ class Certificates extends Action
}
}
/**
* Reads expiry date of certificate from file and decides if renewal is required or not.
*
* @param string $domain Domain for which we check certificate file
* @return bool True, if certificate needs to be renewed
* @throws Exception
*/
private function isRenewRequired(string $domain, Log $log): bool
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
if (\file_exists($certPath)) {
$validTo = null;
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? 0;
if (empty($validTo)) {
$log->addTag('certificateDomain', $domain);
throw new Exception('Unable to read certificate file (cert.pem).');
}
// LetsEncrypt allows renewal 30 days before expiry
$expiryInAdvance = (60 * 60 * 24 * 30);
if ($validTo - $expiryInAdvance > \time()) {
$log->addTag('certificateDomain', $domain);
$log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
return false;
}
}
return true;
}
/**
* LetsEncrypt communication to issue certificate (using certbot CLI)
*
* @param string $folder Folder into which certificates should be generated
* @param string $domain Domain to generate certificate for
* @return array Named array with keys 'stdout' and 'stderr', both string
* @throws Exception
*/
private function issueCertificate(string $folder, string $domain, string $email): array
{
$stdout = '';
$stderr = '';
$staging = (App::isProduction()) ? '' : ' --dry-run';
$exit = Console::execute("certbot certonly -v --webroot --noninteractive --agree-tos{$staging}"
. " --email " . $email
. " --cert-name " . $folder
. " -w " . APP_STORAGE_CERTIFICATES
. " -d {$domain}", '', $stdout, $stderr);
// Unexpected error, usually 5XX, API limits, ...
if ($exit !== 0) {
throw new Exception('Failed to issue a certificate with message: ' . $stderr);
}
return [
'stdout' => $stdout,
'stderr' => $stderr
];
}
/**
* Read new renew date from certificate file generated by Let's Encrypt
*
* @param string $domain Domain which certificate was generated for
* @return string
* @throws \Utopia\Database\Exception
*/
private function getRenewDate(string $domain): string
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? null;
$dt = (new \DateTime())->setTimestamp($validTo);
return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days
}
/**
* Method to take files from Let's Encrypt, and put it into Traefik.
*
* @param string $domain Domain which certificate was generated for
* @param string $folder Folder in which certificates were generated
* @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error
* @return void
* @throws Exception
*/
private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void
{
// Prepare folder in storage for domain
$path = APP_STORAGE_CERTIFICATES . '/' . $domain;
if (!\is_readable($path)) {
if (!\mkdir($path, 0755, true)) {
throw new Exception('Failed to create path for certificate.');
}
}
// Move generated files
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) {
throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) {
throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) {
throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) {
throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
$config = \implode(PHP_EOL, [
"tls:",
" certificates:",
" - certFile: /storage/certificates/{$domain}/fullchain.pem",
" keyFile: /storage/certificates/{$domain}/privkey.pem"
]);
// Save configuration into Traefik using our new cert files
if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) {
throw new Exception('Failed to save Traefik configuration.');
}
}
/**
* Method to make sure information about error is delivered to admnistrator.
*
@@ -477,8 +339,14 @@ class Certificates extends Action
*/
private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void
{
$rule = $dbForConsole->getDocument('rules', md5($domain));
// TODO: @christyjacob remove once we migrate the rules in 1.7.x
if (System::getEnv('_APP_RULES_FORMAT') === 'md5') {
$rule = $dbForConsole->getDocument('rules', md5($domain));
} else {
$rule = $dbForConsole->findOne('rules', [
Query::equal('domain', [$domain]),
]);
}
if (!$rule->isEmpty()) {
$rule->setAttribute('certificateId', $certificateId);
@@ -494,6 +362,10 @@ class Certificates extends Action
$project = $dbForConsole->getDocument('projects', $projectId);
if ($project->isEmpty()) {
return;
}
/** Trigger Webhook */
$ruleModel = new Rule();
$queueForEvents
+26 -15
View File
@@ -11,6 +11,7 @@ use Utopia\Database\Document;
use Utopia\Database\Exception as DatabaseException;
use Utopia\Database\Exception\Authorization;
use Utopia\Database\Exception\Conflict;
use Utopia\Database\Exception\NotFound;
use Utopia\Database\Exception\Restricted;
use Utopia\Database\Exception\Structure;
use Utopia\Database\Query;
@@ -251,23 +252,21 @@ class Databases extends Action
try {
try {
if ($status !== 'failed') {
if ($type === Database::VAR_RELATIONSHIP) {
if ($options['twoWay']) {
$relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']);
if ($relatedCollection->isEmpty()) {
throw new DatabaseException('Collection not found');
}
$relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']);
if ($type === Database::VAR_RELATIONSHIP) {
if ($options['twoWay']) {
$relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']);
if ($relatedCollection->isEmpty()) {
throw new DatabaseException('Collection not found');
}
if (!$dbForProject->deleteRelationship('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
$dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'stuck'));
throw new DatabaseException('Failed to delete Relationship');
}
} elseif (!$dbForProject->deleteAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
throw new DatabaseException('Failed to delete Attribute');
$relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']);
}
if (!$dbForProject->deleteRelationship('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
$dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'stuck'));
throw new DatabaseException('Failed to delete Relationship');
}
} elseif (!$dbForProject->deleteAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
throw new DatabaseException('Failed to delete Attribute');
}
$dbForProject->deleteDocument('attributes', $attribute->getId());
@@ -275,6 +274,18 @@ class Databases extends Action
if (!$relatedAttribute->isEmpty()) {
$dbForProject->deleteDocument('attributes', $relatedAttribute->getId());
}
} catch (NotFound $e) {
Console::error($e->getMessage());
$dbForProject->deleteDocument('attributes', $attribute->getId());
if (!$relatedAttribute->isEmpty()) {
$dbForProject->deleteDocument('attributes', $relatedAttribute->getId());
}
throw $e;
} catch (\Throwable $e) {
Console::error($e->getMessage());
+22 -28
View File
@@ -3,6 +3,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Auth\Auth;
use Appwrite\Certificates\Adapter as CertificatesAdapter;
use Appwrite\Extend\Exception;
use Executor\Executor;
use Throwable;
@@ -50,18 +51,22 @@ class Deletes extends Action
->inject('deviceForFunctions')
->inject('deviceForBuilds')
->inject('deviceForCache')
->inject('certificates')
->inject('abuseRetention')
->inject('executionRetention')
->inject('auditRetention')
->inject('log')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log));
->callback(
fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) =>
$this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log)
);
}
/**
* @throws Exception
* @throws Throwable
*/
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void
{
$payload = $message->getPayload() ?? [];
@@ -84,10 +89,10 @@ class Deletes extends Action
case DELETE_TYPE_DOCUMENT:
switch ($document->getCollection()) {
case DELETE_TYPE_PROJECTS:
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document);
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document);
break;
case DELETE_TYPE_FUNCTIONS:
$this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project);
$this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project);
break;
case DELETE_TYPE_DEPLOYMENTS:
$this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project);
@@ -102,7 +107,7 @@ class Deletes extends Action
$this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project);
break;
case DELETE_TYPE_RULES:
$this->deleteRule($dbForConsole, $document);
$this->deleteRule($dbForConsole, $document, $certificates);
break;
default:
Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
@@ -110,7 +115,7 @@ class Deletes extends Action
}
break;
case DELETE_TYPE_TEAM_PROJECTS:
$this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $document);
$this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $certificates, $document);
break;
case DELETE_TYPE_EXECUTIONS:
$this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
@@ -264,7 +269,7 @@ class Deletes extends Action
MESSAGE_TYPE_EMAIL => 'emailTotal',
MESSAGE_TYPE_SMS => 'smsTotal',
MESSAGE_TYPE_PUSH => 'pushTotal',
default => throw new Exception('Invalid target provider type'),
default => throw new Exception('Invalid target CertificatesAdapter type'),
};
$dbForProject->decreaseDocumentAttribute(
'topics',
@@ -442,7 +447,7 @@ class Deletes extends Action
* @throws Structure
* @throws Exception
*/
private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Document $document): void
private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void
{
$projects = $dbForConsole->find('projects', [
@@ -455,7 +460,7 @@ class Deletes extends Action
$deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId());
$deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId());
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project);
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $project);
$dbForConsole->deleteDocument('projects', $project->getId());
}
}
@@ -473,7 +478,7 @@ class Deletes extends Action
* @throws Authorization
* @throws DatabaseException
*/
private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void
private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Document $document): void
{
$projectInternalId = $document->getInternalId();
$projectId = $document->getId();
@@ -537,8 +542,8 @@ class Deletes extends Action
// Delete project and function rules
$this->deleteByGroup('rules', [
Query::equal('projectInternalId', [$projectInternalId])
], $dbForConsole, function (Document $document) use ($dbForConsole) {
$this->deleteRule($dbForConsole, $document);
], $dbForConsole, function (Document $document) use ($dbForConsole, $certificates) {
$this->deleteRule($dbForConsole, $document, $certificates);
});
// Delete Keys
@@ -746,7 +751,7 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void
private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
@@ -761,8 +766,8 @@ class Deletes extends Action
Query::equal('resourceType', ['function']),
Query::equal('resourceInternalId', [$functionInternalId]),
Query::equal('projectInternalId', [$project->getInternalId()])
], $dbForConsole, function (Document $document) use ($project, $dbForConsole) {
$this->deleteRule($dbForConsole, $document);
], $dbForConsole, function (Document $document) use ($project, $dbForConsole, $certificates) {
$this->deleteRule($dbForConsole, $document, $certificates);
});
/**
@@ -1048,21 +1053,10 @@ class Deletes extends Action
* @param Document $document rule document
* @return void
*/
private function deleteRule(Database $dbForConsole, Document $document): void
private function deleteRule(Database $dbForConsole, Document $document, CertificatesAdapter $certificates): void
{
$domain = $document->getAttribute('domain');
$directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
$checkTraversal = realpath($directory) === $directory;
if ($checkTraversal && is_dir($directory)) {
// Delete files, so Traefik is aware of change
array_map('unlink', glob($directory . '/*.*'));
rmdir($directory);
Console::info("Deleted certificate files for {$domain}");
} else {
Console::info("No certificate files found for {$domain}");
}
$certificates->deleteCertificate($domain);
// Delete certificate document, so Appwrite is aware of change
if (isset($document['certificateId'])) {
+62 -10
View File
@@ -15,10 +15,16 @@ use Utopia\System\System;
const METRIC_COLLECTION_LEVEL_STORAGE = 4;
const METRIC_DATABASE_LEVEL_STORAGE = 3;
const METRIC_PROJECT_LEVEL_STORAGE = 2;
const MAP_ARRAY_SIZE_MAX = 400000;
class UsageDump extends Action
{
protected array $stats = [];
protected array $map = [];
protected array $timeMap = [];
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
@@ -57,6 +63,11 @@ class UsageDump extends Action
throw new Exception('Missing payload');
}
if (count($this->map) >= self::MAP_ARRAY_SIZE_MAX) {
$this->handleMapSize();
//$this->map = [];
}
// TODO: rename both usage workers @shimonewman
foreach ($payload['stats'] ?? [] as $stats) {
$project = new Document($stats['project'] ?? []);
@@ -88,16 +99,7 @@ class UsageDump extends Action
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => System::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if(array_key_exists($id, $this->map)) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
@@ -113,6 +115,38 @@ class UsageDump extends Action
$value
);
}
} else {
$this->map[$id] = null;
$this->timeMap[time()] = $id;
try {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => System::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats',
$id,
'value',
$value
);
}
}
}
}
}
@@ -122,6 +156,24 @@ class UsageDump extends Action
}
}
private function handleMapSize(): void
{
$now = time();
$thirtyDaysAgo = $now - (60 * 60 * 24 * 30); //30 days
$filteredKeys = [];
foreach ($this->timeMap as $key => $value) {
if ($key >= $thirtyDaysAgo && $key < $now) {
$filteredKeys[$key] = $value;
}
}
foreach ($filteredKeys as $key => $value) {
unset($this->map[$value]);
unset($this->timeMap[$key]);
}
}
private function handleDatabaseStorage(string $key, Database $dbForProject): void
{
$data = explode('.', $key);
+8
View File
@@ -310,6 +310,14 @@ trait UsersBase
$this->assertNotEmpty($session['secret']);
$this->assertNotEmpty($session['expire']);
$this->assertEquals('server', $session['provider']);
$response = $this->client->call(Client::METHOD_GET, '/account', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-session' => $session['secret']
]);
$this->assertEquals(200, $response['headers']['status-code']);
}