From 6e09bcf6c64f6e46554aedd832a143e2170dcc81 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Wed, 27 Nov 2024 18:10:28 +0000 Subject: [PATCH 001/275] Bump appwrite version to 1.6.1-RC1 --- README-CN.md | 6 +++--- README.md | 6 +++--- app/init.php | 2 +- src/Appwrite/Migration/Migration.php | 1 + 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README-CN.md b/README-CN.md index 92a9bf9806..e43e9ac0d0 100644 --- a/README-CN.md +++ b/README-CN.md @@ -67,7 +67,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.6.1 + appwrite/appwrite:1.6.1-RC1 ``` ### Windows @@ -79,7 +79,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.6.1 + appwrite/appwrite:1.6.1-RC1 ``` #### PowerShell @@ -89,7 +89,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.6.1 + appwrite/appwrite:1.6.1-RC1 ``` 运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index ab57e65c4c..1d121e5407 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.6.1 + appwrite/appwrite:1.6.1-RC1 ``` ### Windows @@ -87,7 +87,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.6.1 + appwrite/appwrite:1.6.1-RC1 ``` #### PowerShell @@ -97,7 +97,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.6.1 + appwrite/appwrite:1.6.1-RC1 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/init.php b/app/init.php index 30ece74553..3cccabe78d 100644 --- a/app/init.php +++ b/app/init.php @@ -123,7 +123,7 @@ const APP_USER_ACCESS = 24 * 60 * 60; // 24 hours const APP_PROJECT_ACCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 4318; -const APP_VERSION_STABLE = '1.6.1'; +const APP_VERSION_STABLE = '1.6.1-RC1'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 19f69b1a4f..71b5706ed7 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -92,6 +92,7 @@ abstract class Migration '1.5.11' => 'V20', '1.6.0' => 'V21', '1.6.1' => 'V21', + '1.6.1-RC1' => 'V21', ]; /** From cce6222184c861aedd0966c9d380686623ba60d0 Mon Sep 17 00:00:00 2001 From: Gurjeet Singh Virdee <73753957+gurjeetsinghvirdee@users.noreply.github.com> Date: Fri, 20 Dec 2024 22:24:39 +0530 Subject: [PATCH 002/275] docs: update CLI commands in documentation --- docs/sdks/cli/GETTING_STARTED.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/sdks/cli/GETTING_STARTED.md b/docs/sdks/cli/GETTING_STARTED.md index 564fb4d5f9..a7c3174516 100644 --- a/docs/sdks/cli/GETTING_STARTED.md +++ b/docs/sdks/cli/GETTING_STARTED.md @@ -59,7 +59,7 @@ My Awesome Function You can now deploy this function using ```sh -$ appwrite deploy function +$ appwrite push function ? Which functions would you like to deploy? My Awesome Function (61d1a4c81dfcd95bc834) ℹ Info Deploying function My Awesome Function ( 61d1a4c81dfcd95bc834 ) @@ -73,7 +73,7 @@ Your function has now been deployed on your Appwrite server! As soon as the buil Similarly, you can deploy all your collections to your Appwrite server using ```sh -appwrite deploy collections +appwrite push collections ``` > ### Note @@ -98,7 +98,7 @@ $ appwrite users list To create a document you can use the following command ```sh -$ appwrite database createDocument --collectionId --documentId 'unique()' --data '{ "Name": "Iron Man" }' --permissions 'read("any")' 'read("team:abc")' +$ appwrite databases create-document --database-id= --collection-id= --document-id="unique()" --data '{"name": "Walter O Brein"}' ``` ### Some Gotchas @@ -140,4 +140,4 @@ The Appwrite CLI can also work in a CI environment. The initialisation of the CL ```sh appwrite client --endpoint http://localhost/v1 --projectId --key -``` \ No newline at end of file +``` From c451074278cf86c40ad69f4a4dc4c126995ad5a3 Mon Sep 17 00:00:00 2001 From: Gurjeet Singh Virdee <73753957+gurjeetsinghvirdee@users.noreply.github.com> Date: Mon, 6 Jan 2025 01:29:06 +0530 Subject: [PATCH 003/275] restored the permissions term --- docs/sdks/cli/GETTING_STARTED.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sdks/cli/GETTING_STARTED.md b/docs/sdks/cli/GETTING_STARTED.md index a7c3174516..fbd7dca96b 100644 --- a/docs/sdks/cli/GETTING_STARTED.md +++ b/docs/sdks/cli/GETTING_STARTED.md @@ -98,7 +98,7 @@ $ appwrite users list To create a document you can use the following command ```sh -$ appwrite databases create-document --database-id= --collection-id= --document-id="unique()" --data '{"name": "Walter O Brein"}' +$ appwrite databases create-document --database-id= --collection-id= --document-id="unique()" --data '{"name": "Walter O Brein"} --permissions 'read("any")' 'read("team:abc")' ``` ### Some Gotchas From a39341ada8d4494fc0be3789ad286b9a9a50673f Mon Sep 17 00:00:00 2001 From: Gurjeet Singh Virdee <73753957+gurjeetsinghvirdee@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:48:54 +0530 Subject: [PATCH 004/275] fix: add missing single quote --- docs/sdks/cli/GETTING_STARTED.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sdks/cli/GETTING_STARTED.md b/docs/sdks/cli/GETTING_STARTED.md index fbd7dca96b..3f3c0ca56c 100644 --- a/docs/sdks/cli/GETTING_STARTED.md +++ b/docs/sdks/cli/GETTING_STARTED.md @@ -98,7 +98,7 @@ $ appwrite users list To create a document you can use the following command ```sh -$ appwrite databases create-document --database-id= --collection-id= --document-id="unique()" --data '{"name": "Walter O Brein"} --permissions 'read("any")' 'read("team:abc")' +$ appwrite databases create-document --database-id= --collection-id= --document-id="unique()" --data '{"name": "Walter O Brein"}' --permissions 'read("any")' 'read("team:abc")' ``` ### Some Gotchas From 8508e6009bf062038845daeeb68fb4a23436811a Mon Sep 17 00:00:00 2001 From: Gurjeet Singh Virdee <73753957+gurjeetsinghvirdee@users.noreply.github.com> Date: Fri, 24 Jan 2025 19:13:56 +0530 Subject: [PATCH 005/275] refactor: cleaned up syntax by removing unnecessary '=' --- docs/sdks/cli/GETTING_STARTED.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sdks/cli/GETTING_STARTED.md b/docs/sdks/cli/GETTING_STARTED.md index 3f3c0ca56c..1cadb1bbda 100644 --- a/docs/sdks/cli/GETTING_STARTED.md +++ b/docs/sdks/cli/GETTING_STARTED.md @@ -98,7 +98,7 @@ $ appwrite users list To create a document you can use the following command ```sh -$ appwrite databases create-document --database-id= --collection-id= --document-id="unique()" --data '{"name": "Walter O Brein"}' --permissions 'read("any")' 'read("team:abc")' +$ appwrite databases create-document --database-id --collection-id --document-id "unique()" --data '{"name": "Walter O Brein"}' --permissions 'read("any")' 'read("team:abc")' ``` ### Some Gotchas From 2de4e7166230be73007cf2fa7daddf527f31e7c3 Mon Sep 17 00:00:00 2001 From: Laura Du Ry <123562732+LauraDuRy@users.noreply.github.com> Date: Wed, 29 Jan 2025 09:58:04 +0100 Subject: [PATCH 006/275] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f72133baa6..847c67e965 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -> Appwrite Init has concluded! You can check out all the latest announcements [on our Init website](https://appwrite.io/init) :rocket: +> [Get started with Appwrite](https://cloud.appwrite.io/)

From 507c8bd3ec0a11780a84a9050a9458806d5f8157 Mon Sep 17 00:00:00 2001 From: Laura Du Ry <123562732+LauraDuRy@users.noreply.github.com> Date: Wed, 29 Jan 2025 10:02:40 +0100 Subject: [PATCH 007/275] Update README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 847c67e965..2b5d1bcc6c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -> [Get started with Appwrite](https://cloud.appwrite.io/) +> [Get started with Appwrite](https://apwr.dev/appcloud)

@@ -24,8 +24,6 @@ English | [简体中文](README-CN.md) -[**Announcing Appwrite Cloud Public Beta! Sign up today!**](https://cloud.appwrite.io) - Appwrite is an end-to-end backend server for Web, Mobile, Native, or Backend apps packaged as a set of Docker microservices. Appwrite abstracts the complexity and repetitiveness required to build a modern backend API from scratch and allows you to build secure apps faster. Using Appwrite, you can easily integrate your app with user authentication and multiple sign-in methods, a database for storing and querying users and team data, storage and file management, image manipulation, Cloud Functions, and [more services](https://appwrite.io/docs). From 891c1c4093ae762a003e68c50343e106f8565cab Mon Sep 17 00:00:00 2001 From: Laura Du Ry <123562732+LauraDuRy@users.noreply.github.com> Date: Wed, 29 Jan 2025 10:04:15 +0100 Subject: [PATCH 008/275] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b5d1bcc6c..9bb1aa5c58 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ English | [简体中文](README-CN.md) Appwrite is an end-to-end backend server for Web, Mobile, Native, or Backend apps packaged as a set of Docker microservices. Appwrite abstracts the complexity and repetitiveness required to build a modern backend API from scratch and allows you to build secure apps faster. -Using Appwrite, you can easily integrate your app with user authentication and multiple sign-in methods, a database for storing and querying users and team data, storage and file management, image manipulation, Cloud Functions, and [more services](https://appwrite.io/docs). +Using Appwrite, you can easily integrate your app with user authentication and multiple sign-in methods, a database for storing and querying users and team data, storage and file management, image manipulation, Cloud Functions, messaging, and [more services](https://appwrite.io/docs).


From df017bccb29bf09e9f645bd5004547ee51da307f Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Fri, 31 Jan 2025 22:00:32 +0000 Subject: [PATCH 009/275] chore: bump appwrite version to 1.6.1 --- README-CN.md | 6 +++--- README.md | 8 ++++---- app/init.php | 2 +- src/Appwrite/Migration/Migration.php | 1 - 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/README-CN.md b/README-CN.md index e43e9ac0d0..92a9bf9806 100644 --- a/README-CN.md +++ b/README-CN.md @@ -67,7 +67,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.6.1-RC1 + appwrite/appwrite:1.6.1 ``` ### Windows @@ -79,7 +79,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.6.1-RC1 + appwrite/appwrite:1.6.1 ``` #### PowerShell @@ -89,7 +89,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.6.1-RC1 + appwrite/appwrite:1.6.1 ``` 运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index 1d121e5407..a433da3c84 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Table of Contents: - [Upgrade from an Older Version](#upgrade-from-an-older-version) - [One-Click Setups](#one-click-setups) - [Getting Started](#getting-started) - - [Services](#services) + - [Products](#products) - [SDKs](#sdks) - [Client](#client) - [Server](#server) @@ -75,7 +75,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.6.1-RC1 + appwrite/appwrite:1.6.1 ``` ### Windows @@ -87,7 +87,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.6.1-RC1 + appwrite/appwrite:1.6.1 ``` #### PowerShell @@ -97,7 +97,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.6.1-RC1 + appwrite/appwrite:1.6.1 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/init.php b/app/init.php index 3cccabe78d..30ece74553 100644 --- a/app/init.php +++ b/app/init.php @@ -123,7 +123,7 @@ const APP_USER_ACCESS = 24 * 60 * 60; // 24 hours const APP_PROJECT_ACCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 4318; -const APP_VERSION_STABLE = '1.6.1-RC1'; +const APP_VERSION_STABLE = '1.6.1'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 71b5706ed7..19f69b1a4f 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -92,7 +92,6 @@ abstract class Migration '1.5.11' => 'V20', '1.6.0' => 'V21', '1.6.1' => 'V21', - '1.6.1-RC1' => 'V21', ]; /** From 0547dbc4543072f847c8a52a15ee688848e71031 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Mon, 3 Feb 2025 22:34:08 +0000 Subject: [PATCH 010/275] fix: use github var instead of secret for DOCKERHUB_USERNAME DOCKERHUB_USERNAME was moved to be a variable since it doesn't need to be secret. --- .github/workflows/publish.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f914e662d3..0ed82dd853 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -26,7 +26,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4475a49809..712d30dac0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker From 375f6432142f462fe8a052c1ea7138dea4755fed Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 03:29:32 +0000 Subject: [PATCH 011/275] refactor: triggering realtime events with queueForRealtime --- app/controllers/api/functions.php | 52 ++++++--------- app/worker.php | 5 ++ src/Appwrite/Event/Event.php | 23 +++++++ src/Appwrite/Event/Realtime.php | 2 +- src/Appwrite/Platform/Workers/Databases.php | 74 +++++++++------------ 5 files changed, 80 insertions(+), 76 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 14255ef7a4..0f9f5211f4 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -6,13 +6,13 @@ use Appwrite\Event\Build; use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; use Appwrite\Event\Validator\FunctionEvent; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\Validator\Headers; use Appwrite\Functions\Validator\RuntimeSpecification; -use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Platform\Tasks\ScheduleExecutions; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -194,9 +194,10 @@ App::post('/v1/functions') ->inject('user') ->inject('queueForEvents') ->inject('queueForBuilds') + ->inject('queueForRealtime') ->inject('dbForPlatform') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $timelimit, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $timelimit, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Realtime $queueForRealtime, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; // Temporary abuse check @@ -386,18 +387,18 @@ App::post('/v1/functions') ])) ); - /** Trigger Webhook */ $ruleModel = new Rule(); $ruleCreate = $queueForEvents - ->setClass(Event::WEBHOOK_CLASS_NAME) - ->setQueue(Event::WEBHOOK_QUEUE_NAME); + ->setProject($project) + ->setEvent('rules.[ruleId].create') + ->setParam('ruleId', $rule->getId()) + ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); + /** Trigger Webhook */ $ruleCreate - ->setProject($project) - ->setEvent('rules.[ruleId].create') - ->setParam('ruleId', $rule->getId()) - ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))) + ->setClass(Event::WEBHOOK_CLASS_NAME) + ->setQueue(Event::WEBHOOK_QUEUE_NAME) ->trigger(); /** Trigger Functions */ @@ -406,31 +407,16 @@ App::post('/v1/functions') ->setQueue(Event::FUNCTIONS_QUEUE_NAME) ->trigger(); - /** Trigger realtime event */ - $allEvents = Event::generateEvents('rules.[ruleId].create', [ - 'ruleId' => $rule->getId(), - ]); + /** Trigger Realtime Events */ + $queueForRealtime + ->from($ruleCreate) + ->setProjectId('console') + ->trigger(); - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $rule, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $rule->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - Realtime::send( - projectId: $project->getId(), - payload: $rule->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + $queueForRealtime + ->from($ruleCreate) + ->setProjectId($project->getId()) + ->trigger(); } $queueForEvents->setParam('functionId', $function->getId()); diff --git a/app/worker.php b/app/worker.php index 605474e9f1..ad6bf475f9 100644 --- a/app/worker.php +++ b/app/worker.php @@ -13,6 +13,7 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; use Appwrite\Event\StatsUsageDump; /** remove */ @@ -317,6 +318,10 @@ Server::setResource('queueForFunctions', function (Publisher $publisher) { return new Func($publisher); }, ['publisher']); +Server::setResource('queueForRealtime', function () { + return new Realtime(); +}, []); + Server::setResource('queueForCertificates', function (Publisher $publisher) { return new Certificate($publisher); }, ['publisher']); diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 0edffdf4dc..8085a836a8 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -66,6 +66,7 @@ class Event protected ?Document $project = null; protected ?Document $user = null; protected ?string $userId = null; + protected ?string $projectId = null; protected bool $paused = false; /** @@ -151,6 +152,18 @@ class Event return $this; } + /** + * Set projectId for this event. + * + * @param string $projectId + * @return self + */ + public function setProjectId(string $projectId): self + { + $this->projectId = $projectId; + return $this; + } + /** * Get project for this event. * @@ -161,6 +174,16 @@ class Event return $this->project; } + /** + * Get projectId for this event. + * + * @return ?string + */ + public function getProjectId(): ?string + { + return $this->projectId; + } + /** * Set user for this event. * diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index f4f00b59d4..8c302bbabf 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -54,7 +54,7 @@ class Realtime extends Event ); RealtimeAdapter::send( - projectId: $target['projectId'] ?? $this->getProject()->getId(), + projectId: $this->getProjectId() ?? $target['projectId'] ?? $this->getProject()->getId(), payload: $this->getRealtimePayload(), events: $allEvents, channels: $target['channels'], diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 441b09b4cc..50a3fa52f3 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -3,7 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Event; -use Appwrite\Messaging\Adapter\Realtime; +use Appwrite\Event\Realtime; use Exception; use Utopia\CLI\Console; use Utopia\Database\Database; @@ -38,7 +38,8 @@ class Databases extends Action ->inject('dbForPlatform') ->inject('dbForProject') ->inject('log') - ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log) => $this->action($message, $project, $dbForPlatform, $dbForProject, $log)); + ->inject('queueForRealtime') + ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log, Realtime $queueForRealtime) => $this->action($message, $project, $dbForPlatform, $dbForProject, $log, $queueForRealtime)); } /** @@ -47,10 +48,11 @@ class Databases extends Action * @param Database $dbForPlatform * @param Database $dbForProject * @param Log $log + * @param Realtime $queueForRealtime * @return void * @throws \Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log, Realtime $queueForRealtime): void { $payload = $message->getPayload() ?? []; @@ -75,10 +77,10 @@ class Databases extends Action match (\strval($type)) { DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject), DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject), - DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject), - DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject), - DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject), - DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject), + DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), + DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), + DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), + DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), default => throw new \Exception('No database operation for type: ' . \strval($type)), }; } @@ -90,13 +92,14 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject + * @param Realtime $queueForRealtime * @return void * @throws Authorization * @throws Conflict * @throws \Exception * @throws \Throwable */ - private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject): void + private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime): void { if ($collection->isEmpty()) { throw new Exception('Missing collection'); @@ -106,12 +109,6 @@ class Databases extends Action } $projectId = $project->getId(); - - $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update', [ - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'attributeId' => $attribute->getId() - ]); /** * TODO @christyjacob4 verify if this is still the case * Fetch attribute from the database, since with Resque float values are loosing informations. @@ -200,7 +197,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $attribute, $project, $projectId, $events); + $this->trigger($database, $collection, $attribute, $project, $queueForRealtime); if (! $relatedCollection->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); @@ -217,13 +214,14 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject + * @param Realtime $queueForRealtime * @return void * @throws Authorization * @throws Conflict * @throws \Exception * @throws \Throwable **/ - private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject): void + private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime): void { if ($collection->isEmpty()) { throw new Exception('Missing collection'); @@ -312,7 +310,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $attribute, $project, $projectId, $events); + $this->trigger($database, $collection, $attribute, $project, $queueForRealtime); } // The underlying database removes/rebuilds indexes when attribute is removed @@ -358,7 +356,7 @@ class Databases extends Action } if ($exists) { // Delete the duplicate if created, else update in db - $this->deleteIndex($database, $collection, $index, $project, $dbForPlatform, $dbForProject); + $this->deleteIndex($database, $collection, $index, $project, $dbForPlatform, $dbForProject, $queueForRealtime); } else { $dbForProject->updateDocument('indexes', $index->getId(), $index); } @@ -381,6 +379,7 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject + * @param Realtime $queueForRealtime * @return void * @throws Authorization * @throws Conflict @@ -388,7 +387,7 @@ class Databases extends Action * @throws DatabaseException * @throws \Throwable */ - private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject): void + private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime): void { if ($collection->isEmpty()) { throw new Exception('Missing collection'); @@ -430,7 +429,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $index, $project, $projectId, $events); + $this->trigger($database, $collection, $index, $project, $queueForRealtime); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); } } @@ -442,6 +441,7 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject + * @param Realtime $queueForRealtime * @return void * @throws Authorization * @throws Conflict @@ -449,7 +449,7 @@ class Databases extends Action * @throws DatabaseException * @throws \Throwable */ - private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject): void + private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime): void { if ($collection->isEmpty()) { throw new Exception('Missing collection'); @@ -490,7 +490,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $index, $project, $projectId, $events); + $this->trigger($database, $collection, $index, $project, $queueForRealtime); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId()); } } @@ -617,26 +617,16 @@ class Databases extends Action Document $collection, Document $attribute, Document $project, - string $projectId, - array $events + Realtime $queueForRealtime ): void { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $attribute, - project: $project, - ); - Realtime::send( - projectId: 'console', - payload: $attribute->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $queueForRealtime + ->setProject($project) + ->setProjectId('console') + ->setEvent('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') + ->setParam('databaseId', $database->getId()) + ->setParam('collectionId', $collection->getId()) + ->setParam('attributeId', $attribute->getId()) + ->setPayload($attribute->getArrayCopy()) + ->trigger(); } } From 2c4c42de054ad6388f53562c29621b913a58864d Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 03:53:18 +0000 Subject: [PATCH 012/275] docs: fix database worker --- src/Appwrite/Platform/Workers/Databases.php | 40 ++++++++------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 50a3fa52f3..6db88a618f 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Event\Event; use Appwrite\Event\Realtime; use Exception; use Utopia\CLI\Console; @@ -57,7 +56,7 @@ class Databases extends Action $payload = $message->getPayload() ?? []; if (empty($payload)) { - throw new \Exception('Missing payload'); + throw new Exception('Missing payload'); } $type = $payload['type']; @@ -81,7 +80,7 @@ class Databases extends Action DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), - default => throw new \Exception('No database operation for type: ' . \strval($type)), + default => throw new Exception('No database operation for type: ' . \strval($type)), }; } @@ -166,7 +165,7 @@ class Databases extends Action break; default: if (!$dbForProject->createAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters)) { - throw new \Exception('Failed to create Attribute'); + throw new Exception('Failed to create Attribute'); } } @@ -231,15 +230,8 @@ class Databases extends Action } $projectId = $project->getId(); - - $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [ - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'attributeId' => $attribute->getId() - ]); $collectionId = $collection->getId(); $key = $attribute->getAttribute('key', ''); - $status = $attribute->getAttribute('status', ''); $type = $attribute->getAttribute('type', ''); $project = $dbForPlatform->getDocument('projects', $projectId); $options = $attribute->getAttribute('options', []); @@ -397,12 +389,6 @@ class Databases extends Action } $projectId = $project->getId(); - - $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [ - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'indexId' => $index->getId() - ]); $collectionId = $collection->getId(); $key = $index->getAttribute('key', ''); $type = $index->getAttribute('type', ''); @@ -459,12 +445,6 @@ class Databases extends Action } $projectId = $project->getId(); - - $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [ - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'indexId' => $index->getId() - ]); $key = $index->getAttribute('key'); $status = $index->getAttribute('status', ''); $project = $dbForPlatform->getDocument('projects', $projectId); @@ -568,14 +548,14 @@ class Databases extends Action /** - * @param string $collection collectionID + * @param string $collectionId * @param array $queries * @param Database $database * @param callable|null $callback * @return void * @throws Exception */ - protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void + protected function deleteByGroup(string $collectionId, array $queries, Database $database, callable $callback = null): void { $count = 0; $chunk = 0; @@ -587,7 +567,7 @@ class Databases extends Action while ($sum === $limit) { $chunk++; - $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + $results = $database->find($collectionId, \array_merge([Query::limit($limit)], $queries)); $sum = count($results); @@ -612,6 +592,14 @@ class Databases extends Action Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } + /** + * @param Document $database + * @param Document $collection + * @param Document $attribute + * @param Document $project + * @param Realtime $queueForRealtime + * @return void + */ protected function trigger( Document $database, Document $collection, From 47fbb777ed2893e8230316d0386febed76e407a4 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 04:03:02 +0000 Subject: [PATCH 013/275] chore: refactor migrations realtime queue --- src/Appwrite/Platform/Workers/Migrations.php | 66 +++++++++----------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index cd567f6fa3..50d6002d28 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -3,8 +3,7 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; -use Appwrite\Event\Event; -use Appwrite\Messaging\Adapter\Realtime; +use Appwrite\Event\Realtime; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -54,13 +53,14 @@ class Migrations extends Action ->inject('dbForProject') ->inject('dbForPlatform') ->inject('logError') - ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError)); + ->inject('queueForRealtime') + ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError, $queueForRealtime)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime): void { $payload = $message->getPayload() ?? []; @@ -87,7 +87,7 @@ class Migrations extends Action return; } - $this->processMigration($migration); + $this->processMigration($migration, $queueForRealtime); } /** @@ -155,34 +155,24 @@ class Migrations extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function updateMigrationDocument(Document $migration, Document $project): Document + protected function updateMigrationDocument(Document $migration, Document $project, Realtime $queueForRealtime): Document { - /** Trigger Realtime */ - $allEvents = Event::generateEvents('migrations.[migrationId].update', [ - 'migrationId' => $migration->getId(), - ]); + /** Trigger Realtime Events */ + $queueForRealtime + ->setProject($project) + ->setProjectId('console') + ->setEvent('migrations.[migrationId].update') + ->setParam('migrationId', $migration->getId()) + ->setPayload($migration->getArrayCopy()) + ->trigger(); - $target = Realtime::fromPayload( - event: $allEvents[0], - payload: $migration, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $migration->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - ); - - Realtime::send( - projectId: $project->getId(), - payload: $migration->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - ); + $queueForRealtime + ->setProject($project) + ->setProjectId($project->getId()) + ->setEvent('migrations.[migrationId].update') + ->setParam('migrationId', $migration->getId()) + ->setPayload($migration->getArrayCopy()) + ->trigger(); return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration); } @@ -241,7 +231,7 @@ class Migrations extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function processMigration(Document $migration): void + protected function processMigration(Document $migration, Realtime $queueForRealtime): void { $project = $this->project; $projectDocument = $this->dbForPlatform->getDocument('projects', $project->getId()); @@ -265,7 +255,7 @@ class Migrations extends Action $migration->setAttribute('stage', 'processing'); $migration->setAttribute('status', 'processing'); - $this->updateMigrationDocument($migration, $projectDocument); + $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); $source = $this->processSource($migration); $destination = $this->processDestination($migration, $tempAPIKey); @@ -279,14 +269,14 @@ class Migrations extends Action /** Start Transfer */ $migration->setAttribute('stage', 'migrating'); - $this->updateMigrationDocument($migration, $projectDocument); + $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); $transfer->run( $migration->getAttribute('resources'), - function () use ($migration, $transfer, $projectDocument) { + function () use ($migration, $transfer, $projectDocument, $queueForRealtime) { $migration->setAttribute('resourceData', json_encode($transfer->getCache())); $migration->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); - $this->updateMigrationDocument($migration, $projectDocument); + $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); }, $migration->getAttribute('resourceId'), $migration->getAttribute('resourceType') @@ -323,7 +313,7 @@ class Migrations extends Action } $migration->setAttribute('errors', $errorMessages); - $this->updateMigrationDocument($migration, $projectDocument); + $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); return; } @@ -364,7 +354,7 @@ class Migrations extends Action $migration->setAttribute('errors', $errorMessages); } } finally { - $this->updateMigrationDocument($migration, $projectDocument); + $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); if ($migration->getAttribute('status', '') === 'failed') { Console::error('Migration('.$migration->getInternalId().':'.$migration->getId().') failed, Project('.$this->project->getInternalId().':'.$this->project->getId().')'); From bcdae7f4e37ff7d348d54f39f2991676a89e77c4 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 04:21:36 +0000 Subject: [PATCH 014/275] chore: refactor certificate realtime, fix database --- .../Platform/Workers/Certificates.php | 76 ++++++++----------- src/Appwrite/Platform/Workers/Databases.php | 23 +++--- 2 files changed, 46 insertions(+), 53 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 7e220b2734..c7f345f77a 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -6,7 +6,7 @@ use Appwrite\Certificates\Adapter as CertificatesAdapter; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; -use Appwrite\Messaging\Adapter\Realtime; +use Appwrite\Event\Realtime; use Appwrite\Network\Validator\CNAME; use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; @@ -47,11 +47,12 @@ class Certificates extends Action ->inject('queueForMails') ->inject('queueForEvents') ->inject('queueForFunctions') + ->inject('queueForRealtime') ->inject('log') ->inject('certificates') ->callback( - fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates) => - $this->action($message, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates) + fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates) => + $this->action($message, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $queueForRealtime, $log, $certificates) ); } @@ -61,13 +62,14 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @param Log $log * @param CertificatesAdapter $certificates * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates): void + public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates): void { $payload = $message->getPayload() ?? []; @@ -81,7 +83,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates, $skipRenewCheck); + $this->execute($domain, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck); } /** @@ -90,13 +92,14 @@ class Certificates extends Action * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @param CertificatesAdapter $certificates * @param bool $skipRenewCheck * @return void * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain @@ -186,7 +189,7 @@ class Certificates extends Action $certificate->setAttribute('updated', DateTime::now()); // Save all changes we made to certificate document into database - $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForFunctions); + $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForFunctions, $queueForRealtime); } } @@ -199,13 +202,14 @@ class Certificates extends Action * @param Database $dbForPlatform Database connection for console * @param Event $queueForEvents * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @return void * @throws \Utopia\Database\Exception * @throws Authorization * @throws Conflict * @throws Structure */ - private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions): void + private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime): void { // Check if update or insert required $certificateDocument = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain])]); @@ -219,7 +223,7 @@ class Certificates extends Action } $certificateId = $certificate->getId(); - $this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForFunctions); + $this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForFunctions, $queueForRealtime); } /** @@ -338,7 +342,7 @@ class Certificates extends Action * * @return void */ - private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions): void + private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime): void { // TODO: @christyjacob remove once we migrate the rules in 1.7.x if (System::getEnv('_APP_RULES_FORMAT') === 'md5') { @@ -367,50 +371,34 @@ class Certificates extends Action return; } - /** Trigger Webhook */ $ruleModel = new Rule(); + $queueForEvents + ->setProject($project) + ->setEvent('rules.[ruleId].update') + ->setParam('ruleId', $rule->getId()) + ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); + + /** Trigger Webhook */ $queueForEvents ->setQueue(Event::WEBHOOK_QUEUE_NAME) ->setClass(Event::WEBHOOK_CLASS_NAME) - ->setProject($project) - ->setEvent('rules.[ruleId].update') - ->setParam('ruleId', $rule->getId()) - ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))) ->trigger(); - /** Trigger Functions */ $queueForFunctions - ->setProject($project) - ->setEvent('rules.[ruleId].update') - ->setParam('ruleId', $rule->getId()) - ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))) + ->from($queueForEvents) ->trigger(); - /** Trigger realtime event */ - $allEvents = Event::generateEvents('rules.[ruleId].update', [ - 'ruleId' => $rule->getId(), - ]); - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $rule, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $rule->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - Realtime::send( - projectId: $project->getId(), - payload: $rule->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + /** Trigger Realtime Events */ + $queueForRealtime + ->from($queueForEvents) + ->setProjectId('console') + ->trigger(); + + $queueForRealtime + ->from($queueForEvents) + ->setProjectId($project->getId()) + ->trigger(); } } } diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 6db88a618f..b617da3e33 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -36,9 +36,9 @@ class Databases extends Action ->inject('project') ->inject('dbForPlatform') ->inject('dbForProject') - ->inject('log') ->inject('queueForRealtime') - ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log, Realtime $queueForRealtime) => $this->action($message, $project, $dbForPlatform, $dbForProject, $log, $queueForRealtime)); + ->inject('log') + ->callback(fn (Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime, Log $log) => $this->action($message, $project, $dbForPlatform, $dbForProject, $queueForRealtime, $log)); } /** @@ -46,12 +46,12 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject - * @param Log $log * @param Realtime $queueForRealtime + * @param Log $log * @return void * @throws \Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log, Realtime $queueForRealtime): void + public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime, Log $log): void { $payload = $message->getPayload() ?? []; @@ -108,6 +108,7 @@ class Databases extends Action } $projectId = $project->getId(); + $event = "databases.[databaseId].collections.[collectionId].attributes.[attributeId].update"; /** * TODO @christyjacob4 verify if this is still the case * Fetch attribute from the database, since with Resque float values are loosing informations. @@ -196,7 +197,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $attribute, $project, $queueForRealtime); + $this->trigger($database, $collection, $attribute, $project, $event, $queueForRealtime); if (! $relatedCollection->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); @@ -230,6 +231,7 @@ class Databases extends Action } $projectId = $project->getId(); + $event = 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete'; $collectionId = $collection->getId(); $key = $attribute->getAttribute('key', ''); $type = $attribute->getAttribute('type', ''); @@ -302,7 +304,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $attribute, $project, $queueForRealtime); + $this->trigger($database, $collection, $attribute, $project, $event, $queueForRealtime); } // The underlying database removes/rebuilds indexes when attribute is removed @@ -389,6 +391,7 @@ class Databases extends Action } $projectId = $project->getId(); + $event = 'databases.[databaseId].collections.[collectionId].indexes.[indexId].update'; $collectionId = $collection->getId(); $key = $index->getAttribute('key', ''); $type = $index->getAttribute('type', ''); @@ -415,7 +418,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $index, $project, $queueForRealtime); + $this->trigger($database, $collection, $index, $project, $event, $queueForRealtime); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); } } @@ -445,6 +448,7 @@ class Databases extends Action } $projectId = $project->getId(); + $event = 'databases.[databaseId].collections.[collectionId].indexes.[indexId].delete'; $key = $index->getAttribute('key'); $status = $index->getAttribute('status', ''); $project = $dbForPlatform->getDocument('projects', $projectId); @@ -470,7 +474,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $index, $project, $queueForRealtime); + $this->trigger($database, $collection, $index, $project, $event, $queueForRealtime); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId()); } } @@ -605,12 +609,13 @@ class Databases extends Action Document $collection, Document $attribute, Document $project, + string $event, Realtime $queueForRealtime ): void { $queueForRealtime ->setProject($project) ->setProjectId('console') - ->setEvent('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') + ->setEvent($event) ->setParam('databaseId', $database->getId()) ->setParam('collectionId', $collection->getId()) ->setParam('attributeId', $attribute->getId()) From f666e18154ecf2b9b0e51dd3c64b2fe400cd0c80 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 05:00:45 +0000 Subject: [PATCH 015/275] chore: fix databases worker realtime queueing --- src/Appwrite/Platform/Workers/Databases.php | 30 ++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index b617da3e33..b0c6a9df73 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -197,7 +197,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $attribute, $project, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, $attribute, null, $attribute, $event, $queueForRealtime); if (! $relatedCollection->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); @@ -304,7 +304,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $attribute, $project, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, $attribute, null, $attribute, $event, $queueForRealtime); } // The underlying database removes/rebuilds indexes when attribute is removed @@ -418,7 +418,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $index, $project, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, null, $index, $index, $event, $queueForRealtime); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); } } @@ -474,7 +474,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $index, $project, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, null, $index, $index, $event, $queueForRealtime); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId()); } } @@ -599,16 +599,20 @@ class Databases extends Action /** * @param Document $database * @param Document $collection - * @param Document $attribute * @param Document $project + * @param Document|null $attribute + * @param Document|null $index + * @param Document $payload * @param Realtime $queueForRealtime * @return void */ protected function trigger( Document $database, Document $collection, - Document $attribute, Document $project, + Document|null $attribute = null, + Document|null $index = null, + Document $payload, string $event, Realtime $queueForRealtime ): void { @@ -617,9 +621,17 @@ class Databases extends Action ->setProjectId('console') ->setEvent($event) ->setParam('databaseId', $database->getId()) - ->setParam('collectionId', $collection->getId()) - ->setParam('attributeId', $attribute->getId()) - ->setPayload($attribute->getArrayCopy()) + ->setParam('collectionId', $collection->getId()); + + if ($attribute !== null) { + $queueForRealtime->setParam('attributeId', $attribute->getId()); + } + if ($index !== null) { + $queueForRealtime->setParam('indexId', $index->getId()); + } + + $queueForRealtime + ->setPayload($payload->getArrayCopy()) ->trigger(); } } From 532160705ffbde2ec6f008546d66ddcbe5e8048c Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 05:50:30 +0000 Subject: [PATCH 016/275] chore: refactor realtime queueing in functions worker --- src/Appwrite/Platform/Workers/Functions.php | 58 +++++++++------------ 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index 0a7c39c02f..f485db8648 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -5,8 +5,8 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; -use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Utopia\Response\Model\Execution; use Exception; use Executor\Executor; @@ -45,14 +45,15 @@ class Functions extends Action ->inject('message') ->inject('dbForProject') ->inject('queueForFunctions') + ->inject('queueForRealtime') ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('log') ->inject('isResourceBlocked') - ->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked)); + ->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked)); } - public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void + public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void { $payload = $message->getPayload() ?? []; @@ -137,6 +138,7 @@ class Functions extends Action log: $log, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, + queueForRealtime: $queueForRealtime, queueForStatsUsage: $queueForStatsUsage, queueForEvents: $queueForEvents, project: $project, @@ -177,6 +179,7 @@ class Functions extends Action log: $log, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, + queueForRealtime: $queueForRealtime, queueForStatsUsage: $queueForStatsUsage, queueForEvents: $queueForEvents, project: $project, @@ -199,6 +202,7 @@ class Functions extends Action log: $log, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, + queueForRealtime: $queueForRealtime, queueForStatsUsage: $queueForStatsUsage, queueForEvents: $queueForEvents, project: $project, @@ -284,6 +288,7 @@ class Functions extends Action * @param Log $log * @param Database $dbForProject * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @param StatsUsage $queueForStatsUsage * @param Event $queueForEvents * @param Document $project @@ -308,6 +313,7 @@ class Functions extends Action Log $log, Database $dbForProject, Func $queueForFunctions, + Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Event $queueForEvents, Document $project, @@ -564,20 +570,21 @@ class Functions extends Action ; } - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - /** Trigger Webhook */ $executionModel = new Execution(); $queueForEvents - ->setQueue(Event::WEBHOOK_QUEUE_NAME) - ->setClass(Event::WEBHOOK_CLASS_NAME) ->setProject($project) ->setUser($user) ->setEvent('functions.[functionId].executions.[executionId].update') ->setParam('functionId', $function->getId()) ->setParam('executionId', $execution->getId()) - ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))) + ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))); + + /** Trigger Webhook */ + $queueForEvents + ->setQueue(Event::WEBHOOK_QUEUE_NAME) + ->setClass(Event::WEBHOOK_CLASS_NAME) ->trigger(); /** Trigger Functions */ @@ -585,31 +592,16 @@ class Functions extends Action ->from($queueForEvents) ->trigger(); - /** Trigger realtime event */ - $allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [ - 'functionId' => $function->getId(), - 'executionId' => $execution->getId() - ]); - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $execution, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $execution->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - Realtime::send( - projectId: $project->getId(), - payload: $execution->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + /** Trigger Realtime Events */ + $queueForRealtime + ->from($queueForEvents) + ->setProjectId('console') + ->trigger(); + + $queueForRealtime + ->from($queueForEvents) + ->setProjectId($project->getId()) + ->trigger(); if (!empty($error)) { throw new Exception($error, $errorCode); From 5076303c33a600da6b91a4a8ebe7cb18d79d325c Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Feb 2025 06:06:43 +0000 Subject: [PATCH 017/275] chore: refactor realtime queueing in builds worker --- src/Appwrite/Platform/Workers/Builds.php | 115 +++++++++-------------- 1 file changed, 47 insertions(+), 68 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index e7cbbd5088..f3f733f5eb 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -5,8 +5,8 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; -use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Utopia\Response\Model\Deployment; use Appwrite\Vcs\Comment; use Exception; @@ -50,12 +50,13 @@ class Builds extends Action ->inject('dbForPlatform') ->inject('queueForEvents') ->inject('queueForFunctions') + ->inject('queueForRealtime') ->inject('queueForStatsUsage') ->inject('cache') ->inject('dbForProject') ->inject('deviceForFunctions') ->inject('log') - ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); } /** @@ -64,6 +65,7 @@ class Builds extends Action * @param Database $dbForPlatform * @param Event $queueForEvents * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @param StatsUsage $queueForStatsUsage * @param Cache $cache * @param Database $dbForProject @@ -72,7 +74,7 @@ class Builds extends Action * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void { $payload = $message->getPayload() ?? []; @@ -93,7 +95,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log); + $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log); break; default: @@ -104,6 +106,7 @@ class Builds extends Action /** * @param Device $deviceForFunctions * @param Func $queueForFunctions + * @param Realtime $queueForRealtime * @param Event $queueForEvents * @param StatsUsage $queueForStatsUsage * @param Database $dbForPlatform @@ -118,7 +121,7 @@ class Builds extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void + protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void { $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); @@ -152,10 +155,7 @@ class Builds extends Action } // Realtime preparation - $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [ - 'functionId' => $function->getId(), - 'deploymentId' => $deployment->getId() - ]); + $event = "functions.[functionId].deployments.[deploymentId].update"; $startTime = DateTime::now(); $durationStart = \microtime(true); @@ -369,21 +369,16 @@ class Builds extends Action $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); /** - * Send realtime Event + * Trigger Realtime Event */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + $queueForRealtime + ->setProject($project) + ->setProjectId('console') + ->setEvent($event) + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($build->getArrayCopy()) + ->trigger(); } $tmpPath = '/tmp/builds/' . $buildId; @@ -449,21 +444,15 @@ class Builds extends Action ->from($deploymentUpdate) ->trigger(); - /** Trigger Realtime */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + /** Trigger Realtime Event */ + $queueForRealtime + ->setProject($project) + ->setProjectId('console') + ->setEvent($event) + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($build->getArrayCopy()) + ->trigger(); $vars = []; @@ -556,12 +545,12 @@ class Builds extends Action $err = $error; } }), - Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err, &$isCanceled) { + Co\go(function () use ($executor, $project, $function, $deployment, &$response, &$build, $dbForProject, $event, &$err, $queueForRealtime, &$isCanceled) { try { $executor->getLogs( deploymentId: $deployment->getId(), projectId: $project->getId(), - callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $allEvents, $project, &$isCanceled) { + callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $event, $project, $function, $deployment, $queueForRealtime, &$isCanceled) { if ($isCanceled) { return; } @@ -586,21 +575,16 @@ class Builds extends Action $build = $dbForProject->updateDocument('builds', $build->getId(), $build); /** - * Send realtime Event + * Trigger Realtime Event */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + $queueForRealtime + ->setProject($project) + ->setProjectId('console') + ->setEvent($event) + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($build->getArrayCopy()) + ->trigger(); } } ); @@ -688,21 +672,16 @@ class Builds extends Action } } finally { /** - * Send realtime Event + * Trigger Realtime Event */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); + $queueForRealtime + ->setProject($project) + ->setProjectId('console') + ->setEvent($event) + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($build->getArrayCopy()) + ->trigger(); /** Trigger usage queue */ if ($build->getAttribute('status') === 'ready') { From 940f4d12af617977b5f8312658e0454dee4f7d5f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 11 Feb 2025 07:52:37 +0000 Subject: [PATCH 018/275] Remove usage and usage dump in favor of stats-usage and stats-usage-dump --- Dockerfile | 4 +- app/init.php | 3 - app/worker.php | 8 - bin/worker-usage | 3 - bin/worker-usage-dump | 3 - src/Appwrite/Platform/Services/Workers.php | 8 - src/Appwrite/Platform/Workers/Usage.php | 292 -------------------- src/Appwrite/Platform/Workers/UsageDump.php | 286 ------------------- 8 files changed, 1 insertion(+), 606 deletions(-) delete mode 100644 bin/worker-usage delete mode 100644 bin/worker-usage-dump delete mode 100644 src/Appwrite/Platform/Workers/Usage.php delete mode 100644 src/Appwrite/Platform/Workers/UsageDump.php diff --git a/Dockerfile b/Dockerfile index 2bb9f80d9e..01ad5d3fba 100755 --- a/Dockerfile +++ b/Dockerfile @@ -88,9 +88,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-stats-usage && \ chmod +x /usr/local/bin/worker-stats-usage-dump && \ chmod +x /usr/local/bin/stats-resources && \ - chmod +x /usr/local/bin/worker-stats-resources && \ - chmod +x /usr/local/bin/worker-usage && \ - chmod +x /usr/local/bin/worker-usage-dump + chmod +x /usr/local/bin/worker-stats-resources # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/app/init.php b/app/init.php index 0c6746de3c..9bd8cf839c 100644 --- a/app/init.php +++ b/app/init.php @@ -1197,9 +1197,6 @@ App::setResource('queueForAudits', function (Queue\Publisher $publisher) { App::setResource('queueForFunctions', function (Queue\Publisher $publisher) { return new Func($publisher); }, ['publisher']); -App::setResource('queueForUsage', function (Queue\Publisher $publisher) { - return new Usage($publisher); -}, ['publisher']); App::setResource('queueForCertificates', function (Queue\Publisher $publisher) { return new Certificate($publisher); }, ['publisher']); diff --git a/app/worker.php b/app/worker.php index 605474e9f1..dc02da1bb7 100644 --- a/app/worker.php +++ b/app/worker.php @@ -269,14 +269,6 @@ Server::setResource('consumer', function (Group $pools) { return $pools->get('consumer')->pop()->getResource(); }, ['pools']); -Server::setResource('queueForUsage', function (Publisher $publisher) { - return new Usage($publisher); -}, ['publisher']); - -Server::setResource('queueForUsageDump', function (Publisher $publisher) { - return new UsageDump($publisher); -}, ['publisher']); - Server::setResource('queueForStatsUsage', function (Publisher $publisher) { return new StatsUsage($publisher); }, ['publisher']); diff --git a/bin/worker-usage b/bin/worker-usage deleted file mode 100644 index e39ce8477c..0000000000 --- a/bin/worker-usage +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/worker.php usage $@ \ No newline at end of file diff --git a/bin/worker-usage-dump b/bin/worker-usage-dump deleted file mode 100644 index 43ca87fcb3..0000000000 --- a/bin/worker-usage-dump +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/worker.php usage-dump $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index e121ee35f7..4f4095aca4 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -14,10 +14,6 @@ use Appwrite\Platform\Workers\Migrations; use Appwrite\Platform\Workers\StatsResources; use Appwrite\Platform\Workers\StatsUsage; use Appwrite\Platform\Workers\StatsUsageDump; -/** remove */ -use Appwrite\Platform\Workers\Usage; -use Appwrite\Platform\Workers\UsageDump; -/** /remove */ use Appwrite\Platform\Workers\Webhooks; use Utopia\Platform\Service; @@ -40,10 +36,6 @@ class Workers extends Service ->addAction(StatsUsage::getName(), new StatsUsage()) ->addAction(Migrations::getName(), new Migrations()) ->addAction(StatsResources::getName(), new StatsResources()) - /** Remove */ - ->addAction(UsageDump::getName(), new UsageDump()) - ->addAction(Usage::getName(), new Usage()) - /** /remove */ ; } } diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php deleted file mode 100644 index 3687eeab67..0000000000 --- a/src/Appwrite/Platform/Workers/Usage.php +++ /dev/null @@ -1,292 +0,0 @@ -desc('Usage worker') - ->inject('message') - ->inject('project') - ->inject('getProjectDB') - ->inject('queueForUsageDump') - ->callback(function (Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump) { - $this->action($message, $project, $getProjectDB, $queueForUsageDump); - }); - - $this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); - $this->lastTriggeredTime = time(); - } - - /** - * @param Message $message - * @param Document $project - * @param callable $getProjectDB - * @param UsageDump $queueForUsageDump - * @return void - * @throws \Utopia\Database\Exception - * @throws Exception - */ - public function action(Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump): void - { - $payload = $message->getPayload() ?? []; - if (empty($payload)) { - throw new Exception('Missing payload'); - } - - - if (empty($project->getAttribute('database'))) { - var_dump($payload); - return; - } - - $projectId = $project->getInternalId(); - foreach ($payload['reduce'] ?? [] as $document) { - if (empty($document)) { - continue; - } - - $this->reduce( - project: $project, - document: new Document($document), - metrics: $payload['metrics'], - getProjectDB: $getProjectDB - ); - } - - - $this->stats[$projectId]['project'] = [ - '$id' => $project->getId(), - '$internalId' => $project->getInternalId(), - 'database' => $project->getAttribute('database'), - ]; - $this->stats[$projectId]['receivedAt'] = DateTime::now(); - foreach ($payload['metrics'] ?? [] as $metric) { - $this->keys++; - if (!isset($this->stats[$projectId]['keys'][$metric['key']])) { - $this->stats[$projectId]['keys'][$metric['key']] = $metric['value']; - continue; - } - - $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; - } - - // If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) - if ( - $this->keys >= self::KEYS_THRESHOLD || - (time() - $this->lastTriggeredTime > $this->aggregationInterval && $this->keys > 0) - ) { - Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); - - $queueForUsageDump - ->setStats($this->stats) - ->trigger(); - - $this->stats = []; - $this->keys = 0; - $this->lastTriggeredTime = time(); - } - } - - /** - * On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files. - * When we remove a parent document we need to deduct his children aggregation from the project scope. - * @param Document $project - * @param Document $document - * @param array $metrics - * @param callable $getProjectDB - * @return void - */ - private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void - { - $dbForProject = $getProjectDB($project); - - try { - switch (true) { - case $document->getCollection() === 'users': // users - $sessions = count($document->getAttribute(METRIC_SESSIONS, 0)); - if (!empty($sessions)) { - $metrics[] = [ - 'key' => METRIC_SESSIONS, - 'value' => ($sessions * -1), - ]; - } - break; - case $document->getCollection() === 'databases': // databases - $collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS))); - $documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS))); - if (!empty($collections['value'])) { - $metrics[] = [ - 'key' => METRIC_COLLECTIONS, - 'value' => ($collections['value'] * -1), - ]; - } - - if (!empty($documents['value'])) { - $metrics[] = [ - 'key' => METRIC_DOCUMENTS, - 'value' => ($documents['value'] * -1), - ]; - } - break; - case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections - $parts = explode('_', $document->getCollection()); - $databaseInternalId = $parts[1] ?? 0; - $documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS))); - - if (!empty($documents['value'])) { - $metrics[] = [ - 'key' => METRIC_DOCUMENTS, - 'value' => ($documents['value'] * -1), - ]; - $metrics[] = [ - 'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), - 'value' => ($documents['value'] * -1), - ]; - } - break; - - case $document->getCollection() === 'buckets': - $files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES))); - $storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE))); - - if (!empty($files['value'])) { - $metrics[] = [ - 'key' => METRIC_FILES, - 'value' => ($files['value'] * -1), - ]; - } - - if (!empty($storage['value'])) { - $metrics[] = [ - 'key' => METRIC_FILES_STORAGE, - 'value' => ($storage['value'] * -1), - ]; - } - break; - - case $document->getCollection() === 'functions': - $deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS))); - $deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE))); - $builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS))); - $buildsSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS))); - $buildsFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED))); - $buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE))); - $buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE))); - $buildsComputeSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS))); - $buildsComputeFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED))); - $executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS))); - $executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE))); - - if (!empty($deployments['value'])) { - $metrics[] = [ - 'key' => METRIC_DEPLOYMENTS, - 'value' => ($deployments['value'] * -1), - ]; - } - - if (!empty($deploymentsStorage['value'])) { - $metrics[] = [ - 'key' => METRIC_DEPLOYMENTS_STORAGE, - 'value' => ($deploymentsStorage['value'] * -1), - ]; - } - - if (!empty($builds['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS, - 'value' => ($builds['value'] * -1), - ]; - } - - if (!empty($buildsSuccess['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_SUCCESS, - 'value' => ($buildsSuccess['value'] * -1), - ]; - } - - if (!empty($buildsFailed['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_FAILED, - 'value' => ($buildsFailed['value'] * -1), - ]; - } - - if (!empty($buildsStorage['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_STORAGE, - 'value' => ($buildsStorage['value'] * -1), - ]; - } - - if (!empty($buildsCompute['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_COMPUTE, - 'value' => ($buildsCompute['value'] * -1), - ]; - } - - if (!empty($buildsComputeSuccess['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_COMPUTE_SUCCESS, - 'value' => ($buildsComputeSuccess['value'] * -1), - ]; - } - - if (!empty($buildsComputeFailed['value'])) { - $metrics[] = [ - 'key' => METRIC_BUILDS_COMPUTE_FAILED, - 'value' => ($buildsComputeFailed['value'] * -1), - ]; - } - - if (!empty($executions['value'])) { - $metrics[] = [ - 'key' => METRIC_EXECUTIONS, - 'value' => ($executions['value'] * -1), - ]; - } - - if (!empty($executionsCompute['value'])) { - $metrics[] = [ - 'key' => METRIC_EXECUTIONS_COMPUTE, - 'value' => ($executionsCompute['value'] * -1), - ]; - } - break; - default: - break; - } - } catch (\Throwable $e) { - console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}"); - } - } -} diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php deleted file mode 100644 index 2f1d13f29a..0000000000 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ /dev/null @@ -1,286 +0,0 @@ - 'Y-m-d H:00', - '1d' => 'Y-m-d 00:00', - 'inf' => '0000-00-00 00:00' - ]; - - public static function getName(): string - { - return 'usage-dump'; - } - - /** - * @throws \Exception - */ - public function __construct() - { - $this - ->inject('message') - ->inject('getProjectDB') - ->callback(function (Message $message, callable $getProjectDB) { - $this->action($message, $getProjectDB); - }); - } - - /** - * @param Message $message - * @param callable $getProjectDB - * @return void - * @throws Exception - * @throws \Utopia\Database\Exception - */ - public function action(Message $message, callable $getProjectDB): void - { - $payload = $message->getPayload() ?? []; - if (empty($payload)) { - throw new Exception('Missing payload'); - } - - - foreach ($payload['stats'] ?? [] as $stats) { - $project = new Document($stats['project'] ?? []); - - /** - * End temp bug fallback - */ - $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; - $receivedAt = $stats['receivedAt'] ?? 'NONE'; - if ($numberOfKeys === 0) { - continue; - } - - console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys); - - try { - $dbForProject = $getProjectDB($project); - foreach ($stats['keys'] ?? [] as $key => $value) { - if ($value == 0) { - continue; - } - - if (str_contains($key, METRIC_DATABASES_STORAGE)) { - try { - $this->handleDatabaseStorage($key, $dbForProject); - } catch (\Exception $e) { - console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); - } - continue; - } - - foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); - $id = \md5("{$time}_{$period}_{$key}"); - - try { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), - ])); - } catch (Duplicate $th) { - if ($value < 0) { - $dbForProject->decreaseDocumentAttribute( - 'stats', - $id, - 'value', - abs($value) - ); - } else { - $dbForProject->increaseDocumentAttribute( - 'stats', - $id, - 'value', - $value - ); - } - } - } - } - } catch (\Exception $e) { - console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); - } - } - } - - private function handleDatabaseStorage(string $key, Database $dbForProject): void - { - $data = explode('.', $key); - $start = microtime(true); - - $updateMetric = function (Database $dbForProject, int $value, string $key, string $period, string|null $time) { - $id = \md5("{$time}_{$period}_{$key}"); - - try { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), - ])); - } catch (Duplicate $th) { - if ($value < 0) { - $dbForProject->decreaseDocumentAttribute( - 'stats', - $id, - 'value', - abs($value) - ); - } else { - $dbForProject->increaseDocumentAttribute( - 'stats', - $id, - 'value', - $value - ); - } - } - }; - - foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); - $id = \md5("{$time}_{$period}_{$key}"); - - $value = 0; - $previousValue = 0; - try { - $previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0); - } catch (\Exception $e) { - // No previous value - } - - switch (count($data)) { - // Collection Level - case METRIC_COLLECTION_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); - $databaseInternalId = $data[0]; - $collectionInternalId = $data[1]; - - try { - $value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - - // Compare with previous value - $diff = $value - $previousValue; - - if ($diff === 0) { - break; - } - - // Update Collection - $updateMetric($dbForProject, $diff, $key, $period, $time); - - // Update Database - $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); - $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $diff, $projectKey, $period, $time); - break; - // Database Level - case METRIC_DATABASE_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); - $databaseInternalId = $data[0]; - - $collections = []; - try { - $collections = $dbForProject->find('database_' . $databaseInternalId); - } catch (\Exception $e) { - // Database not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - - foreach ($collections as $collection) { - try { - $value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - } - - $diff = $value - $previousValue; - - if ($diff === 0) { - break; - } - - // Update Database - $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); - $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $diff, $projectKey, $period, $time); - break; - // Project Level - case METRIC_PROJECT_LEVEL_STORAGE: - Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); - // Get all project databases - $databases = $dbForProject->find('database'); - - // Recalculate all databases - foreach ($databases as $database) { - $collections = $dbForProject->find('database_' . $database->getInternalId()); - - foreach ($collections as $collection) { - try { - $value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - } catch (\Exception $e) { - // Collection not found - if ($e->getMessage() !== 'Collection not found') { - throw $e; - } - } - } - } - - $diff = $value - $previousValue; - - // Update Project - $projectKey = METRIC_DATABASES_STORAGE; - $updateMetric($dbForProject, $diff, $projectKey, $period, $time); - break; - } - } - - $end = microtime(true); - - console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); - } -} From f2fc45da02af88d2e1d69d3e431d78065d6f9a9f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 11 Feb 2025 09:46:00 +0000 Subject: [PATCH 019/275] fix format --- app/worker.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/worker.php b/app/worker.php index dc02da1bb7..7fb96f1208 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,8 +16,6 @@ use Appwrite\Event\Migration; use Appwrite\Event\StatsUsage; use Appwrite\Event\StatsUsageDump; /** remove */ -use Appwrite\Event\Usage; -use Appwrite\Event\UsageDump; /** /remove */ use Appwrite\Platform\Appwrite; use Swoole\Runtime; From 721b9b9aea7a025ffbbc6ea950773eaf3eb71ca3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 13 Feb 2025 00:52:20 +0000 Subject: [PATCH 020/275] remove old events --- src/Appwrite/Event/Usage.php | 78 -------------------------------- src/Appwrite/Event/UsageDump.php | 44 ------------------ 2 files changed, 122 deletions(-) delete mode 100644 src/Appwrite/Event/Usage.php delete mode 100644 src/Appwrite/Event/UsageDump.php diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php deleted file mode 100644 index c70cea5c73..0000000000 --- a/src/Appwrite/Event/Usage.php +++ /dev/null @@ -1,78 +0,0 @@ -setQueue(Event::USAGE_QUEUE_NAME) - ->setClass(Event::USAGE_CLASS_NAME); - } - - /** - * Add reduce. - * - * @param Document $document - * @return self - */ - public function addReduce(Document $document): self - { - $this->reduce[] = $document; - - return $this; - } - - /** - * Add metric. - * - * @param string $key - * @param int $value - * @return self - */ - public function addMetric(string $key, int $value): self - { - - $this->metrics[] = [ - 'key' => $key, - 'value' => $value, - ]; - - return $this; - } - - /** - * Prepare the payload for the usage event. - * - * @return array - */ - protected function preparePayload(): array - { - return [ - 'project' => $this->project, - 'reduce' => $this->reduce, - 'metrics' => $this->metrics, - ]; - } - - /** - * Sends metrics to the usage worker. - * - * @return string|bool - */ - public function trigger(): string|bool - { - parent::trigger(); - $this->metrics = []; - return true; - } -} diff --git a/src/Appwrite/Event/UsageDump.php b/src/Appwrite/Event/UsageDump.php deleted file mode 100644 index a70716e94f..0000000000 --- a/src/Appwrite/Event/UsageDump.php +++ /dev/null @@ -1,44 +0,0 @@ -setQueue(Event::USAGE_DUMP_QUEUE_NAME) - ->setClass(Event::USAGE_DUMP_CLASS_NAME); - } - - /** - * Add Stats. - * - * @param array $stats - * @return self - */ - public function setStats(array $stats): self - { - $this->stats = $stats; - - return $this; - } - - /** - * Prepare the payload for the usage dump event. - * - * @return array - */ - protected function preparePayload(): array - { - return [ - 'stats' => $this->stats, - ]; - } -} From 745947621fec399e80815f451dabb81402b19f16 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 13 Feb 2025 04:54:29 +0000 Subject: [PATCH 021/275] Feat: count for targets --- app/init.php | 2 ++ .../Platform/Workers/StatsResources.php | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/app/init.php b/app/init.php index 0c6746de3c..7db665842a 100644 --- a/app/init.php +++ b/app/init.php @@ -308,6 +308,8 @@ const METRIC_WEBHOOKS = 'webhooks'; const METRIC_PLATFORMS = 'platforms'; const METRIC_PROVIDERS = 'providers'; const METRIC_TOPICS = 'topics'; +const METRIC_TARGETS = 'targets'; +const METRIC_PROVIDER_TYPE_TARGETS = '{providerType}.targets'; const METRIC_KEYS = 'keys'; const METRIC_RESOURCE_TYPE_ID_BUILDS = '{resourceType}.{resourceInternalId}.builds'; const METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE = '{resourceType}.{resourceInternalId}.builds.storage'; diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 639e90a867..882fc53571 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -129,10 +129,23 @@ class StatsResources extends Action ]); $teams = $dbForProject->count('teams'); $functions = $dbForProject->count('functions'); + $messages = $dbForProject->count('messages'); $providers = $dbForProject->count('providers'); $topics = $dbForProject->count('topics'); + $targets = $dbForProject->count('targets'); + + $emailTargets = $dbForProject->count('targets', [ + Query::equal('providerType', [MESSAGE_TYPE_EMAIL]) + ]); + $pushTargets = $dbForProject->count('targets', [ + Query::equal('providerType', [MESSAGE_TYPE_PUSH]) + ]); + $smsTargets = $dbForProject->count('targets', [ + Query::equal('providerType', [MESSAGE_TYPE_SMS]) + ]); + $metrics = [ METRIC_DATABASES => $databases, METRIC_BUCKETS => $buckets, @@ -148,6 +161,10 @@ class StatsResources extends Action METRIC_PROVIDERS => $providers, METRIC_TOPICS => $topics, METRIC_KEYS => $keys, + METRIC_TARGETS => $targets, + [str_replace('{providerType}', MESSAGE_TYPE_EMAIL, METRIC_PROVIDER_TYPE_TARGETS)] => $emailTargets, + [str_replace('{providerType}', MESSAGE_TYPE_PUSH, METRIC_PROVIDER_TYPE_TARGETS)] => $pushTargets, + [str_replace('{providerType}', MESSAGE_TYPE_SMS, METRIC_PROVIDER_TYPE_TARGETS)] => $smsTargets, ]; foreach ($metrics as $metric => $value) { From 503b8a0cde2543136ae26b49008ff0360f9b8930 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 13 Feb 2025 04:58:14 +0000 Subject: [PATCH 022/275] spacing --- src/Appwrite/Platform/Workers/StatsResources.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 882fc53571..cdd23043bd 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -133,9 +133,7 @@ class StatsResources extends Action $messages = $dbForProject->count('messages'); $providers = $dbForProject->count('providers'); $topics = $dbForProject->count('topics'); - $targets = $dbForProject->count('targets'); - $emailTargets = $dbForProject->count('targets', [ Query::equal('providerType', [MESSAGE_TYPE_EMAIL]) ]); From 1e446948e8dd6e67c450f685c87e20d1c685db39 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 13 Feb 2025 05:24:00 +0000 Subject: [PATCH 023/275] Fix error --- src/Appwrite/Platform/Workers/StatsResources.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index cdd23043bd..48965fa2d7 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -160,9 +160,9 @@ class StatsResources extends Action METRIC_TOPICS => $topics, METRIC_KEYS => $keys, METRIC_TARGETS => $targets, - [str_replace('{providerType}', MESSAGE_TYPE_EMAIL, METRIC_PROVIDER_TYPE_TARGETS)] => $emailTargets, - [str_replace('{providerType}', MESSAGE_TYPE_PUSH, METRIC_PROVIDER_TYPE_TARGETS)] => $pushTargets, - [str_replace('{providerType}', MESSAGE_TYPE_SMS, METRIC_PROVIDER_TYPE_TARGETS)] => $smsTargets, + str_replace('{providerType}', MESSAGE_TYPE_EMAIL, METRIC_PROVIDER_TYPE_TARGETS) => $emailTargets, + str_replace('{providerType}', MESSAGE_TYPE_PUSH, METRIC_PROVIDER_TYPE_TARGETS) => $pushTargets, + str_replace('{providerType}', MESSAGE_TYPE_SMS, METRIC_PROVIDER_TYPE_TARGETS) => $smsTargets, ]; foreach ($metrics as $metric => $value) { From a74c6e0b10368c16977e576e90fef76be0a9e2cc Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Feb 2025 07:58:56 +0000 Subject: [PATCH 024/275] chore: added targets to realtime, refactor webhook queues --- app/controllers/api/functions.php | 22 ++++----- src/Appwrite/Event/Event.php | 45 +++++++++---------- src/Appwrite/Event/Realtime.php | 38 +++++++++++----- src/Appwrite/Platform/Workers/Builds.php | 30 +++++++------ .../Platform/Workers/Certificates.php | 33 +++++++------- src/Appwrite/Platform/Workers/Databases.php | 2 +- src/Appwrite/Platform/Workers/Functions.php | 22 ++++----- src/Appwrite/Platform/Workers/Migrations.php | 10 +---- 8 files changed, 103 insertions(+), 99 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 0f9f5211f4..644aee4336 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -9,6 +9,7 @@ use Appwrite\Event\Func; use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; use Appwrite\Event\Validator\FunctionEvent; +use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\Validator\Headers; @@ -194,10 +195,12 @@ App::post('/v1/functions') ->inject('user') ->inject('queueForEvents') ->inject('queueForBuilds') + ->inject('queueForWebhooks') + ->inject('queueForFunctions') ->inject('queueForRealtime') ->inject('dbForPlatform') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $timelimit, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Realtime $queueForRealtime, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $timelimit, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; // Temporary abuse check @@ -396,26 +399,19 @@ App::post('/v1/functions') ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); /** Trigger Webhook */ - $ruleCreate - ->setClass(Event::WEBHOOK_CLASS_NAME) - ->setQueue(Event::WEBHOOK_QUEUE_NAME) + $queueForWebhooks + ->from($ruleCreate) ->trigger(); /** Trigger Functions */ - $ruleCreate - ->setClass(Event::FUNCTIONS_CLASS_NAME) - ->setQueue(Event::FUNCTIONS_QUEUE_NAME) + $queueForFunctions + ->from($ruleCreate) ->trigger(); /** Trigger Realtime Events */ $queueForRealtime ->from($ruleCreate) - ->setProjectId('console') - ->trigger(); - - $queueForRealtime - ->from($ruleCreate) - ->setProjectId($project->getId()) + ->setTargets(['console', $project->getId()]) ->trigger(); } diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 8085a836a8..2187210e34 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -62,11 +62,11 @@ class Event protected array $params = []; protected array $sensitive = []; protected array $payload = []; + protected array $targets = []; protected array $context = []; protected ?Document $project = null; protected ?Document $user = null; protected ?string $userId = null; - protected ?string $projectId = null; protected bool $paused = false; /** @@ -152,18 +152,6 @@ class Event return $this; } - /** - * Set projectId for this event. - * - * @param string $projectId - * @return self - */ - public function setProjectId(string $projectId): self - { - $this->projectId = $projectId; - return $this; - } - /** * Get project for this event. * @@ -174,16 +162,6 @@ class Event return $this->project; } - /** - * Get projectId for this event. - * - * @return ?string - */ - public function getProjectId(): ?string - { - return $this->projectId; - } - /** * Set user for this event. * @@ -255,6 +233,27 @@ class Event return $this->payload; } + /** + * Get targets for this event. + * + * @return array + */ + public function setTargets(array $targets): self + { + $this->targets = $targets; + return $this; + } + + /** + * Get targets for this event. + * + * @return array + */ + public function getTargets(): array + { + return $this->targets; + } + /** * Set context for this event. * diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index 8c302bbabf..d16057ad2c 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -53,17 +53,33 @@ class Realtime extends Event bucket: $bucket, ); - RealtimeAdapter::send( - projectId: $this->getProjectId() ?? $target['projectId'] ?? $this->getProject()->getId(), - payload: $this->getRealtimePayload(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'permissionsChanged' => $target['permissionsChanged'], - 'userId' => $this->getParam('userId') - ] - ); + if (!empty($this->getTargets())) { + foreach ($this->getTargets() as $targetProjectId) { + RealtimeAdapter::send( + projectId: $targetProjectId, + payload: $this->getRealtimePayload(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'], + options: [ + 'permissionsChanged' => $target['permissionsChanged'], + 'userId' => $this->getParam('userId') + ] + ); + } + } else { + RealtimeAdapter::send( + projectId: $target['projectId'] ?? $this->getProject()->getId(), + payload: $this->getRealtimePayload(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'], + options: [ + 'permissionsChanged' => $target['permissionsChanged'], + 'userId' => $this->getParam('userId') + ] + ); + } return true; } diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 0bc0937da0..8120adccb6 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -7,6 +7,7 @@ use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; +use Appwrite\Event\Webhook; use Appwrite\Utopia\Response\Model\Deployment; use Appwrite\Vcs\Comment; use Exception; @@ -49,6 +50,7 @@ class Builds extends Action ->inject('project') ->inject('dbForPlatform') ->inject('queueForEvents') + ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') ->inject('queueForStatsUsage') @@ -57,8 +59,8 @@ class Builds extends Action ->inject('deviceForFunctions') ->inject('isResourceBlocked') ->inject('log') - ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) => - $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log)); + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) => + $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log)); } /** @@ -66,6 +68,7 @@ class Builds extends Action * @param Document $project * @param Database $dbForPlatform * @param Event $queueForEvents + * @param Webhook $queueForWebhooks * @param Func $queueForFunctions * @param Realtime $queueForRealtime * @param StatsUsage $queueForStatsUsage @@ -76,7 +79,7 @@ class Builds extends Action * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void { $payload = $message->getPayload() ?? []; @@ -97,7 +100,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log); + $this->buildDeployment($deviceForFunctions, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log); break; default: @@ -107,6 +110,7 @@ class Builds extends Action /** * @param Device $deviceForFunctions + * @param Webhook $queueForWebhooks * @param Func $queueForFunctions * @param Realtime $queueForRealtime * @param Event $queueForEvents @@ -123,7 +127,7 @@ class Builds extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void + protected function buildDeployment(Device $deviceForFunctions, Webhooks $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void { $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); @@ -379,7 +383,7 @@ class Builds extends Action */ $queueForRealtime ->setProject($project) - ->setProjectId('console') + ->setTargets(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) @@ -431,19 +435,19 @@ class Builds extends Action $this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform); } - /** Trigger Webhook */ $deploymentModel = new Deployment(); $deploymentUpdate = $queueForEvents - ->setQueue(Event::WEBHOOK_QUEUE_NAME) - ->setClass(Event::WEBHOOK_CLASS_NAME) ->setProject($project) ->setEvent('functions.[functionId].deployments.[deploymentId].update') ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))); - $deploymentUpdate->trigger(); + /** Trigger Webhook */ + $queueForWebhooks + ->from($deploymentUpdate) + ->trigger(); /** Trigger Functions */ $queueForFunctions @@ -453,7 +457,7 @@ class Builds extends Action /** Trigger Realtime Event */ $queueForRealtime ->setProject($project) - ->setProjectId('console') + ->setTargets(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) @@ -585,7 +589,7 @@ class Builds extends Action */ $queueForRealtime ->setProject($project) - ->setProjectId('console') + ->setTargets(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) @@ -682,7 +686,7 @@ class Builds extends Action */ $queueForRealtime ->setProject($project) - ->setProjectId('console') + ->setTargets(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index c7f345f77a..009ac24021 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -7,6 +7,7 @@ use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Realtime; +use Appwrite\Event\Webhook; use Appwrite\Network\Validator\CNAME; use Appwrite\Template\Template; use Appwrite\Utopia\Response\Model\Rule; @@ -46,13 +47,14 @@ class Certificates extends Action ->inject('dbForPlatform') ->inject('queueForMails') ->inject('queueForEvents') + ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') ->inject('log') ->inject('certificates') ->callback( - fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates) => - $this->action($message, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $queueForRealtime, $log, $certificates) + fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates) => + $this->action($message, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates) ); } @@ -61,6 +63,7 @@ class Certificates extends Action * @param Database $dbForPlatform * @param Mail $queueForMails * @param Event $queueForEvents + * @param Webhook $queueForWebhooks * @param Func $queueForFunctions * @param Realtime $queueForRealtime * @param Log $log @@ -69,7 +72,7 @@ class Certificates extends Action * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates): void + public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates): void { $payload = $message->getPayload() ?? []; @@ -83,7 +86,7 @@ class Certificates extends Action $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck); + $this->execute($domain, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck); } /** @@ -99,7 +102,7 @@ class Certificates extends Action * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void + private function execute(Domain $domain, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain @@ -189,7 +192,7 @@ class Certificates extends Action $certificate->setAttribute('updated', DateTime::now()); // Save all changes we made to certificate document into database - $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForFunctions, $queueForRealtime); + $this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } } @@ -209,7 +212,7 @@ class Certificates extends Action * @throws Conflict * @throws Structure */ - private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime): void + private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime): void { // Check if update or insert required $certificateDocument = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain])]); @@ -223,7 +226,7 @@ class Certificates extends Action } $certificateId = $certificate->getId(); - $this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForFunctions, $queueForRealtime); + $this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime); } /** @@ -342,7 +345,7 @@ class Certificates extends Action * * @return void */ - private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Realtime $queueForRealtime): void + private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime): void { // TODO: @christyjacob remove once we migrate the rules in 1.7.x if (System::getEnv('_APP_RULES_FORMAT') === 'md5') { @@ -379,9 +382,8 @@ class Certificates extends Action ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules()))); /** Trigger Webhook */ - $queueForEvents - ->setQueue(Event::WEBHOOK_QUEUE_NAME) - ->setClass(Event::WEBHOOK_CLASS_NAME) + $queueForWebhooks + ->from($queueForEvents) ->trigger(); /** Trigger Functions */ @@ -392,12 +394,7 @@ class Certificates extends Action /** Trigger Realtime Events */ $queueForRealtime ->from($queueForEvents) - ->setProjectId('console') - ->trigger(); - - $queueForRealtime - ->from($queueForEvents) - ->setProjectId($project->getId()) + ->setTargets(['console', $projectId]) ->trigger(); } } diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 7d58521b82..ae75a74deb 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -604,7 +604,7 @@ class Databases extends Action ): void { $queueForRealtime ->setProject($project) - ->setProjectId('console') + ->setTargets(['console']) ->setEvent($event) ->setParam('databaseId', $database->getId()) ->setParam('collectionId', $collection->getId()); diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index f485db8648..73b02be85e 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -7,6 +7,7 @@ use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; +use Appwrite\Event\Webhook; use Appwrite\Utopia\Response\Model\Execution; use Exception; use Executor\Executor; @@ -44,16 +45,17 @@ class Functions extends Action ->inject('project') ->inject('message') ->inject('dbForProject') + ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('log') ->inject('isResourceBlocked') - ->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked)); + ->callback(fn (Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked)); } - public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void + public function action(Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void { $payload = $message->getPayload() ?? []; @@ -137,6 +139,7 @@ class Functions extends Action $this->execute( log: $log, dbForProject: $dbForProject, + queueForWebhooks: $queueForWebhooks, queueForFunctions: $queueForFunctions, queueForRealtime: $queueForRealtime, queueForStatsUsage: $queueForStatsUsage, @@ -178,6 +181,7 @@ class Functions extends Action $this->execute( log: $log, dbForProject: $dbForProject, + queueForWebhooks: $queueForWebhooks, queueForFunctions: $queueForFunctions, queueForRealtime: $queueForRealtime, queueForStatsUsage: $queueForStatsUsage, @@ -201,6 +205,7 @@ class Functions extends Action $this->execute( log: $log, dbForProject: $dbForProject, + queueForWebhooks: $queueForWebhooks, queueForFunctions: $queueForFunctions, queueForRealtime: $queueForRealtime, queueForStatsUsage: $queueForStatsUsage, @@ -312,6 +317,7 @@ class Functions extends Action private function execute( Log $log, Database $dbForProject, + Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, @@ -582,9 +588,8 @@ class Functions extends Action ->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules()))); /** Trigger Webhook */ - $queueForEvents - ->setQueue(Event::WEBHOOK_QUEUE_NAME) - ->setClass(Event::WEBHOOK_CLASS_NAME) + $queueForWebhooks + ->from($queueForEvents) ->trigger(); /** Trigger Functions */ @@ -595,12 +600,7 @@ class Functions extends Action /** Trigger Realtime Events */ $queueForRealtime ->from($queueForEvents) - ->setProjectId('console') - ->trigger(); - - $queueForRealtime - ->from($queueForEvents) - ->setProjectId($project->getId()) + ->setTargets(['console', $project->getId()]) ->trigger(); if (!empty($error)) { diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 50d6002d28..541c171a22 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -160,15 +160,7 @@ class Migrations extends Action /** Trigger Realtime Events */ $queueForRealtime ->setProject($project) - ->setProjectId('console') - ->setEvent('migrations.[migrationId].update') - ->setParam('migrationId', $migration->getId()) - ->setPayload($migration->getArrayCopy()) - ->trigger(); - - $queueForRealtime - ->setProject($project) - ->setProjectId($project->getId()) + ->setTargets(['console', $project->getId()]) ->setEvent('migrations.[migrationId].update') ->setParam('migrationId', $migration->getId()) ->setPayload($migration->getArrayCopy()) From 235a357ca3aa22876881840d262ca34853473331 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Feb 2025 11:37:07 +0000 Subject: [PATCH 025/275] chore: initialized queueForWebhooks in worker init --- app/worker.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/worker.php b/app/worker.php index ad6bf475f9..1088a9b6db 100644 --- a/app/worker.php +++ b/app/worker.php @@ -20,6 +20,7 @@ use Appwrite\Event\StatsUsageDump; use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; /** /remove */ +use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; @@ -314,6 +315,10 @@ Server::setResource('queueForAudits', function (Publisher $publisher) { return new Audit($publisher); }, ['publisher']); +Server::setResource('queueForWebhooks', function (Publisher $publisher) { + return new Webhook($publisher); +}, ['publisher']); + Server::setResource('queueForFunctions', function (Publisher $publisher) { return new Func($publisher); }, ['publisher']); From 9f7aaf702913548ce52cc7311973f0ba7de97f03 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Feb 2025 12:51:19 +0000 Subject: [PATCH 026/275] chore: fix warning in database worker --- src/Appwrite/Event/Event.php | 3 ++- src/Appwrite/Platform/Workers/Databases.php | 28 ++++++++++----------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 2187210e34..7e15cabca9 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -234,8 +234,9 @@ class Event } /** - * Get targets for this event. + * Set targets for this event. * + * @param array $targets * @return array */ public function setTargets(array $targets): self diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index ae75a74deb..b85285ae72 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -197,7 +197,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $project, $attribute, null, $attribute, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, $event, $queueForRealtime, $attribute); if (! $relatedCollection->isEmpty()) { $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId()); @@ -304,7 +304,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $project, $attribute, null, $attribute, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, $event, $queueForRealtime, $attribute); } // The underlying database removes/rebuilds indexes when attribute is removed @@ -418,7 +418,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $project, null, $index, $index, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, $event, $queueForRealtime, null, $index); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId); } } @@ -474,7 +474,7 @@ class Databases extends Action throw $e; } finally { - $this->trigger($database, $collection, $project, null, $index, $index, $event, $queueForRealtime); + $this->trigger($database, $collection, $project, $event, $queueForRealtime, null, $index); $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId()); } } @@ -586,21 +586,19 @@ class Databases extends Action * @param Document $database * @param Document $collection * @param Document $project + * @param Realtime $queueForRealtime * @param Document|null $attribute * @param Document|null $index - * @param Document $payload - * @param Realtime $queueForRealtime * @return void */ protected function trigger( Document $database, Document $collection, Document $project, + string $event, + Realtime $queueForRealtime, Document|null $attribute = null, Document|null $index = null, - Document $payload, - string $event, - Realtime $queueForRealtime ): void { $queueForRealtime ->setProject($project) @@ -610,14 +608,16 @@ class Databases extends Action ->setParam('collectionId', $collection->getId()); if ($attribute !== null) { - $queueForRealtime->setParam('attributeId', $attribute->getId()); + $queueForRealtime + ->setParam('attributeId', $attribute->getId()) + ->setPayload($attribute->getArrayCopy()); } if ($index !== null) { - $queueForRealtime->setParam('indexId', $index->getId()); + $queueForRealtime + ->setParam('indexId', $index->getId()) + ->setPayload($index->getArrayCopy()); } - $queueForRealtime - ->setPayload($payload->getArrayCopy()) - ->trigger(); + $queueForRealtime->trigger(); } } From 88a6616cf8572a6a1132b1527dfde654cbb8a819 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Feb 2025 13:05:50 +0000 Subject: [PATCH 027/275] chore: fix naming --- src/Appwrite/Platform/Workers/Builds.php | 2 +- src/Appwrite/Platform/Workers/Databases.php | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 8120adccb6..c3853a1dde 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -127,7 +127,7 @@ class Builds extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Webhooks $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void + protected function buildDeployment(Device $deviceForFunctions, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void { $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index b85285ae72..74101dd1eb 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -516,7 +516,6 @@ class Databases extends Action $collectionId = $collection->getId(); $collectionInternalId = $collection->getInternalId(); - $databaseId = $database->getId(); $databaseInternalId = $database->getInternalId(); $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); From 55e7633bd12c2631f03f9bc75e5967e1b600f0cd Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Feb 2025 14:04:59 +0000 Subject: [PATCH 028/275] chore: shift targets to realtime --- src/Appwrite/Event/Event.php | 23 ----------------------- src/Appwrite/Event/Realtime.php | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 7e15cabca9..0edffdf4dc 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -62,7 +62,6 @@ class Event protected array $params = []; protected array $sensitive = []; protected array $payload = []; - protected array $targets = []; protected array $context = []; protected ?Document $project = null; protected ?Document $user = null; @@ -233,28 +232,6 @@ class Event return $this->payload; } - /** - * Set targets for this event. - * - * @param array $targets - * @return array - */ - public function setTargets(array $targets): self - { - $this->targets = $targets; - return $this; - } - - /** - * Get targets for this event. - * - * @return array - */ - public function getTargets(): array - { - return $this->targets; - } - /** * Set context for this event. * diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index d16057ad2c..e04bbfffb4 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -7,10 +7,17 @@ use Utopia\Database\Document; class Realtime extends Event { + protected array $targets = []; + public function __construct() { } + /** + * Get Realtime payload for this event. + * + * @return array + */ public function getRealtimePayload(): array { $payload = []; @@ -24,6 +31,28 @@ class Realtime extends Event return $payload; } + /** + * Set targets for this realtime event. + * + * @param array $targets + * @return array + */ + public function setTargets(array $targets): self + { + $this->targets = $targets; + return $this; + } + + /** + * Get targets for this realtime event. + * + * @return array + */ + public function getTargets(): array + { + return $this->targets; + } + /** * Execute Event. * From 6235a1c729627302bb991afdf419906abb848b99 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 10:33:58 +0000 Subject: [PATCH 029/275] chore: make min/max params optional for attribute update --- app/controllers/api/databases.php | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index c563d1e8e0..a4b5683512 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2339,14 +2339,19 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/integ ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', '', new Key(), 'Attribute Key.') ->param('required', null, new Boolean(), 'Is attribute required?') - ->param('min', null, new Integer(), 'Minimum value to enforce on new documents') - ->param('max', null, new Integer(), 'Maximum value to enforce on new documents') + ->param('min', null, new Integer(), 'Minimum value to enforce on new documents', true) + ->param('max', null, new Integer(), 'Maximum value to enforce on new documents', true) ->param('default', null, new Nullable(new Integer()), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->param('newKey', null, new Key(), 'New attribute key.', true) ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?int $min, ?int $max, ?int $default, ?string $newKey, Response $response, Database $dbForProject, Event $queueForEvents) { + + // Ensure attribute default is within range + $min = \is_null($min) ? PHP_INT_MIN : $min; + $max = \is_null($max) ? PHP_INT_MAX : $max; + $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, @@ -2398,14 +2403,19 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/float ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', '', new Key(), 'Attribute Key.') ->param('required', null, new Boolean(), 'Is attribute required?') - ->param('min', null, new FloatValidator(), 'Minimum value to enforce on new documents') - ->param('max', null, new FloatValidator(), 'Maximum value to enforce on new documents') + ->param('min', null, new FloatValidator(), 'Minimum value to enforce on new documents', true) + ->param('max', null, new FloatValidator(), 'Maximum value to enforce on new documents', true) ->param('default', null, new Nullable(new FloatValidator()), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->param('newKey', null, new Key(), 'New attribute key.', true) ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?float $min, ?float $max, ?float $default, ?string $newKey, Response $response, Database $dbForProject, Event $queueForEvents) { + + // Ensure attribute default is within range + $min = \is_null($min) ? -PHP_FLOAT_MAX : $min; + $max = \is_null($max) ? PHP_FLOAT_MAX : $max; + $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, From ccb7dfa839311a78c624c516800ed1cc0d142a53 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 11:15:05 +0000 Subject: [PATCH 030/275] chore: added test cases --- .../e2e/Services/Databases/DatabasesBase.php | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 0f57f94515..6a4839274c 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -464,6 +464,38 @@ trait DatabasesBase $this->assertEquals(400, $attribute['headers']['status-code']); $this->assertStringContainsString('Index length is longer than the maximum: 76', $attribute['body']['message']); + + $integer = $this->client->call(Client::METHOD_POST, '/databases/'.$databaseId.'/collections/'.$collection['body']['$id'].'/attributes/integer', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'integer', + 'required' => false, + 'min' => 1, + 'max' => 5, + 'default' => 3 + ]); + $this->assertEquals(202, $integer['headers']['status-code']); + + sleep(1); + + /** + * Update integer default value to 10 + */ + $integer = $this->client->call(Client::METHOD_PATCH, '/databases/'.$databaseId.'/collections/'.$collection['body']['$id'].'/attributes/integer/'.$integer['body']['key'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]), [ + 'key' => 'integer', + 'required' => false, + 'default' => 10 + ]); + + $this->assertEquals(200, $integer['headers']['status-code']); + $this->assertEquals(PHP_INT_MIN, $integer['body']['min']); + $this->assertEquals(PHP_INT_MAX, $integer['body']['max']); } public function testUpdateAttributeEnum(): void From b6166063db2c237284027bfc1c93d3267a9caafa Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Feb 2025 00:51:29 +1300 Subject: [PATCH 031/275] Add back remove shared v2 --- app/http.php | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/app/http.php b/app/http.php index b9aa69a7cc..451a25a601 100644 --- a/app/http.php +++ b/app/http.php @@ -304,6 +304,54 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg } }); + $projectCollections = $collections['projects']; + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + $sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', '')); + $sharedTablesV2 = \array_diff($sharedTables, $sharedTablesV1); + + $cache = $app->getResource('cache'); + + foreach ($sharedTablesV2 as $hostname) { + $adapter = $pools + ->get($hostname) + ->pop() + ->getResource(); + + $dbForProject = (new Database($adapter, $cache)) + ->setDatabase('appwrite') + ->setSharedTables(true) + ->setTenant(null) + ->setNamespace(System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', '')); + + try { + Console::success('[Setup] - Creating project database: ' . $hostname . '...'); + $dbForProject->create(); + } catch (Duplicate) { + Console::success('[Setup] - Skip: metadata table already exists'); + } + + if ($dbForProject->getCollection(Audit::COLLECTION)->isEmpty()) { + $audit = new Audit($dbForProject); + $audit->setup(); + } + + foreach ($projectCollections as $key => $collection) { + if (($collection['$collection'] ?? '') !== Database::METADATA) { + continue; + } + if (!$dbForProject->getCollection($key)->isEmpty()) { + continue; + } + + $attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']); + $indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']); + + Console::success('[Setup] - Creating project collection: ' . $collection['$id'] . '...'); + + $dbForProject->createCollection($key, $attributes, $indexes); + } + } + $pools->reclaim(); Console::success('[Setup] - Server database init completed...'); }); From b0ced9b351746a23c4fd976f6ff0e022e459a541 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 13:37:13 +0000 Subject: [PATCH 032/275] chore: updated tests --- .../e2e/Services/Databases/DatabasesBase.php | 32 ------ .../Databases/DatabasesCustomServerTest.php | 102 +++++++++--------- 2 files changed, 50 insertions(+), 84 deletions(-) diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 6a4839274c..0f57f94515 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -464,38 +464,6 @@ trait DatabasesBase $this->assertEquals(400, $attribute['headers']['status-code']); $this->assertStringContainsString('Index length is longer than the maximum: 76', $attribute['body']['message']); - - $integer = $this->client->call(Client::METHOD_POST, '/databases/'.$databaseId.'/collections/'.$collection['body']['$id'].'/attributes/integer', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'key' => 'integer', - 'required' => false, - 'min' => 1, - 'max' => 5, - 'default' => 3 - ]); - $this->assertEquals(202, $integer['headers']['status-code']); - - sleep(1); - - /** - * Update integer default value to 10 - */ - $integer = $this->client->call(Client::METHOD_PATCH, '/databases/'.$databaseId.'/collections/'.$collection['body']['$id'].'/attributes/integer/'.$integer['body']['key'], array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'], - ]), [ - 'key' => 'integer', - 'required' => false, - 'default' => 10 - ]); - - $this->assertEquals(200, $integer['headers']['status-code']); - $this->assertEquals(PHP_INT_MIN, $integer['body']['min']); - $this->assertEquals(PHP_INT_MAX, $integer['body']['max']); } public function testUpdateAttributeEnum(): void diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index b501e2119e..ac1df82c9d 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -2309,6 +2309,30 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(100, $new['body']['min']); $this->assertEquals(2000, $new['body']['max']); + $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'required' => false, + 'default' => 100, + 'min' => 0, + ]); + + $this->assertEquals(200, $update['headers']['status-code']); + + $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'required' => false, + 'default' => 100, + 'max' => 0, + ]); + + $this->assertEquals(200, $update['headers']['status-code']); + /** * Test against failure */ @@ -2368,32 +2392,6 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(400, $update['headers']['status-code']); $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); - $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'required' => false, - 'default' => 100, - 'min' => 0, - ]); - - $this->assertEquals(400, $update['headers']['status-code']); - $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); - - $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'required' => false, - 'default' => 100, - 'max' => 0, - ]); - - $this->assertEquals(400, $update['headers']['status-code']); - $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); - $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -2572,6 +2570,32 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(123.456, $new['body']['min']); $this->assertEquals(2000, $new['body']['max']); + $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'required' => false, + 'default' => 123.456, + 'min' => 0.0, + ]); + + $this->assertEquals(200, $update['headers']['status-code']); + $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); + + $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'required' => false, + 'default' => 123.456, + 'max' => 0.0, + ]); + + $this->assertEquals(200, $update['headers']['status-code']); + $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); + /** * Test against failure */ @@ -2631,32 +2655,6 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(400, $update['headers']['status-code']); $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); - $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'required' => false, - 'default' => 123.456, - 'min' => 0.0, - ]); - - $this->assertEquals(400, $update['headers']['status-code']); - $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); - - $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'required' => false, - 'default' => 123.456, - 'max' => 0.0, - ]); - - $this->assertEquals(400, $update['headers']['status-code']); - $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); - $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], From 077fe10d808a04c2a500c6d560633b5163375429 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 14:12:14 +0000 Subject: [PATCH 033/275] chore: fix tests --- tests/e2e/Services/Databases/DatabasesCustomServerTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index ac1df82c9d..213584c3df 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -2327,7 +2327,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'required' => false, - 'default' => 100, + 'default' => -10, 'max' => 0, ]); @@ -2589,7 +2589,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'required' => false, - 'default' => 123.456, + 'default' => -123.456, 'max' => 0.0, ]); From 4e71e85b5b6941a863c5bd1238878769684f543d Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 14:32:34 +0000 Subject: [PATCH 034/275] chore: bump utopia-php/image version --- composer.json | 2 +- composer.lock | 63 ++++++++++++++++++++++++++------------------------- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/composer.json b/composer.json index d3ceb8b7a9..e182ac0fff 100644 --- a/composer.json +++ b/composer.json @@ -56,7 +56,7 @@ "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", "utopia-php/fetch": "0.3.*", - "utopia-php/image": "0.7.*", + "utopia-php/image": "0.8.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.14.*", diff --git a/composer.lock b/composer.lock index 11d2ba4c2f..123ff3eade 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b17c58729c4380afcba7714e9bced863", + "content-hash": "b1d26d441fe1fb6a26978d99ef01985f", "packages": [ { "name": "adhocore/jwt", @@ -1757,16 +1757,16 @@ }, { "name": "php-amqplib/php-amqplib", - "version": "v3.7.2", + "version": "v3.7.3", "source": { "type": "git", "url": "https://github.com/php-amqplib/php-amqplib.git", - "reference": "738a73eb0019b6c99d9bc25d7a0c0dd8f56a5199" + "reference": "9f50fe69a9f1a19e2cb25596a354d705de36fe59" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-amqplib/php-amqplib/zipball/738a73eb0019b6c99d9bc25d7a0c0dd8f56a5199", - "reference": "738a73eb0019b6c99d9bc25d7a0c0dd8f56a5199", + "url": "https://api.github.com/repos/php-amqplib/php-amqplib/zipball/9f50fe69a9f1a19e2cb25596a354d705de36fe59", + "reference": "9f50fe69a9f1a19e2cb25596a354d705de36fe59", "shasum": "" }, "require": { @@ -1832,9 +1832,9 @@ ], "support": { "issues": "https://github.com/php-amqplib/php-amqplib/issues", - "source": "https://github.com/php-amqplib/php-amqplib/tree/v3.7.2" + "source": "https://github.com/php-amqplib/php-amqplib/tree/v3.7.3" }, - "time": "2024-11-21T09:21:41+00:00" + "time": "2025-02-18T20:11:13+00:00" }, { "name": "php-http/discovery", @@ -3966,25 +3966,26 @@ }, { "name": "utopia-php/image", - "version": "0.7.0", + "version": "0.8.0", "source": { "type": "git", "url": "https://github.com/utopia-php/image.git", - "reference": "fcea143edbad524bf871ddbebe801d981f91f181" + "reference": "dcae5b1c6deb3ff6865f4e68f012b3709c289bca" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/image/zipball/fcea143edbad524bf871ddbebe801d981f91f181", - "reference": "fcea143edbad524bf871ddbebe801d981f91f181", + "url": "https://api.github.com/repos/utopia-php/image/zipball/dcae5b1c6deb3ff6865f4e68f012b3709c289bca", + "reference": "dcae5b1c6deb3ff6865f4e68f012b3709c289bca", "shasum": "" }, "require": { + "ext-gd": "*", "ext-imagick": "*", "php": ">=8.1" }, "require-dev": { "laravel/pint": "1.2.*", - "phpstan/phpstan": "1.9.x-dev", + "phpstan/phpstan": "^1.10.0", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.13.1" }, @@ -4008,9 +4009,9 @@ ], "support": { "issues": "https://github.com/utopia-php/image/issues", - "source": "https://github.com/utopia-php/image/tree/0.7.0" + "source": "https://github.com/utopia-php/image/tree/0.8.0" }, - "time": "2024-10-02T05:45:38+00:00" + "time": "2025-02-20T11:49:03+00:00" }, { "name": "utopia-php/locale", @@ -5370,16 +5371,16 @@ }, { "name": "laravel/pint", - "version": "v1.20.0", + "version": "v1.21.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "53072e8ea22213a7ed168a8a15b96fbb8b82d44b" + "reference": "531fa0871fbde719c51b12afa3a443b8f4e4b425" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/53072e8ea22213a7ed168a8a15b96fbb8b82d44b", - "reference": "53072e8ea22213a7ed168a8a15b96fbb8b82d44b", + "url": "https://api.github.com/repos/laravel/pint/zipball/531fa0871fbde719c51b12afa3a443b8f4e4b425", + "reference": "531fa0871fbde719c51b12afa3a443b8f4e4b425", "shasum": "" }, "require": { @@ -5387,15 +5388,15 @@ "ext-mbstring": "*", "ext-tokenizer": "*", "ext-xml": "*", - "php": "^8.1.0" + "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.66.0", - "illuminate/view": "^10.48.25", - "larastan/larastan": "^2.9.12", - "laravel-zero/framework": "^10.48.25", + "friendsofphp/php-cs-fixer": "^3.68.5", + "illuminate/view": "^11.42.0", + "larastan/larastan": "^3.0.4", + "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", - "nunomaduro/termwind": "^1.17.0", + "nunomaduro/termwind": "^2.3", "pestphp/pest": "^2.36.0" }, "bin": [ @@ -5432,7 +5433,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-01-14T16:20:53+00:00" + "time": "2025-02-18T03:18:57+00:00" }, { "name": "matthiasmullie/minify", @@ -6190,16 +6191,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "2.0.1", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "72e51f7c32c5aef7c8b462195b8c599b11199893" + "reference": "9b30d6fd026b2c132b3985ce6b23bec09ab3aa68" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/72e51f7c32c5aef7c8b462195b8c599b11199893", - "reference": "72e51f7c32c5aef7c8b462195b8c599b11199893", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/9b30d6fd026b2c132b3985ce6b23bec09ab3aa68", + "reference": "9b30d6fd026b2c132b3985ce6b23bec09ab3aa68", "shasum": "" }, "require": { @@ -6231,9 +6232,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/2.0.1" + "source": "https://github.com/phpstan/phpdoc-parser/tree/2.1.0" }, - "time": "2025-02-13T12:25:43+00:00" + "time": "2025-02-19T13:28:12+00:00" }, { "name": "phpunit/php-code-coverage", From f9cc7882aa78e3ab6e66a202afdf0ebfcbbe3f5c Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 14:43:21 +0000 Subject: [PATCH 035/275] chore: fix tests --- tests/e2e/Services/Databases/DatabasesCustomServerTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 213584c3df..ad581eecb6 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -2581,7 +2581,6 @@ class DatabasesCustomServerTest extends Scope ]); $this->assertEquals(200, $update['headers']['status-code']); - $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); $update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([ 'content-type' => 'application/json', From c660b9d858b9cb9b0e2ee85f8e5f64d30d555cb5 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 14:56:23 +0000 Subject: [PATCH 036/275] chore: fix tests --- tests/e2e/Services/Databases/DatabasesCustomServerTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index ad581eecb6..75526b9928 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -2593,7 +2593,6 @@ class DatabasesCustomServerTest extends Scope ]); $this->assertEquals(200, $update['headers']['status-code']); - $this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']); /** * Test against failure From 397a6a34eda3c89128f4d274f8b62fd2721627ca Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 17:30:26 +0000 Subject: [PATCH 037/275] chore: fix null errors on team invite --- app/controllers/api/teams.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 06e653c105..0c1bc4d794 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -715,7 +715,7 @@ App::post('/v1/teams/:teamId/memberships') ->setSubject($subject) ->setBody($body) ->setRecipient($invitee->getAttribute('email')) - ->setName($invitee->getAttribute('name')) + ->setName($name) ->setVariables($emailVariables) ->trigger(); @@ -781,7 +781,7 @@ App::post('/v1/teams/:teamId/memberships') ->dynamic( $membership ->setAttribute('teamName', $team->getAttribute('name')) - ->setAttribute('userName', $invitee->getAttribute('name')) + ->setAttribute('userName', $name) ->setAttribute('userEmail', $invitee->getAttribute('email')), Response::MODEL_MEMBERSHIP ); From 9d9e44a14f969265a3d3c9990bc328617ce59553 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Feb 2025 17:36:42 +0000 Subject: [PATCH 038/275] chore: remove redundant check --- app/controllers/api/teams.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 0c1bc4d794..3007c1af45 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -488,7 +488,7 @@ App::post('/v1/teams/:teamId/memberships') } $email = \strtolower($email); - $name = (empty($name)) ? $email : $name; + $name = empty($name) ? $email : $name; $team = $dbForProject->getDocument('teams', $teamId); if ($team->isEmpty()) { @@ -507,7 +507,7 @@ App::post('/v1/teams/:teamId/memberships') } $email = $invitee->getAttribute('email', ''); $phone = $invitee->getAttribute('phone', ''); - $name = empty($name) ? $invitee->getAttribute('name', '') : $name; + $name = $invitee->getAttribute('name', '') ?: $name; } elseif (!empty($email)) { $invitee = $dbForProject->findOne('users', [Query::equal('email', [$email])]); // Get user by email address if (!$invitee->isEmpty() && !empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { From d6c2dfb9157f845828bc5cc60cccccb3385d4b3a Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Feb 2025 03:27:46 +0000 Subject: [PATCH 039/275] chore: updated name --- app/controllers/api/teams.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 3007c1af45..763aca64f0 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -715,7 +715,7 @@ App::post('/v1/teams/:teamId/memberships') ->setSubject($subject) ->setBody($body) ->setRecipient($invitee->getAttribute('email')) - ->setName($name) + ->setName($invitee->getAttribute('name', '')) ->setVariables($emailVariables) ->trigger(); @@ -781,7 +781,7 @@ App::post('/v1/teams/:teamId/memberships') ->dynamic( $membership ->setAttribute('teamName', $team->getAttribute('name')) - ->setAttribute('userName', $name) + ->setAttribute('userName', $invitee->getAttribute('name')) ->setAttribute('userEmail', $invitee->getAttribute('email')), Response::MODEL_MEMBERSHIP ); From b1629b24ad2fbde59a3867ae061e307f28ef2504 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Feb 2025 11:44:04 +0000 Subject: [PATCH 040/275] chore: added file transformation stats to usage endpoint for bucket --- app/controllers/api/storage.php | 3 +++ src/Appwrite/Utopia/Response/Model/UsageBuckets.php | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f180c22acf..efd82ba40e 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1894,6 +1894,7 @@ App::get('/v1/storage/:bucketId/usage') $metrics = [ str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES), str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE), + str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_FILES_TRANSFORMATIONS), ]; @@ -1948,5 +1949,7 @@ App::get('/v1/storage/:bucketId/usage') 'filesStorageTotal' => $usage[$metrics[1]]['total'], 'files' => $usage[$metrics[0]]['data'], 'storage' => $usage[$metrics[1]]['data'], + 'fileTransformations' => $usage[$metrics[2]]['data'], + 'fileTransformationsTotal' => $usage[$metrics[2]]['total'], ]), Response::MODEL_USAGE_BUCKETS); }); diff --git a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php index 2f528ac9d1..ab8d129857 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php +++ b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php @@ -42,6 +42,19 @@ class UsageBuckets extends Model 'example' => [], 'array' => true ]) + ->addRule('fileTransformations', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of files transformations per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('fileTransformationsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of files transformations.', + 'default' => 0, + 'example' => 0, + ]) ; } From 4ed4c660d5aff0ee10cbdaad8ff2e15293b3ed88 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Feb 2025 12:53:17 +0000 Subject: [PATCH 041/275] chore: added tests for file transformations --- tests/e2e/Services/Storage/StorageConsoleClientTest.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index 5b6731b35e..b3a6ad9274 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -104,5 +104,7 @@ class StorageConsoleClientTest extends Scope $this->assertIsNumeric($response['body']['filesStorageTotal']); $this->assertIsArray($response['body']['files']); $this->assertIsArray($response['body']['storage']); + $this->assertIsArray($response['body']['fileTransformations']); + $this->assertIsNumeric($response['body']['fileTransformationsTotal']); } } From 0e0bab4c40a54a310b9d07599372ec8be8d06ac1 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Feb 2025 13:28:49 +0000 Subject: [PATCH 042/275] chore: updated test --- tests/e2e/Services/Storage/StorageConsoleClientTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index b3a6ad9274..b2624746ca 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -98,7 +98,7 @@ class StorageConsoleClientTest extends Scope ]); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(5, count($response['body'])); + $this->assertEquals(7, count($response['body'])); $this->assertEquals('24h', $response['body']['range']); $this->assertIsNumeric($response['body']['filesTotal']); $this->assertIsNumeric($response['body']['filesStorageTotal']); From 65dd5fe9b3f0e8a178effc388c9287b5157f7559 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 11:34:28 +0530 Subject: [PATCH 043/275] feat: compute usage for OPTIONS and 4XX errors --- app/controllers/general.php | 15 +++++++++++++-- app/controllers/shared/api.php | 1 + docker-compose.yml | 14 +++++++------- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 6db4a8b28d..ae6db43c70 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -737,7 +737,8 @@ App::options() ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { + ->inject('project') + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname, Document $project) { /* * Appwrite Router */ @@ -760,6 +761,16 @@ App::options() ->addHeader('Access-Control-Allow-Origin', $origin) ->addHeader('Access-Control-Allow-Credentials', 'true') ->noContent(); + + /** OPTIONS requests in utopia do not execute shutdown handlers, as a result we need to track the OPTIONS requests explicitly + * @see https://github.com/utopia-php/http/blob/0.33.16/src/App.php#L825-L855 + */ + $queueForStatsUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize()) + ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) + ->setProject($project) + ->trigger(); }); App::error() @@ -874,7 +885,7 @@ App::error() } } - if ($publish && $project->getId() !== 'console') { + if ($project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; $file = $request->getFiles('file'); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 7f7b73ab0c..64a9980bcb 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -803,6 +803,7 @@ App::shutdown() } } + var_dump("in shutdown"); if ($project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; diff --git a/docker-compose.yml b/docker-compose.yml index facf0e6db9..3316e7dcf8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -934,13 +934,13 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - appwrite-assistant: - container_name: appwrite-assistant - image: appwrite/assistant:0.7.0 - networks: - - appwrite - environment: - - _APP_ASSISTANT_OPENAI_API_KEY + # appwrite-assistant: + # container_name: appwrite-assistant + # image: appwrite/assistant:0.7.0 + # networks: + # - appwrite + # environment: + # - _APP_ASSISTANT_OPENAI_API_KEY openruntimes-executor: container_name: openruntimes-executor From 12b494982ba2b016081660197dd5c76557a88521 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 11:37:54 +0530 Subject: [PATCH 044/275] feat: compute usage for OPTIONS and 4XX errors --- app/controllers/general.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index ae6db43c70..65979d3475 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -885,7 +885,10 @@ App::error() } } - if ($project->getId() !== 'console') { + /** + * If its not a publishable error, track usage stats. Publishable errors are >= 500 or those explicitly marked as publish=true in errors.php + */ + if (!$publish && $project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; $file = $request->getFiles('file'); From 2359c1adb6734afd4088de7609fa58107a30d185 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 11:38:59 +0530 Subject: [PATCH 045/275] feat: enable assistant --- docker-compose.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3316e7dcf8..facf0e6db9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -934,13 +934,13 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - # appwrite-assistant: - # container_name: appwrite-assistant - # image: appwrite/assistant:0.7.0 - # networks: - # - appwrite - # environment: - # - _APP_ASSISTANT_OPENAI_API_KEY + appwrite-assistant: + container_name: appwrite-assistant + image: appwrite/assistant:0.7.0 + networks: + - appwrite + environment: + - _APP_ASSISTANT_OPENAI_API_KEY openruntimes-executor: container_name: openruntimes-executor From 30327f86fd967f92486040deb5f929d4e95bdb2c Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 11:39:18 +0530 Subject: [PATCH 046/275] feat: enable assistant --- app/controllers/shared/api.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 64a9980bcb..7f7b73ab0c 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -803,7 +803,6 @@ App::shutdown() } } - var_dump("in shutdown"); if ($project->getId() !== 'console') { if (!Auth::isPrivilegedUser(Authorization::getRoles())) { $fileSize = 0; From cf41d44de4b299d2535a1098b6b769be5aac96a7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 14:47:57 +0530 Subject: [PATCH 047/275] fix: general tests --- src/Appwrite/Platform/Workers/StatsUsage.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index c4d8b0e8d2..a755f723a0 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -17,13 +17,20 @@ class StatsUsage extends Action private int $lastTriggeredTime = 0; private int $keys = 0; private const INFINITY_PERIOD = '_inf_'; - private const KEYS_THRESHOLD = 10000; + private const BATCH_SIZE_DEVELOPMENT = 1; + private const BATCH_SIZE_PRODUCTION = 10_000; public static function getName(): string { return 'stats-usage'; } + private function getBatchSize(): int + { + return System::getEnv('_APP_ENV', 'development') === 'development' + ? self::BATCH_SIZE_DEVELOPMENT + : self::BATCH_SIZE_PRODUCTION; + } /** * @throws Exception */ @@ -86,7 +93,7 @@ class StatsUsage extends Action // If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( - $this->keys >= self::KEYS_THRESHOLD || + $this->keys >= $this->getBatchSize() || (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); From f682e4ac6f8f04cd4b9158633b5c4c04a895cf7e Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 14:56:15 +0530 Subject: [PATCH 048/275] chore: remoove sleep() for usage stats, since batch size is now 1 --- tests/e2e/General/UsageTest.php | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index c0d0c80eb1..3e22f827bd 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -134,8 +134,6 @@ class UsageTest extends Scope #[Retry(count: 1)] public function testUsersStats(array $data): array { - sleep(self::WAIT); - $requestsTotal = $data['requestsTotal']; $response = $this->client->call( @@ -318,8 +316,6 @@ class UsageTest extends Scope $storageTotal = $data['storageTotal']; $filesTotal = $data['filesTotal']; - sleep(self::WAIT); - $response = $this->client->call( Client::METHOD_GET, '/project/usage', @@ -476,8 +472,6 @@ class UsageTest extends Scope $requestsTotal += 1; - sleep(self::WAIT); - for ($i = 0; $i < self::CREATE; $i++) { $name = uniqid() . ' collection'; @@ -539,8 +533,6 @@ class UsageTest extends Scope $collectionsTotal = $data['collectionsTotal']; $documentsTotal = $data['documentsTotal']; - sleep(self::WAIT); - $response = $this->client->call( Client::METHOD_GET, '/project/usage', @@ -709,8 +701,6 @@ class UsageTest extends Scope // $this->assertEquals(201, $response['headers']['status-code']); // } - // sleep(self::WAIT); - // for ($i = 0; $i < 3; $i++) { // try { // $newProjectMetrics = $this->client->call( @@ -752,7 +742,6 @@ class UsageTest extends Scope // if ($i === 2) { // throw $e; // } - // sleep(self::WAIT); // continue; // } // } @@ -792,8 +781,6 @@ class UsageTest extends Scope // $this->assertEquals(204, $response['headers']['status-code']); // } - // sleep(self::WAIT); - // for ($i = 0; $i < 3; $i++) { // try { // $newProjectMetrics = $this->client->call( @@ -835,7 +822,6 @@ class UsageTest extends Scope // if ($i === 2) { // throw $e; // } - // sleep(self::WAIT); // continue; // } // } @@ -1027,8 +1013,6 @@ class UsageTest extends Scope $executionTime = $data['executionTime']; $executions = $data['executions']; - sleep(self::WAIT); - $response = $this->client->call( Client::METHOD_GET, '/functions/' . $functionId . '/usage?range=30d', @@ -1152,7 +1136,6 @@ class UsageTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); - sleep(self::WAIT + 20); $tries = 0; while (true) { From 94108e6f3fd9bbc7ba277d6a974e05c36feb2eb9 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 15:23:34 +0530 Subject: [PATCH 049/275] chore: remove sleep() for usage stats, since batch size is now 1 --- docker-compose.yml | 14 +++++++------- phpunit.xml | 2 +- tests/e2e/General/UsageTest.php | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index facf0e6db9..3316e7dcf8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -934,13 +934,13 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - appwrite-assistant: - container_name: appwrite-assistant - image: appwrite/assistant:0.7.0 - networks: - - appwrite - environment: - - _APP_ASSISTANT_OPENAI_API_KEY + # appwrite-assistant: + # container_name: appwrite-assistant + # image: appwrite/assistant:0.7.0 + # networks: + # - appwrite + # environment: + # - _APP_ASSISTANT_OPENAI_API_KEY openruntimes-executor: container_name: openruntimes-executor diff --git a/phpunit.xml b/phpunit.xml index 4c4e55ea4e..598b730908 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -6,7 +6,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false" + stopOnFailure="true" > diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 3e22f827bd..8b15461857 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -307,7 +307,7 @@ class UsageTest extends Scope /** * @depends testPrepareStorageStats */ - #[Retry(count: 1)] + #[Retry(count: 10)] public function testStorageStats(array $data): array { $bucketId = $data['bucketId']; @@ -488,6 +488,8 @@ class UsageTest extends Scope ] ); + var_dump($response['body']); + $this->assertEquals($name, $response['body']['name']); $this->assertNotEmpty($response['body']['$id']); From f25df670021e9a5c801ebf4cf4dec871b5a6814f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 15:39:37 +0530 Subject: [PATCH 050/275] chore: remove sleep() for usage stats, since batch size is now 1 --- tests/e2e/General/UsageTest.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 8b15461857..df780d8f47 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -23,7 +23,7 @@ class UsageTest extends Scope use SideServer; use FunctionsBase; - private const WAIT = 35; + private const WAIT = 5; private const CREATE = 20; protected string $projectId; @@ -470,6 +470,8 @@ class UsageTest extends Scope $this->assertEquals('name', $response['body']['key']); + sleep(self::WAIT); + $requestsTotal += 1; for ($i = 0; $i < self::CREATE; $i++) { @@ -488,8 +490,6 @@ class UsageTest extends Scope ] ); - var_dump($response['body']); - $this->assertEquals($name, $response['body']['name']); $this->assertNotEmpty($response['body']['$id']); @@ -535,6 +535,8 @@ class UsageTest extends Scope $collectionsTotal = $data['collectionsTotal']; $documentsTotal = $data['documentsTotal']; + sleep(self::WAIT); + $response = $this->client->call( Client::METHOD_GET, '/project/usage', From 4c492723892ed80c848a230bd662165d21bf397a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 15:40:54 +0530 Subject: [PATCH 051/275] chore: enable assistant --- docker-compose.yml | 14 +++++++------- phpunit.xml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3316e7dcf8..facf0e6db9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -934,13 +934,13 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - # appwrite-assistant: - # container_name: appwrite-assistant - # image: appwrite/assistant:0.7.0 - # networks: - # - appwrite - # environment: - # - _APP_ASSISTANT_OPENAI_API_KEY + appwrite-assistant: + container_name: appwrite-assistant + image: appwrite/assistant:0.7.0 + networks: + - appwrite + environment: + - _APP_ASSISTANT_OPENAI_API_KEY openruntimes-executor: container_name: openruntimes-executor diff --git a/phpunit.xml b/phpunit.xml index 598b730908..4c4e55ea4e 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -6,7 +6,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="true" + stopOnFailure="false" > From 3f50532c916c4a4ceeff9935968e88a5ef668b48 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 15:42:53 +0530 Subject: [PATCH 052/275] chore: remove sleep in functions tests --- tests/e2e/Services/Functions/FunctionsCustomServerTest.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 3ed4ca727e..806d07f9f7 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -1686,9 +1686,6 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(1, count($executions['body']['executions'])); }); - // Await Aggregation - sleep(System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', 30)); - $this->assertEventually(function () use ($functionId) { $response = $this->getFunctionUsage($functionId, [ 'range' => '24h' From 895807032725091310e8cfdfe5a4fca54e08fa09 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 23 Feb 2025 15:48:33 +0530 Subject: [PATCH 053/275] chore: linter --- tests/e2e/Services/Functions/FunctionsCustomServerTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 806d07f9f7..182e6902a3 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -13,7 +13,6 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Database\Validator\Datetime as DatetimeValidator; -use Utopia\System\System; class FunctionsCustomServerTest extends Scope { From 06d22f910c8440beca129354cc0fe29cf4057bc2 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 23 Feb 2025 20:51:38 +0200 Subject: [PATCH 054/275] disable logs display --- app/controllers/api/account.php | 6 ++++-- app/controllers/api/databases.php | 18 ++++++++++++------ app/controllers/api/messaging.php | 24 ++++++++++++++++-------- app/controllers/api/teams.php | 6 ++++-- app/controllers/api/users.php | 6 ++++-- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 21a04800b5..bef26ad25c 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2750,8 +2750,10 @@ App::get('/v1/account/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getInternalId()), - 'logs' => $output, + //'total' => $audit->countLogsByUser($user->getInternalId()), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index c563d1e8e0..3859132fd6 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -723,8 +723,10 @@ App::get('/v1/databases/:databaseId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); @@ -1112,8 +1114,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); @@ -3763,8 +3767,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index d7d3750ccf..06d49394aa 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -1047,8 +1047,10 @@ App::get('/v1/messaging/providers/:providerId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); @@ -2276,8 +2278,10 @@ App::get('/v1/messaging/topics/:topicId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); @@ -2686,8 +2690,10 @@ App::get('/v1/messaging/subscribers/:subscriberId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); @@ -3451,8 +3457,10 @@ App::get('/v1/messaging/messages/:messageId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 763aca64f0..7a9f21d74b 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1415,7 +1415,9 @@ App::get('/v1/teams/:teamId/logs') } } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 962022927f..ca3f042ba4 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -915,8 +915,10 @@ App::get('/v1/users/:userId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getInternalId()), - 'logs' => $output, + //'total' => $audit->countLogsByUser($user->getInternalId()), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); From 6bd14d548fbe36b7018a536b024c8eb20ca58fbb Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 23 Feb 2025 20:58:02 +0200 Subject: [PATCH 055/275] composer --- app/controllers/api/databases.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 3859132fd6..d9d62eac51 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -1115,7 +1115,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') $response->dynamic(new Document([ //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, + //'logs' => $output, 'total' => 0, 'logs' => [], ]), Response::MODEL_LOG_LIST); From 08f4ed2b50c96b827196a32c9de6fe8460afd68f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Feb 2025 15:50:10 +1300 Subject: [PATCH 056/275] Log batches per project --- src/Appwrite/Platform/Workers/Audits.php | 55 ++++++++++++++++-------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index f1ae46eea7..f645463aed 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -46,8 +46,9 @@ class Audits extends Action $this ->desc('Audits worker') ->inject('message') - ->inject('dbForProject') - ->callback(fn ($message, $dbForProject) => $this->action($message, $dbForProject)); + ->inject('getProjectDB') + ->inject('project') + ->callback([$this, 'action']); $this->lastTriggeredTime = time(); } @@ -55,14 +56,15 @@ class Audits extends Action /** * @param Message $message - * @param Database $dbForProject + * @param callable $getProjectDB + * @param Document $project * @return void * @throws Throwable * @throws \Utopia\Database\Exception * @throws Authorization * @throws Structure */ - public function action(Message $message, Database $dbForProject): void + public function action(Message $message, callable $getProjectDB, Document $project): void { $payload = $message->getPayload() ?? []; @@ -103,28 +105,43 @@ class Audits extends Action 'timestamp' => DateTime::formatTz(DateTime::now()) ]; - $this->logs[] = $eventData; + if (isset($this->logs[$project->getInternalId()])) { + $this->logs[$project->getInternalId()]['logs'][] = $eventData; + } else { + $this->logs[$project->getInternalId()] = [ + 'project' => new Document([ + '$id' => $project->getId(), + '$internalId' => $project->getInternalId(), + 'database' => $project->getAttribute('database'), + ]), + 'logs' => [$eventData] + ]; + } // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); - $shouldProcessBatch = count($this->logs) >= $batchSize; - if (!$shouldProcessBatch && count($this->logs) > 0) { - $shouldProcessBatch = (time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; + $shouldProcessBatch = \count($this->logs) >= $batchSize; + if (!$shouldProcessBatch && \count($this->logs) > 0) { + $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } if ($shouldProcessBatch) { - Console::log('Processing batch with ' . count($this->logs) . ' events'); - $audit = new Audit($dbForProject); + foreach ($this->logs as $projectLogs) { + $dbForProject = $getProjectDB($projectLogs['project']); - try { - $audit->logBatch($this->logs); - Console::success('Audit logs processed successfully'); - } catch (Throwable $e) { - Console::error('Error processing audit logs: ' . $e->getMessage()); - } finally { - // Clear the pending events after successful batch processing - $this->logs = []; - $this->lastTriggeredTime = time(); + Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); + $audit = new Audit($dbForProject); + + try { + $audit->logBatch($projectLogs['logs']); + Console::success('Audit logs processed successfully'); + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + } finally { + // Clear the pending events after successful batch processing + $this->logs = []; + $this->lastTriggeredTime = time(); + } } } } From 3f8e1cbedbe98a9d381ae97d07399e30ebba387f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Feb 2025 15:57:04 +1300 Subject: [PATCH 057/275] Fix loop unset --- composer.json | 8 +- composer.lock | 453 +++-------------------- src/Appwrite/Platform/Workers/Audits.php | 23 +- 3 files changed, 73 insertions(+), 411 deletions(-) diff --git a/composer.json b/composer.json index e182ac0fff..640faba78a 100644 --- a/composer.json +++ b/composer.json @@ -85,11 +85,11 @@ "require-dev": { "ext-fileinfo": "*", "appwrite/sdk-generator": "0.40.*", - "phpunit/phpunit": "9.5.20", + "phpunit/phpunit": "9.*", "swoole/ide-helper": "5.1.2", - "textalk/websocket": "1.5.7", - "laravel/pint": "^1.14", - "phpbench/phpbench": "^1.2" + "textalk/websocket": "1.5.*", + "laravel/pint": "1.*", + "phpbench/phpbench": "1.*" }, "provide": { "ext-phpiredis": "*" diff --git a/composer.lock b/composer.lock index 123ff3eade..ea2e425a12 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b1d26d441fe1fb6a26978d99ef01985f", + "content-hash": "ccd64a2bc1115b7f02225c8c06b24483", "packages": [ { "name": "adhocore/jwt", @@ -3919,16 +3919,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.16", + "version": "0.33.17", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "e91d4c560d1b809e25faa63d564fef034363b50f" + "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/e91d4c560d1b809e25faa63d564fef034363b50f", - "reference": "e91d4c560d1b809e25faa63d564fef034363b50f", + "url": "https://api.github.com/repos/utopia-php/http/zipball/73fac6fbce9f56282dba4e52a58cf836ec434644", + "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644", "shasum": "" }, "require": { @@ -3960,9 +3960,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.16" + "source": "https://github.com/utopia-php/http/tree/0.33.17" }, - "time": "2025-01-16T15:58:50+00:00" + "time": "2025-02-24T17:35:48+00:00" }, { "name": "utopia-php/image", @@ -5177,77 +5177,32 @@ }, "time": "2024-09-05T10:17:24+00:00" }, - { - "name": "doctrine/deprecations", - "version": "1.1.4", - "source": { - "type": "git", - "url": "https://github.com/doctrine/deprecations.git", - "reference": "31610dbb31faa98e6b5447b62340826f54fbc4e9" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/doctrine/deprecations/zipball/31610dbb31faa98e6b5447b62340826f54fbc4e9", - "reference": "31610dbb31faa98e6b5447b62340826f54fbc4e9", - "shasum": "" - }, - "require": { - "php": "^7.1 || ^8.0" - }, - "require-dev": { - "doctrine/coding-standard": "^9 || ^12", - "phpstan/phpstan": "1.4.10 || 2.0.3", - "phpstan/phpstan-phpunit": "^1.0 || ^2", - "phpunit/phpunit": "^7.5 || ^8.5 || ^9.5", - "psr/log": "^1 || ^2 || ^3" - }, - "suggest": { - "psr/log": "Allows logging deprecations via PSR-3 logger implementation" - }, - "type": "library", - "autoload": { - "psr-4": { - "Doctrine\\Deprecations\\": "src" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "description": "A small layer on top of trigger_error(E_USER_DEPRECATED) or PSR-3 logging with options to disable all deprecations or selectively for packages.", - "homepage": "https://www.doctrine-project.org/", - "support": { - "issues": "https://github.com/doctrine/deprecations/issues", - "source": "https://github.com/doctrine/deprecations/tree/1.1.4" - }, - "time": "2024-12-07T21:18:45+00:00" - }, { "name": "doctrine/instantiator", - "version": "1.5.0", + "version": "2.0.0", "source": { "type": "git", "url": "https://github.com/doctrine/instantiator.git", - "reference": "0a0fa9780f5d4e507415a065172d26a98d02047b" + "reference": "c6222283fa3f4ac679f8b9ced9a4e23f163e80d0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/instantiator/zipball/0a0fa9780f5d4e507415a065172d26a98d02047b", - "reference": "0a0fa9780f5d4e507415a065172d26a98d02047b", + "url": "https://api.github.com/repos/doctrine/instantiator/zipball/c6222283fa3f4ac679f8b9ced9a4e23f163e80d0", + "reference": "c6222283fa3f4ac679f8b9ced9a4e23f163e80d0", "shasum": "" }, "require": { - "php": "^7.1 || ^8.0" + "php": "^8.1" }, "require-dev": { - "doctrine/coding-standard": "^9 || ^11", + "doctrine/coding-standard": "^11", "ext-pdo": "*", "ext-phar": "*", - "phpbench/phpbench": "^0.16 || ^1", - "phpstan/phpstan": "^1.4", - "phpstan/phpstan-phpunit": "^1", - "phpunit/phpunit": "^7.5 || ^8.5 || ^9.5", - "vimeo/psalm": "^4.30 || ^5.4" + "phpbench/phpbench": "^1.2", + "phpstan/phpstan": "^1.9.4", + "phpstan/phpstan-phpunit": "^1.3", + "phpunit/phpunit": "^9.5.27", + "vimeo/psalm": "^5.4" }, "type": "library", "autoload": { @@ -5274,7 +5229,7 @@ ], "support": { "issues": "https://github.com/doctrine/instantiator/issues", - "source": "https://github.com/doctrine/instantiator/tree/1.5.0" + "source": "https://github.com/doctrine/instantiator/tree/2.0.0" }, "funding": [ { @@ -5290,7 +5245,7 @@ "type": "tidelift" } ], - "time": "2022-12-30T00:15:36+00:00" + "time": "2022-12-30T00:23:10+00:00" }, { "name": "doctrine/lexer", @@ -5944,298 +5899,6 @@ ], "time": "2025-01-26T19:54:45+00:00" }, - { - "name": "phpdocumentor/reflection-common", - "version": "2.2.0", - "source": { - "type": "git", - "url": "https://github.com/phpDocumentor/ReflectionCommon.git", - "reference": "1d01c49d4ed62f25aa84a747ad35d5a16924662b" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/ReflectionCommon/zipball/1d01c49d4ed62f25aa84a747ad35d5a16924662b", - "reference": "1d01c49d4ed62f25aa84a747ad35d5a16924662b", - "shasum": "" - }, - "require": { - "php": "^7.2 || ^8.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-2.x": "2.x-dev" - } - }, - "autoload": { - "psr-4": { - "phpDocumentor\\Reflection\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Jaap van Otterdijk", - "email": "opensource@ijaap.nl" - } - ], - "description": "Common reflection classes used by phpdocumentor to reflect the code structure", - "homepage": "http://www.phpdoc.org", - "keywords": [ - "FQSEN", - "phpDocumentor", - "phpdoc", - "reflection", - "static analysis" - ], - "support": { - "issues": "https://github.com/phpDocumentor/ReflectionCommon/issues", - "source": "https://github.com/phpDocumentor/ReflectionCommon/tree/2.x" - }, - "time": "2020-06-27T09:03:43+00:00" - }, - { - "name": "phpdocumentor/reflection-docblock", - "version": "5.6.1", - "source": { - "type": "git", - "url": "https://github.com/phpDocumentor/ReflectionDocBlock.git", - "reference": "e5e784149a09bd69d9a5e3b01c5cbd2e2bd653d8" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/ReflectionDocBlock/zipball/e5e784149a09bd69d9a5e3b01c5cbd2e2bd653d8", - "reference": "e5e784149a09bd69d9a5e3b01c5cbd2e2bd653d8", - "shasum": "" - }, - "require": { - "doctrine/deprecations": "^1.1", - "ext-filter": "*", - "php": "^7.4 || ^8.0", - "phpdocumentor/reflection-common": "^2.2", - "phpdocumentor/type-resolver": "^1.7", - "phpstan/phpdoc-parser": "^1.7|^2.0", - "webmozart/assert": "^1.9.1" - }, - "require-dev": { - "mockery/mockery": "~1.3.5 || ~1.6.0", - "phpstan/extension-installer": "^1.1", - "phpstan/phpstan": "^1.8", - "phpstan/phpstan-mockery": "^1.1", - "phpstan/phpstan-webmozart-assert": "^1.2", - "phpunit/phpunit": "^9.5", - "psalm/phar": "^5.26" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "5.x-dev" - } - }, - "autoload": { - "psr-4": { - "phpDocumentor\\Reflection\\": "src" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Mike van Riel", - "email": "me@mikevanriel.com" - }, - { - "name": "Jaap van Otterdijk", - "email": "opensource@ijaap.nl" - } - ], - "description": "With this component, a library can provide support for annotations via DocBlocks or otherwise retrieve information that is embedded in a DocBlock.", - "support": { - "issues": "https://github.com/phpDocumentor/ReflectionDocBlock/issues", - "source": "https://github.com/phpDocumentor/ReflectionDocBlock/tree/5.6.1" - }, - "time": "2024-12-07T09:39:29+00:00" - }, - { - "name": "phpdocumentor/type-resolver", - "version": "1.10.0", - "source": { - "type": "git", - "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "679e3ce485b99e84c775d28e2e96fade9a7fb50a" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/679e3ce485b99e84c775d28e2e96fade9a7fb50a", - "reference": "679e3ce485b99e84c775d28e2e96fade9a7fb50a", - "shasum": "" - }, - "require": { - "doctrine/deprecations": "^1.0", - "php": "^7.3 || ^8.0", - "phpdocumentor/reflection-common": "^2.0", - "phpstan/phpdoc-parser": "^1.18|^2.0" - }, - "require-dev": { - "ext-tokenizer": "*", - "phpbench/phpbench": "^1.2", - "phpstan/extension-installer": "^1.1", - "phpstan/phpstan": "^1.8", - "phpstan/phpstan-phpunit": "^1.1", - "phpunit/phpunit": "^9.5", - "rector/rector": "^0.13.9", - "vimeo/psalm": "^4.25" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-1.x": "1.x-dev" - } - }, - "autoload": { - "psr-4": { - "phpDocumentor\\Reflection\\": "src" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Mike van Riel", - "email": "me@mikevanriel.com" - } - ], - "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", - "support": { - "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.10.0" - }, - "time": "2024-11-09T15:12:26+00:00" - }, - { - "name": "phpspec/prophecy", - "version": "v1.20.0", - "source": { - "type": "git", - "url": "https://github.com/phpspec/prophecy.git", - "reference": "a0165c648cab6a80311c74ffc708a07bb53ecc93" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/a0165c648cab6a80311c74ffc708a07bb53ecc93", - "reference": "a0165c648cab6a80311c74ffc708a07bb53ecc93", - "shasum": "" - }, - "require": { - "doctrine/instantiator": "^1.2 || ^2.0", - "php": "^7.2 || 8.0.* || 8.1.* || 8.2.* || 8.3.* || 8.4.*", - "phpdocumentor/reflection-docblock": "^5.2", - "sebastian/comparator": "^3.0 || ^4.0 || ^5.0 || ^6.0", - "sebastian/recursion-context": "^3.0 || ^4.0 || ^5.0 || ^6.0" - }, - "require-dev": { - "friendsofphp/php-cs-fixer": "^3.40", - "phpspec/phpspec": "^6.0 || ^7.0", - "phpstan/phpstan": "^1.9", - "phpunit/phpunit": "^8.0 || ^9.0 || ^10.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.x-dev" - } - }, - "autoload": { - "psr-4": { - "Prophecy\\": "src/Prophecy" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Konstantin Kudryashov", - "email": "ever.zet@gmail.com", - "homepage": "http://everzet.com" - }, - { - "name": "Marcello Duarte", - "email": "marcello.duarte@gmail.com" - } - ], - "description": "Highly opinionated mocking framework for PHP 5.3+", - "homepage": "https://github.com/phpspec/prophecy", - "keywords": [ - "Double", - "Dummy", - "dev", - "fake", - "mock", - "spy", - "stub" - ], - "support": { - "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/v1.20.0" - }, - "time": "2024-11-19T13:12:41+00:00" - }, - { - "name": "phpstan/phpdoc-parser", - "version": "2.1.0", - "source": { - "type": "git", - "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "9b30d6fd026b2c132b3985ce6b23bec09ab3aa68" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/9b30d6fd026b2c132b3985ce6b23bec09ab3aa68", - "reference": "9b30d6fd026b2c132b3985ce6b23bec09ab3aa68", - "shasum": "" - }, - "require": { - "php": "^7.4 || ^8.0" - }, - "require-dev": { - "doctrine/annotations": "^2.0", - "nikic/php-parser": "^5.3.0", - "php-parallel-lint/php-parallel-lint": "^1.2", - "phpstan/extension-installer": "^1.0", - "phpstan/phpstan": "^2.0", - "phpstan/phpstan-phpunit": "^2.0", - "phpstan/phpstan-strict-rules": "^2.0", - "phpunit/phpunit": "^9.6", - "symfony/process": "^5.2" - }, - "type": "library", - "autoload": { - "psr-4": { - "PHPStan\\PhpDocParser\\": [ - "src/" - ] - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "description": "PHPDoc parser with support for nullable, intersection and generic types", - "support": { - "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/2.1.0" - }, - "time": "2025-02-19T13:28:12+00:00" - }, { "name": "phpunit/php-code-coverage", "version": "9.2.32", @@ -6557,55 +6220,50 @@ }, { "name": "phpunit/phpunit", - "version": "9.5.20", + "version": "9.6.22", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "12bc8879fb65aef2138b26fc633cb1e3620cffba" + "reference": "f80235cb4d3caa59ae09be3adf1ded27521d1a9c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/12bc8879fb65aef2138b26fc633cb1e3620cffba", - "reference": "12bc8879fb65aef2138b26fc633cb1e3620cffba", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/f80235cb4d3caa59ae09be3adf1ded27521d1a9c", + "reference": "f80235cb4d3caa59ae09be3adf1ded27521d1a9c", "shasum": "" }, "require": { - "doctrine/instantiator": "^1.3.1", + "doctrine/instantiator": "^1.5.0 || ^2", "ext-dom": "*", "ext-json": "*", "ext-libxml": "*", "ext-mbstring": "*", "ext-xml": "*", "ext-xmlwriter": "*", - "myclabs/deep-copy": "^1.10.1", - "phar-io/manifest": "^2.0.3", - "phar-io/version": "^3.0.2", + "myclabs/deep-copy": "^1.12.1", + "phar-io/manifest": "^2.0.4", + "phar-io/version": "^3.2.1", "php": ">=7.3", - "phpspec/prophecy": "^1.12.1", - "phpunit/php-code-coverage": "^9.2.13", - "phpunit/php-file-iterator": "^3.0.5", + "phpunit/php-code-coverage": "^9.2.32", + "phpunit/php-file-iterator": "^3.0.6", "phpunit/php-invoker": "^3.1.1", - "phpunit/php-text-template": "^2.0.3", - "phpunit/php-timer": "^5.0.2", - "sebastian/cli-parser": "^1.0.1", - "sebastian/code-unit": "^1.0.6", - "sebastian/comparator": "^4.0.5", - "sebastian/diff": "^4.0.3", - "sebastian/environment": "^5.1.3", - "sebastian/exporter": "^4.0.3", - "sebastian/global-state": "^5.0.1", - "sebastian/object-enumerator": "^4.0.3", - "sebastian/resource-operations": "^3.0.3", - "sebastian/type": "^3.0", + "phpunit/php-text-template": "^2.0.4", + "phpunit/php-timer": "^5.0.3", + "sebastian/cli-parser": "^1.0.2", + "sebastian/code-unit": "^1.0.8", + "sebastian/comparator": "^4.0.8", + "sebastian/diff": "^4.0.6", + "sebastian/environment": "^5.1.5", + "sebastian/exporter": "^4.0.6", + "sebastian/global-state": "^5.0.7", + "sebastian/object-enumerator": "^4.0.4", + "sebastian/resource-operations": "^3.0.4", + "sebastian/type": "^3.2.1", "sebastian/version": "^3.0.2" }, - "require-dev": { - "ext-pdo": "*", - "phpspec/prophecy-phpunit": "^2.0.1" - }, "suggest": { - "ext-soap": "*", - "ext-xdebug": "*" + "ext-soap": "To be able to generate mocks based on WSDL files", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" }, "bin": [ "phpunit" @@ -6613,7 +6271,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "9.5-dev" + "dev-master": "9.6-dev" } }, "autoload": { @@ -6644,7 +6302,8 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.5.20" + "security": "https://github.com/sebastianbergmann/phpunit/security/policy", + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.22" }, "funding": [ { @@ -6654,9 +6313,13 @@ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpunit/phpunit", + "type": "tidelift" } ], - "time": "2022-04-01T12:37:26+00:00" + "time": "2024-12-05T13:48:26+00:00" }, { "name": "psr/cache", @@ -8520,16 +8183,16 @@ }, { "name": "textalk/websocket", - "version": "1.5.7", + "version": "1.5.8", "source": { "type": "git", "url": "https://github.com/Textalk/websocket-php.git", - "reference": "1712325e99b6bf869ccbf9bf41ab749e7328ea46" + "reference": "d05dbaa97500176447ffb1f1800573f23085ab13" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Textalk/websocket-php/zipball/1712325e99b6bf869ccbf9bf41ab749e7328ea46", - "reference": "1712325e99b6bf869ccbf9bf41ab749e7328ea46", + "url": "https://api.github.com/repos/Textalk/websocket-php/zipball/d05dbaa97500176447ffb1f1800573f23085ab13", + "reference": "d05dbaa97500176447ffb1f1800573f23085ab13", "shasum": "" }, "require": { @@ -8563,9 +8226,9 @@ "description": "WebSocket client and server", "support": { "issues": "https://github.com/Textalk/websocket-php/issues", - "source": "https://github.com/Textalk/websocket-php/tree/1.5.7" + "source": "https://github.com/Textalk/websocket-php/tree/1.5.8" }, - "time": "2022-03-29T09:46:59+00:00" + "time": "2022-04-26T06:28:24+00:00" }, { "name": "theseer/tokenizer", diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index f645463aed..977aff3cfb 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -7,7 +7,6 @@ use Exception; use Throwable; use Utopia\Audit\Audit; use Utopia\CLI\Console; -use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; @@ -126,22 +125,22 @@ class Audits extends Action } if ($shouldProcessBatch) { - foreach ($this->logs as $projectLogs) { - $dbForProject = $getProjectDB($projectLogs['project']); + try { + foreach ($this->logs as $internalId => $projectLogs) { + $dbForProject = $getProjectDB($projectLogs['project']); - Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); - $audit = new Audit($dbForProject); + Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); + $audit = new Audit($dbForProject); - try { $audit->logBatch($projectLogs['logs']); Console::success('Audit logs processed successfully'); - } catch (Throwable $e) { - Console::error('Error processing audit logs: ' . $e->getMessage()); - } finally { - // Clear the pending events after successful batch processing - $this->logs = []; - $this->lastTriggeredTime = time(); + + unset($this->logs[$internalId]); } + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + } finally { + $this->lastTriggeredTime = time(); } } } From 5efd2318616db5622729ba318984373845f1ff18 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Feb 2025 16:56:13 +1300 Subject: [PATCH 058/275] Allow validator to read logs collections from base --- .../Utopia/Database/Validator/Queries/Base.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index af5d59ddfd..e64c55cf63 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -24,7 +24,15 @@ class Base extends Queries public function __construct(string $collection, array $allowedAttributes) { $config = Config::getParam('collections', []); - $collections = array_merge($config['projects'], $config['buckets'], $config['databases'], $config['console']); + + $collections = array_merge( + $config['projects'], + $config['buckets'], + $config['databases'], + $config['console'], + $config['log'] + ); + $collection = $collections[$collection]; // array for constant lookup time $allowedAttributesLookup = []; @@ -35,6 +43,7 @@ class Base extends Queries $attributes = []; foreach ($collection['attributes'] as $attribute) { $key = $attribute['$id']; + if (!isset($allowedAttributesLookup[$key])) { continue; } From 1f560d896fcd962e783a253d8eff9567487b2851 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Feb 2025 16:58:30 +1300 Subject: [PATCH 059/275] Fix plural --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index e64c55cf63..b8cff64214 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -30,7 +30,7 @@ class Base extends Queries $config['buckets'], $config['databases'], $config['console'], - $config['log'] + $config['logs'] ); $collection = $collections[$collection]; From 7ac4a5bf271b7079210996c01b39bc186aa4f207 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 25 Feb 2025 04:43:44 +0000 Subject: [PATCH 060/275] chore: update naming --- app/controllers/api/storage.php | 6 +++--- src/Appwrite/Utopia/Response/Model/UsageBuckets.php | 4 ++-- tests/e2e/Services/Storage/StorageConsoleClientTest.php | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index efd82ba40e..072a2d4ae4 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1894,7 +1894,7 @@ App::get('/v1/storage/:bucketId/usage') $metrics = [ str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES), str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE), - str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_FILES_TRANSFORMATIONS), + str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_FILES_IMAGES_TRANSFORMED), ]; @@ -1949,7 +1949,7 @@ App::get('/v1/storage/:bucketId/usage') 'filesStorageTotal' => $usage[$metrics[1]]['total'], 'files' => $usage[$metrics[0]]['data'], 'storage' => $usage[$metrics[1]]['data'], - 'fileTransformations' => $usage[$metrics[2]]['data'], - 'fileTransformationsTotal' => $usage[$metrics[2]]['total'], + 'imageTransformations' => $usage[$metrics[2]]['data'], + 'imageTransformationsTotal' => $usage[$metrics[2]]['total'], ]), Response::MODEL_USAGE_BUCKETS); }); diff --git a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php index ab8d129857..ee624a29ad 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php +++ b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php @@ -42,14 +42,14 @@ class UsageBuckets extends Model 'example' => [], 'array' => true ]) - ->addRule('fileTransformations', [ + ->addRule('imageTransformations', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of files transformations per period.', 'default' => [], 'example' => [], 'array' => true ]) - ->addRule('fileTransformationsTotal', [ + ->addRule('imageTransformationsTotal', [ 'type' => self::TYPE_INTEGER, 'description' => 'Total aggregated number of files transformations.', 'default' => 0, diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index b2624746ca..bbb14fb136 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -104,7 +104,7 @@ class StorageConsoleClientTest extends Scope $this->assertIsNumeric($response['body']['filesStorageTotal']); $this->assertIsArray($response['body']['files']); $this->assertIsArray($response['body']['storage']); - $this->assertIsArray($response['body']['fileTransformations']); - $this->assertIsNumeric($response['body']['fileTransformationsTotal']); + $this->assertIsArray($response['body']['imageTransformations']); + $this->assertIsNumeric($response['body']['imageTransformationsTotal']); } } From ab6c5c9f8dbecff758429d86abe5e33e9afe53ae Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 25 Feb 2025 04:47:35 +0000 Subject: [PATCH 061/275] chore: bump utopia messaging --- composer.json | 2 +- composer.lock | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/composer.json b/composer.json index e182ac0fff..748b3e5649 100644 --- a/composer.json +++ b/composer.json @@ -59,7 +59,7 @@ "utopia-php/image": "0.8.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", - "utopia-php/messaging": "0.14.*", + "utopia-php/messaging": "0.16.*", "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.3", diff --git a/composer.lock b/composer.lock index 123ff3eade..2a3bb56ed4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b1d26d441fe1fb6a26978d99ef01985f", + "content-hash": "5e45619efebf755a1753eb91535bf387", "packages": [ { "name": "adhocore/jwt", @@ -3919,16 +3919,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.16", + "version": "0.33.17", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "e91d4c560d1b809e25faa63d564fef034363b50f" + "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/e91d4c560d1b809e25faa63d564fef034363b50f", - "reference": "e91d4c560d1b809e25faa63d564fef034363b50f", + "url": "https://api.github.com/repos/utopia-php/http/zipball/73fac6fbce9f56282dba4e52a58cf836ec434644", + "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644", "shasum": "" }, "require": { @@ -3960,9 +3960,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.16" + "source": "https://github.com/utopia-php/http/tree/0.33.17" }, - "time": "2025-01-16T15:58:50+00:00" + "time": "2025-02-24T17:35:48+00:00" }, { "name": "utopia-php/image", @@ -4120,16 +4120,16 @@ }, { "name": "utopia-php/messaging", - "version": "0.14.1", + "version": "0.16.0", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "4ba356a3aa382802727f7e13e0f0152bcc1fc535" + "reference": "5f3083697102b1821d6624938186761b1e09c54e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/4ba356a3aa382802727f7e13e0f0152bcc1fc535", - "reference": "4ba356a3aa382802727f7e13e0f0152bcc1fc535", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/5f3083697102b1821d6624938186761b1e09c54e", + "reference": "5f3083697102b1821d6624938186761b1e09c54e", "shasum": "" }, "require": { @@ -4165,9 +4165,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.14.1" + "source": "https://github.com/utopia-php/messaging/tree/0.16.0" }, - "time": "2025-01-28T06:14:28+00:00" + "time": "2025-02-18T08:27:00+00:00" }, { "name": "utopia-php/migration", From 6f854b5eb6ba084ca6dc3aef819fe0b9c685376a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Feb 2025 18:18:25 +1300 Subject: [PATCH 062/275] Remove blocks --- app/controllers/api/account.php | 6 ++---- app/controllers/api/databases.php | 18 ++++++------------ app/controllers/api/messaging.php | 24 ++++++++---------------- app/controllers/api/teams.php | 6 ++---- app/controllers/api/users.php | 6 ++---- 5 files changed, 20 insertions(+), 40 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index bef26ad25c..21a04800b5 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2750,10 +2750,8 @@ App::get('/v1/account/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByUser($user->getInternalId()), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByUser($user->getInternalId()), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index d9d62eac51..c563d1e8e0 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -723,10 +723,8 @@ App::get('/v1/databases/:databaseId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -1114,10 +1112,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -3767,10 +3763,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 06d49394aa..d7d3750ccf 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -1047,10 +1047,8 @@ App::get('/v1/messaging/providers/:providerId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -2278,10 +2276,8 @@ App::get('/v1/messaging/topics/:topicId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -2690,10 +2686,8 @@ App::get('/v1/messaging/subscribers/:subscriberId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -3457,10 +3451,8 @@ App::get('/v1/messaging/messages/:messageId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 7a9f21d74b..763aca64f0 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1415,9 +1415,7 @@ App::get('/v1/teams/:teamId/logs') } } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index ca3f042ba4..962022927f 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -915,10 +915,8 @@ App::get('/v1/users/:userId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByUser($user->getInternalId()), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByUser($user->getInternalId()), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); From f49346485ad303e5a8b989d5519feeb62f509952 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Feb 2025 18:39:12 +1300 Subject: [PATCH 063/275] Revert "Remove blocks" This reverts commit 6f854b5eb6ba084ca6dc3aef819fe0b9c685376a. --- app/controllers/api/account.php | 6 ++++-- app/controllers/api/databases.php | 18 ++++++++++++------ app/controllers/api/messaging.php | 24 ++++++++++++++++-------- app/controllers/api/teams.php | 6 ++++-- app/controllers/api/users.php | 6 ++++-- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 21a04800b5..bef26ad25c 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2750,8 +2750,10 @@ App::get('/v1/account/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getInternalId()), - 'logs' => $output, + //'total' => $audit->countLogsByUser($user->getInternalId()), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index c563d1e8e0..d9d62eac51 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -723,8 +723,10 @@ App::get('/v1/databases/:databaseId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); @@ -1112,8 +1114,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); @@ -3763,8 +3767,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index d7d3750ccf..06d49394aa 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -1047,8 +1047,10 @@ App::get('/v1/messaging/providers/:providerId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); @@ -2276,8 +2278,10 @@ App::get('/v1/messaging/topics/:topicId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); @@ -2686,8 +2690,10 @@ App::get('/v1/messaging/subscribers/:subscriberId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); @@ -3451,8 +3457,10 @@ App::get('/v1/messaging/messages/:messageId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 763aca64f0..7a9f21d74b 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1415,7 +1415,9 @@ App::get('/v1/teams/:teamId/logs') } } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), - 'logs' => $output, + //'total' => $audit->countLogsByResource($resource), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 962022927f..ca3f042ba4 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -915,8 +915,10 @@ App::get('/v1/users/:userId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getInternalId()), - 'logs' => $output, + //'total' => $audit->countLogsByUser($user->getInternalId()), + //'logs' => $output, + 'total' => 0, + 'logs' => [], ]), Response::MODEL_LOG_LIST); }); From b0547015a976a12f11275878e51335c94f99c9a1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Feb 2025 18:51:11 +1300 Subject: [PATCH 064/275] Add temp log date skip --- app/controllers/api/account.php | 4 ++++ app/controllers/api/databases.php | 20 +++++++++++++++++++- app/controllers/api/messaging.php | 16 ++++++++++++++++ app/controllers/api/teams.php | 4 ++++ app/controllers/api/users.php | 4 ++++ 5 files changed, 47 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index bef26ad25c..e96df1fa4f 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2715,6 +2715,10 @@ App::get('/v1/account/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } + // Temp fix for logs, remove after 1 log cycle + $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); + $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + $grouped = Query::groupByType($queries); $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; $offset = $grouped['offset'] ?? 0; diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index d9d62eac51..b6c3af4a52 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -23,6 +23,7 @@ use Utopia\App; use Utopia\Audit\Audit; use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Conflict as ConflictException; @@ -669,6 +670,10 @@ App::get('/v1/databases/:databaseId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } + // Temp fix for logs, remove after 1 log cycle + $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); + $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + $grouped = Query::groupByType($queries); $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; $offset = $grouped['offset'] ?? 0; @@ -1059,7 +1064,16 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $queries = Query::parseQueries($queries); + try { + $queries = Query::parseQueries($queries); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + // Temp fix for logs, remove after 1 log cycle + $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); + $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + $grouped = Query::groupByType($queries); $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; $offset = $grouped['offset'] ?? 0; @@ -3713,6 +3727,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } + // Temp fix for logs, remove after 1 log cycle + $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); + $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + $grouped = Query::groupByType($queries); $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; $offset = $grouped['offset'] ?? 0; diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 06d49394aa..53257a62c9 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -994,6 +994,10 @@ App::get('/v1/messaging/providers/:providerId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } + // Temp fix for logs, remove after 1 log cycle + $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); + $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + $grouped = Query::groupByType($queries); $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; $offset = $grouped['offset'] ?? 0; @@ -2224,6 +2228,10 @@ App::get('/v1/messaging/topics/:topicId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } + // Temp fix for logs, remove after 1 log cycle + $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); + $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + $grouped = Query::groupByType($queries); $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; $offset = $grouped['offset'] ?? 0; @@ -2636,6 +2644,10 @@ App::get('/v1/messaging/subscribers/:subscriberId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } + // Temp fix for logs, remove after 1 log cycle + $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); + $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + $grouped = Query::groupByType($queries); $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; $offset = $grouped['offset'] ?? 0; @@ -3403,6 +3415,10 @@ App::get('/v1/messaging/messages/:messageId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } + // Temp fix for logs, remove after 1 log cycle + $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); + $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + $grouped = Query::groupByType($queries); $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; $offset = $grouped['offset'] ?? 0; diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 7a9f21d74b..cde4171550 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1362,6 +1362,10 @@ App::get('/v1/teams/:teamId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } + // Temp fix for logs, remove after 1 log cycle + $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); + $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + $grouped = Query::groupByType($queries); $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; $offset = $grouped['offset'] ?? 0; diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index ca3f042ba4..021ae6554a 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -862,6 +862,10 @@ App::get('/v1/users/:userId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } + // Temp fix for logs, remove after 1 log cycle + $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); + $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + $grouped = Query::groupByType($queries); $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; $offset = $grouped['offset'] ?? 0; From 2924f408b320ef8eec18a5b242d23e9364948dc6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Feb 2025 19:21:35 +1300 Subject: [PATCH 065/275] Reapply "Remove blocks" This reverts commit f49346485ad303e5a8b989d5519feeb62f509952. --- app/controllers/api/account.php | 6 ++---- app/controllers/api/databases.php | 18 ++++++------------ app/controllers/api/messaging.php | 24 ++++++++---------------- app/controllers/api/teams.php | 6 ++---- app/controllers/api/users.php | 6 ++---- 5 files changed, 20 insertions(+), 40 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index e96df1fa4f..dcf5d5c3ab 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2754,10 +2754,8 @@ App::get('/v1/account/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByUser($user->getInternalId()), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByUser($user->getInternalId()), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index b6c3af4a52..9d93ba52a9 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -728,10 +728,8 @@ App::get('/v1/databases/:databaseId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -1128,10 +1126,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -3785,10 +3781,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 53257a62c9..2960008cd0 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -1051,10 +1051,8 @@ App::get('/v1/messaging/providers/:providerId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -2286,10 +2284,8 @@ App::get('/v1/messaging/topics/:topicId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -2702,10 +2698,8 @@ App::get('/v1/messaging/subscribers/:subscriberId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -3473,10 +3467,8 @@ App::get('/v1/messaging/messages/:messageId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index cde4171550..6df62eccb2 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1419,9 +1419,7 @@ App::get('/v1/teams/:teamId/logs') } } $response->dynamic(new Document([ - //'total' => $audit->countLogsByResource($resource), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByResource($resource), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 021ae6554a..6b39d12d42 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -919,10 +919,8 @@ App::get('/v1/users/:userId/logs') } $response->dynamic(new Document([ - //'total' => $audit->countLogsByUser($user->getInternalId()), - //'logs' => $output, - 'total' => 0, - 'logs' => [], + 'total' => $audit->countLogsByUser($user->getInternalId()), + 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); From 676431c355d93597d34853b95c286d9bd73525e3 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 25 Feb 2025 07:16:42 +0000 Subject: [PATCH 066/275] chore: shifted to logsDb --- app/controllers/api/storage.php | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 072a2d4ae4..972232167b 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1879,9 +1879,12 @@ App::get('/v1/storage/:bucketId/usage') ->param('bucketId', '', new UID(), 'Bucket ID.') ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) ->inject('response') + ->inject('project') ->inject('dbForProject') - ->action(function (string $bucketId, string $range, Response $response, Database $dbForProject) { + ->inject('getLogsDB') + ->action(function (string $bucketId, string $range, Response $response, Document $project, Database $dbForProject, callable $getLogsDB) { + $dbForLogs = call_user_func($getLogsDB, $project); $bucket = $dbForProject->getDocument('buckets', $bucketId); if ($bucket->isEmpty()) { @@ -1894,13 +1897,16 @@ App::get('/v1/storage/:bucketId/usage') $metrics = [ str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES), str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE), - str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_FILES_IMAGES_TRANSFORMED), + str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED), ]; - - Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) { + Authorization::skip(function () use ($dbForProject, $dbForLogs, $bucket, $days, $metrics, &$stats) { foreach ($metrics as $metric) { - $result = $dbForProject->findOne('stats', [ + $db = ($metric === str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED)) + ? $dbForLogs + : $dbForProject; + + $result = $db->findOne('stats', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -1908,7 +1914,7 @@ App::get('/v1/storage/:bucketId/usage') $stats[$metric]['total'] = $result['value'] ?? 0; $limit = $days['limit']; $period = $days['period']; - $results = $dbForProject->find('stats', [ + $results = $db->find('stats', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::limit($limit), From b272a8b1e711210c699d2261e35d75acd34f6d14 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Feb 2025 20:26:00 +1300 Subject: [PATCH 067/275] Pass queries --- app/controllers/api/account.php | 8 ++--- app/controllers/api/databases.php | 24 ++++--------- app/controllers/api/messaging.php | 32 +++++------------- app/controllers/api/teams.php | 8 ++--- app/controllers/api/users.php | 8 ++--- composer.json | 6 ++-- composer.lock | 56 +++++++++++++++---------------- 7 files changed, 51 insertions(+), 91 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index dcf5d5c3ab..e670c68bc4 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2719,13 +2719,9 @@ App::get('/v1/account/logs') $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); - $grouped = Query::groupByType($queries); - $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; - $offset = $grouped['offset'] ?? 0; - $audit = new EventAudit($dbForProject); - $logs = $audit->getLogsByUser($user->getInternalId(), $limit, $offset); + $logs = $audit->getLogsByUser($user->getInternalId(), $queries); $output = []; @@ -2754,7 +2750,7 @@ App::get('/v1/account/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getInternalId()), + 'total' => $audit->countLogsByUser($user->getInternalId(), $queries), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 9d93ba52a9..8980300aef 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -674,13 +674,9 @@ App::get('/v1/databases/:databaseId/logs') $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); - $grouped = Query::groupByType($queries); - $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; - $offset = $grouped['offset'] ?? 0; - $audit = new Audit($dbForProject); $resource = 'database/' . $databaseId; - $logs = $audit->getLogsByResource($resource, $limit, $offset); + $logs = $audit->getLogsByResource($resource, $queries); $output = []; @@ -728,7 +724,7 @@ App::get('/v1/databases/:databaseId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), + 'total' => $audit->countLogsByResource($resource, $queries), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -1072,13 +1068,9 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); - $grouped = Query::groupByType($queries); - $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; - $offset = $grouped['offset'] ?? 0; - $audit = new Audit($dbForProject); $resource = 'database/' . $databaseId . '/collection/' . $collectionId; - $logs = $audit->getLogsByResource($resource, $limit, $offset); + $logs = $audit->getLogsByResource($resource, $queries); $output = []; @@ -1126,7 +1118,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), + 'total' => $audit->countLogsByResource($resource, $queries), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -3727,13 +3719,9 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); - $grouped = Query::groupByType($queries); - $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; - $offset = $grouped['offset'] ?? 0; - $audit = new Audit($dbForProject); $resource = 'database/' . $databaseId . '/collection/' . $collectionId . '/document/' . $document->getId(); - $logs = $audit->getLogsByResource($resource, $limit, $offset); + $logs = $audit->getLogsByResource($resource, $queries); $output = []; @@ -3781,7 +3769,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), + 'total' => $audit->countLogsByResource($resource, $queries), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 2960008cd0..1c9efb6193 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -998,13 +998,9 @@ App::get('/v1/messaging/providers/:providerId/logs') $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); - $grouped = Query::groupByType($queries); - $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; - $offset = $grouped['offset'] ?? 0; - $audit = new Audit($dbForProject); $resource = 'provider/' . $providerId; - $logs = $audit->getLogsByResource($resource, $limit, $offset); + $logs = $audit->getLogsByResource($resource, $queries); $output = []; foreach ($logs as $i => &$log) { @@ -1051,7 +1047,7 @@ App::get('/v1/messaging/providers/:providerId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), + 'total' => $audit->countLogsByResource($resource, $queries), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -2230,13 +2226,9 @@ App::get('/v1/messaging/topics/:topicId/logs') $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); - $grouped = Query::groupByType($queries); - $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; - $offset = $grouped['offset'] ?? 0; - $audit = new Audit($dbForProject); $resource = 'topic/' . $topicId; - $logs = $audit->getLogsByResource($resource, $limit, $offset); + $logs = $audit->getLogsByResource($resource, $queries); $output = []; @@ -2284,7 +2276,7 @@ App::get('/v1/messaging/topics/:topicId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), + 'total' => $audit->countLogsByResource($resource, $queries), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -2644,13 +2636,9 @@ App::get('/v1/messaging/subscribers/:subscriberId/logs') $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); - $grouped = Query::groupByType($queries); - $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; - $offset = $grouped['offset'] ?? 0; - $audit = new Audit($dbForProject); $resource = 'subscriber/' . $subscriberId; - $logs = $audit->getLogsByResource($resource, $limit, $offset); + $logs = $audit->getLogsByResource($resource, $queries); $output = []; @@ -2698,7 +2686,7 @@ App::get('/v1/messaging/subscribers/:subscriberId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), + 'total' => $audit->countLogsByResource($resource, $queries), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); @@ -3413,13 +3401,9 @@ App::get('/v1/messaging/messages/:messageId/logs') $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); - $grouped = Query::groupByType($queries); - $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; - $offset = $grouped['offset'] ?? 0; - $audit = new Audit($dbForProject); $resource = 'message/' . $messageId; - $logs = $audit->getLogsByResource($resource, $limit, $offset); + $logs = $audit->getLogsByResource($resource, $queries); $output = []; @@ -3467,7 +3451,7 @@ App::get('/v1/messaging/messages/:messageId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), + 'total' => $audit->countLogsByResource($resource, $queries), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 6df62eccb2..9defb9b013 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1366,13 +1366,9 @@ App::get('/v1/teams/:teamId/logs') $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); - $grouped = Query::groupByType($queries); - $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; - $offset = $grouped['offset'] ?? 0; - $audit = new Audit($dbForProject); $resource = 'team/' . $team->getId(); - $logs = $audit->getLogsByResource($resource, $limit, $offset); + $logs = $audit->getLogsByResource($resource, $queries); $output = []; @@ -1419,7 +1415,7 @@ App::get('/v1/teams/:teamId/logs') } } $response->dynamic(new Document([ - 'total' => $audit->countLogsByResource($resource), + 'total' => $audit->countLogsByResource($resource, $queries), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 6b39d12d42..aab41991f3 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -866,13 +866,9 @@ App::get('/v1/users/:userId/logs') $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); - $grouped = Query::groupByType($queries); - $limit = $grouped['limit'] ?? APP_LIMIT_COUNT; - $offset = $grouped['offset'] ?? 0; - $audit = new Audit($dbForProject); - $logs = $audit->getLogsByUser($user->getInternalId(), $limit, $offset); + $logs = $audit->getLogsByUser($user->getInternalId(), $queries); $output = []; @@ -919,7 +915,7 @@ App::get('/v1/users/:userId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getInternalId()), + 'total' => $audit->countLogsByUser($user->getInternalId(), $queries), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); diff --git a/composer.json b/composer.json index 640faba78a..6180773b24 100644 --- a/composer.json +++ b/composer.json @@ -45,13 +45,13 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "0.50.*", + "utopia-php/abuse": "0.51.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.51.*", + "utopia-php/audit": "0.54.0", "utopia-php/cache": "0.11.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.59.0", + "utopia-php/database": "0.60.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index ea2e425a12..841d5b51c5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ccd64a2bc1115b7f02225c8c06b24483", + "content-hash": "87a12bb6f58c5dad7488229590e9a3f5", "packages": [ { "name": "adhocore/jwt", @@ -3377,16 +3377,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.50.0", + "version": "0.51.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "3ff67819e9de61506c5ca070a70552f7ebe99f80" + "reference": "661687b03277f1d202a0e8cf9da6e58c97da2b5e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/3ff67819e9de61506c5ca070a70552f7ebe99f80", - "reference": "3ff67819e9de61506c5ca070a70552f7ebe99f80", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/661687b03277f1d202a0e8cf9da6e58c97da2b5e", + "reference": "661687b03277f1d202a0e8cf9da6e58c97da2b5e", "shasum": "" }, "require": { @@ -3394,7 +3394,7 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/database": "0.59.*" + "utopia-php/database": "0.60.*" }, "require-dev": { "laravel/pint": "1.*", @@ -3422,9 +3422,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.50.0" + "source": "https://github.com/utopia-php/abuse/tree/0.51.0" }, - "time": "2025-02-12T09:13:59+00:00" + "time": "2025-02-17T11:10:18+00:00" }, { "name": "utopia-php/analytics", @@ -3474,21 +3474,21 @@ }, { "name": "utopia-php/audit", - "version": "0.51.0", + "version": "0.54.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "a5a4b73a57e27a0fac8025b1d6038e145a1ca04e" + "reference": "1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/a5a4b73a57e27a0fac8025b1d6038e145a1ca04e", - "reference": "a5a4b73a57e27a0fac8025b1d6038e145a1ca04e", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975", + "reference": "1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/database": "0.59.*" + "utopia-php/database": "0.60.*" }, "require-dev": { "laravel/pint": "1.*", @@ -3515,9 +3515,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.51.0" + "source": "https://github.com/utopia-php/audit/tree/0.54.0" }, - "time": "2025-02-12T09:12:44+00:00" + "time": "2025-02-25T07:21:07+00:00" }, { "name": "utopia-php/cache", @@ -3717,16 +3717,16 @@ }, { "name": "utopia-php/database", - "version": "0.59.0", + "version": "0.60.3", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "0eed7f1ad3eb66ff4a7d73b68dd9d3e05089eb18" + "reference": "c4bc4af3f09a91aea76aac75b4b78fa06598c61d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/0eed7f1ad3eb66ff4a7d73b68dd9d3e05089eb18", - "reference": "0eed7f1ad3eb66ff4a7d73b68dd9d3e05089eb18", + "url": "https://api.github.com/repos/utopia-php/database/zipball/c4bc4af3f09a91aea76aac75b4b78fa06598c61d", + "reference": "c4bc4af3f09a91aea76aac75b4b78fa06598c61d", "shasum": "" }, "require": { @@ -3767,9 +3767,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.59.0" + "source": "https://github.com/utopia-php/database/tree/0.60.3" }, - "time": "2025-02-12T08:08:29+00:00" + "time": "2025-02-17T12:46:59+00:00" }, { "name": "utopia-php/domains", @@ -4171,16 +4171,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.19", + "version": "0.6.20", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "3c9497f7a54ef88b1077c48d8326893133ad78eb" + "reference": "8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/3c9497f7a54ef88b1077c48d8326893133ad78eb", - "reference": "3c9497f7a54ef88b1077c48d8326893133ad78eb", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba", + "reference": "8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba", "shasum": "" }, "require": { @@ -4188,7 +4188,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.59.*", + "utopia-php/database": "0.60.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4221,9 +4221,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.19" + "source": "https://github.com/utopia-php/migration/tree/0.6.20" }, - "time": "2025-02-13T07:50:21+00:00" + "time": "2025-02-17T11:02:15+00:00" }, { "name": "utopia-php/mongo", From 03f0e382f4192ed75cbe9f4bb0f27ad561dd30ed Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Feb 2025 21:03:37 +1300 Subject: [PATCH 068/275] Fix dates --- app/controllers/api/account.php | 8 +++-- app/controllers/api/databases.php | 24 ++++++++------ app/controllers/api/messaging.php | 32 ++++++++++++------- app/controllers/api/teams.php | 8 +++-- app/controllers/api/users.php | 8 +++-- .../Account/AccountCustomClientTest.php | 1 + 6 files changed, 51 insertions(+), 30 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index e670c68bc4..438a036274 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2715,9 +2715,11 @@ App::get('/v1/account/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - // Temp fix for logs, remove after 1 log cycle - $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); - $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + // Temp fix for logs + $queries[] = Query::or([ + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), + ]); $audit = new EventAudit($dbForProject); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 8980300aef..3cc9c110c0 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -670,9 +670,11 @@ App::get('/v1/databases/:databaseId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - // Temp fix for logs, remove after 1 log cycle - $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); - $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + // Temp fix for logs + $queries[] = Query::or([ + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), + ]); $audit = new Audit($dbForProject); $resource = 'database/' . $databaseId; @@ -1064,9 +1066,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - // Temp fix for logs, remove after 1 log cycle - $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); - $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + // Temp fix for logs + $queries[] = Query::or([ + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), + ]); $audit = new Audit($dbForProject); $resource = 'database/' . $databaseId . '/collection/' . $collectionId; @@ -3715,9 +3719,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - // Temp fix for logs, remove after 1 log cycle - $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); - $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + // Temp fix for logs + $queries[] = Query::or([ + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), + ]); $audit = new Audit($dbForProject); $resource = 'database/' . $databaseId . '/collection/' . $collectionId . '/document/' . $document->getId(); diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 1c9efb6193..2e89863ec3 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -994,9 +994,11 @@ App::get('/v1/messaging/providers/:providerId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - // Temp fix for logs, remove after 1 log cycle - $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); - $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + // Temp fix for logs + $queries[] = Query::or([ + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), + ]); $audit = new Audit($dbForProject); $resource = 'provider/' . $providerId; @@ -2222,9 +2224,11 @@ App::get('/v1/messaging/topics/:topicId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - // Temp fix for logs, remove after 1 log cycle - $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); - $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + // Temp fix for logs + $queries[] = Query::or([ + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), + ]); $audit = new Audit($dbForProject); $resource = 'topic/' . $topicId; @@ -2632,9 +2636,11 @@ App::get('/v1/messaging/subscribers/:subscriberId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - // Temp fix for logs, remove after 1 log cycle - $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); - $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + // Temp fix for logs + $queries[] = Query::or([ + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), + ]); $audit = new Audit($dbForProject); $resource = 'subscriber/' . $subscriberId; @@ -3397,9 +3403,11 @@ App::get('/v1/messaging/messages/:messageId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - // Temp fix for logs, remove after 1 log cycle - $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); - $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + // Temp fix for logs + $queries[] = Query::or([ + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), + ]); $audit = new Audit($dbForProject); $resource = 'message/' . $messageId; diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 9defb9b013..815bd6db1d 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1362,9 +1362,11 @@ App::get('/v1/teams/:teamId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - // Temp fix for logs, remove after 1 log cycle - $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); - $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + // Temp fix for logs + $queries[] = Query::or([ + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), + ]); $audit = new Audit($dbForProject); $resource = 'team/' . $team->getId(); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index aab41991f3..5e58a610cf 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -862,9 +862,11 @@ App::get('/v1/users/:userId/logs') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - // Temp fix for logs, remove after 1 log cycle - $queries[] = Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-126T00:00+0'))); - $queries[] = Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+0'))); + // Temp fix for logs + $queries[] = Query::or([ + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), + ]); $audit = new Audit($dbForProject); diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 788f949fb3..439fa24fb6 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -277,6 +277,7 @@ class AccountCustomClientTest extends Scope { sleep(5); $session = $data['session'] ?? ''; + /** * Test for SUCCESS */ From 697c30f559a57cb98728ce7d1de5b212b8df281f Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 25 Feb 2025 09:31:24 +0100 Subject: [PATCH 069/275] Init --- app/config/locale/continents.php | 49 +++- app/config/locale/countries.php | 397 ++++++++++++++++--------------- app/controllers/api/locale.php | 4 +- 3 files changed, 243 insertions(+), 207 deletions(-) diff --git a/app/config/locale/continents.php b/app/config/locale/continents.php index 2f1ffc0a53..4033c9ff5a 100644 --- a/app/config/locale/continents.php +++ b/app/config/locale/continents.php @@ -1,11 +1,46 @@ [ + 'name' => 'Africa', + 'latitude' => 8.7832, + 'longitude' => 34.5085 + ], + 'AN' => [ + 'name' => 'Antarctica', + 'latitude' => -82.8628, + 'longitude' => 135.0000 + ], + 'AS' => [ + 'name' => 'Asia', + 'latitude' => 34.0479, + 'longitude' => 100.6197 + ], + 'EU' => [ + 'name' => 'Europe', + 'latitude' => 54.5260, + 'longitude' => 15.2551 + ], + 'NA' => [ + 'name' => 'North America', + 'latitude' => 54.5260, + 'longitude' => -105.2551 + ], + 'OC' => [ + 'name' => 'Oceania', + 'latitude' => -22.7359, + 'longitude' => 140.0188 + ], + 'SA' => [ + 'name' => 'South America', + 'latitude' => -8.7832, + 'longitude' => -55.4915 + ], ]; diff --git a/app/config/locale/countries.php b/app/config/locale/countries.php index bd2cbbbaaa..adfc5d76a6 100644 --- a/app/config/locale/countries.php +++ b/app/config/locale/countries.php @@ -1,209 +1,210 @@ ['name' => 'Afghanistan', 'latitude' => 33.0, 'longitude' => 66.0], + 'AO' => ['name' => 'Angola', 'latitude' => -12.5, 'longitude' => 18.5], + 'AL' => ['name' => 'Albania', 'latitude' => 41.0, 'longitude' => 20.0], + 'AD' => ['name' => 'Andorra', 'latitude' => 42.5, 'longitude' => 1.6], + 'AE' => ['name' => 'United Arab Emirates', 'latitude' => 24.0, 'longitude' => 54.0], + 'AR' => ['name' => 'Argentina', 'latitude' => -34.0, 'longitude' => -64.0], + 'AM' => ['name' => 'Armenia', 'latitude' => 40.0, 'longitude' => 45.0], + 'AG' => ['name' => 'Antigua and Barbuda', 'latitude' => 17.05, 'longitude' => -61.8], + 'AU' => ['name' => 'Australia', 'latitude' => -25.0, 'longitude' => 135.0], + 'AT' => ['name' => 'Austria', 'latitude' => 47.3, 'longitude' => 13.3], + 'AZ' => ['name' => 'Azerbaijan', 'latitude' => 40.5, 'longitude' => 47.5], + 'BI' => ['name' => 'Burundi', 'latitude' => -3.5, 'longitude' => 30.0], + 'BE' => ['name' => 'Belgium', 'latitude' => 50.8, 'longitude' => 4.0], + 'BJ' => ['name' => 'Benin', 'latitude' => 9.5, 'longitude' => 2.25], + 'BF' => ['name' => 'Burkina Faso', 'latitude' => 13.0, 'longitude' => -2.0], + 'BD' => ['name' => 'Bangladesh', 'latitude' => 24.0, 'longitude' => 90.0], + 'BG' => ['name' => 'Bulgaria', 'latitude' => 43.0, 'longitude' => 25.0], + 'BH' => ['name' => 'Bahrain', 'latitude' => 26.0, 'longitude' => 50.5], + 'BS' => ['name' => 'Bahamas', 'latitude' => 24.25, 'longitude' => -76.0], + 'BA' => ['name' => 'Bosnia and Herzegovina', 'latitude' => 44.0, 'longitude' => 18.0], + 'BY' => ['name' => 'Belarus', 'latitude' => 53.0, 'longitude' => 28.0], + 'BZ' => ['name' => 'Belize', 'latitude' => 17.25, 'longitude' => -88.75], + 'BO' => ['name' => 'Bolivia', 'latitude' => -17.0, 'longitude' => -65.0], + 'BR' => ['name' => 'Brazil', 'latitude' => -10.0, 'longitude' => -55.0], + 'BB' => ['name' => 'Barbados', 'latitude' => 13.17, 'longitude' => -59.53], + 'BN' => ['name' => 'Brunei', 'latitude' => 4.5, 'longitude' => 114.67], + 'BT' => ['name' => 'Bhutan', 'latitude' => 27.5, 'longitude' => 90.5], + 'BW' => ['name' => 'Botswana', 'latitude' => -22.0, 'longitude' => 24.0], + 'CF' => ['name' => 'Central African Republic', 'latitude' => 7.0, 'longitude' => 21.0], + 'CA' => ['name' => 'Canada', 'latitude' => 60.0, 'longitude' => -95.0], + 'CH' => ['name' => 'Switzerland', 'latitude' => 47.0, 'longitude' => 8.0], + 'CL' => ['name' => 'Chile', 'latitude' => -30.0, 'longitude' => -71.0], + 'CN' => ['name' => 'China', 'latitude' => 35.0, 'longitude' => 105.0], + 'CI' => ['name' => 'Côte d\'Ivoire', 'latitude' => 8.0, 'longitude' => -5.0], + 'CM' => ['name' => 'Cameroon', 'latitude' => 6.0, 'longitude' => 12.0], + 'CD' => ['name' => 'Democratic Republic of the Congo', 'latitude' => -2.5, 'longitude' => 23.5], + 'CG' => ['name' => 'Republic of the Congo', 'latitude' => -1.0, 'longitude' => 15.0], + 'CO' => ['name' => 'Colombia', 'latitude' => 4.0, 'longitude' => -72.0], + 'KM' => ['name' => 'Comoros', 'latitude' => -12.17, 'longitude' => 44.25], + 'CV' => ['name' => 'Cape Verde', 'latitude' => 16.0, 'longitude' => -24.0], + 'CR' => ['name' => 'Costa Rica', 'latitude' => 10.0, 'longitude' => -84.0], + 'CU' => ['name' => 'Cuba', 'latitude' => 21.5, 'longitude' => -80.0], + 'CY' => ['name' => 'Cyprus', 'latitude' => 35.0, 'longitude' => 33.0], + 'CZ' => ['name' => 'Czech Republic', 'latitude' => 49.75, 'longitude' => 15.5], + 'DE' => ['name' => 'Germany', 'latitude' => 51.0, 'longitude' => 9.0], + 'DJ' => ['name' => 'Djibouti', 'latitude' => 11.5, 'longitude' => 43.0], + 'DM' => ['name' => 'Dominica', 'latitude' => 15.42, 'longitude' => -61.33], + 'DK' => ['name' => 'Denmark', 'latitude' => 56.0, 'longitude' => 10.0], + 'DO' => ['name' => 'Dominican Republic', 'latitude' => 19.0, 'longitude' => -70.67], + 'DZ' => ['name' => 'Algeria', 'latitude' => 28.0, 'longitude' => 3.0], + 'EC' => ['name' => 'Ecuador', 'latitude' => -2.0, 'longitude' => -77.5], + 'EG' => ['name' => 'Egypt', 'latitude' => 27.0, 'longitude' => 30.0], + 'ER' => ['name' => 'Eritrea', 'latitude' => 15.0, 'longitude' => 39.0], + 'ES' => ['name' => 'Spain', 'latitude' => 40.0, 'longitude' => -4.0], + 'EE' => ['name' => 'Estonia', 'latitude' => 59.0, 'longitude' => 26.0], + 'ET' => ['name' => 'Ethiopia', 'latitude' => 8.0, 'longitude' => 38.0], + 'FI' => ['name' => 'Finland', 'latitude' => 64.0, 'longitude' => 26.0], + 'FJ' => ['name' => 'Fiji', 'latitude' => -18.0, 'longitude' => 175.0], + 'FR' => ['name' => 'France', 'latitude' => 46.0, 'longitude' => 2.0], + 'FM' => ['name' => 'Micronesia', 'latitude' => 6.92, 'longitude' => 158.25], + 'GA' => ['name' => 'Gabon', 'latitude' => -1.0, 'longitude' => 11.75], + 'GB' => ['name' => 'United Kingdom', 'latitude' => 54.0, 'longitude' => -2.0], + 'GE' => ['name' => 'Georgia', 'latitude' => 42.0, 'longitude' => 43.5], + 'GH' => ['name' => 'Ghana', 'latitude' => 8.0, 'longitude' => -2.0], + 'GN' => ['name' => 'Guinea', 'latitude' => 11.0, 'longitude' => -10.0], + 'GM' => ['name' => 'Gambia', 'latitude' => 13.47, 'longitude' => -16.57], + 'GW' => ['name' => 'Guinea-Bissau', 'latitude' => 12.0, 'longitude' => -15.0], + 'GQ' => ['name' => 'Equatorial Guinea', 'latitude' => 2.0, 'longitude' => 10.0], + 'GR' => ['name' => 'Greece', 'latitude' => 39.0, 'longitude' => 22.0], + 'GD' => ['name' => 'Grenada', 'latitude' => 12.12, 'longitude' => -61.67], + 'GT' => ['name' => 'Guatemala', 'latitude' => 15.5, 'longitude' => -90.25], + 'GY' => ['name' => 'Guyana', 'latitude' => 5.0, 'longitude' => -59.0], + 'HK' => ['name' => 'Hong Kong', 'latitude' => 22.25, 'longitude' => 114.17], + 'HN' => ['name' => 'Honduras', 'latitude' => 15.0, 'longitude' => -86.5], + 'HR' => ['name' => 'Croatia', 'latitude' => 45.17, 'longitude' => 15.5], + 'HT' => ['name' => 'Haiti', 'latitude' => 19.0, 'longitude' => -72.42], + 'HU' => ['name' => 'Hungary', 'latitude' => 47.0, 'longitude' => 20.0], + 'ID' => ['name' => 'Indonesia', 'latitude' => -5.0, 'longitude' => 120.0], + 'IN' => ['name' => 'India', 'latitude' => 20.0, 'longitude' => 77.0], + 'IE' => ['name' => 'Ireland', 'latitude' => 53.0, 'longitude' => -8.0], + 'IR' => ['name' => 'Iran', 'latitude' => 32.0, 'longitude' => 53.0], + 'IQ' => ['name' => 'Iraq', 'latitude' => 33.0, 'longitude' => 44.0], + 'IS' => ['name' => 'Iceland', 'latitude' => 65.0, 'longitude' => -18.0], + 'IL' => ['name' => 'Israel', 'latitude' => 31.5, 'longitude' => 34.75], + 'IT' => ['name' => 'Italy', 'latitude' => 42.83, 'longitude' => 12.83], + 'JM' => ['name' => 'Jamaica', 'latitude' => 18.25, 'longitude' => -77.5], + 'JO' => ['name' => 'Jordan', 'latitude' => 31.0, 'longitude' => 36.0], + 'JP' => ['name' => 'Japan', 'latitude' => 36.0, 'longitude' => 138.0], + 'KZ' => ['name' => 'Kazakhstan', 'latitude' => 48.0, 'longitude' => 68.0], + 'KE' => ['name' => 'Kenya', 'latitude' => 1.0, 'longitude' => 38.0], + 'KG' => ['name' => 'Kyrgyzstan', 'latitude' => 41.0, 'longitude' => 75.0], + 'KH' => ['name' => 'Cambodia', 'latitude' => 13.0, 'longitude' => 105.0], + 'KI' => ['name' => 'Kiribati', 'latitude' => 1.42, 'longitude' => 173.0], + 'KN' => ['name' => 'Saint Kitts and Nevis', 'latitude' => 17.33, 'longitude' => -62.75], + 'KR' => ['name' => 'South Korea', 'latitude' => 37.0, 'longitude' => 127.5], + 'KW' => ['name' => 'Kuwait', 'latitude' => 29.34, 'longitude' => 47.66], + 'LA' => ['name' => 'Laos', 'latitude' => 18.0, 'longitude' => 105.0], + 'LB' => ['name' => 'Lebanon', 'latitude' => 33.83, 'longitude' => 35.83], + 'LR' => ['name' => 'Liberia', 'latitude' => 6.5, 'longitude' => -9.5], + 'LY' => ['name' => 'Libya', 'latitude' => 25.0, 'longitude' => 17.0], + 'LC' => ['name' => 'Saint Lucia', 'latitude' => 13.88, 'longitude' => -61.13], + 'LI' => ['name' => 'Liechtenstein', 'latitude' => 47.17, 'longitude' => 9.53], + 'LK' => ['name' => 'Sri Lanka', 'latitude' => 7.0, 'longitude' => 81.0], + 'LS' => ['name' => 'Lesotho', 'latitude' => -29.5, 'longitude' => 28.5], + 'LT' => ['name' => 'Lithuania', 'latitude' => 56.0, 'longitude' => 24.0], + 'LU' => ['name' => 'Luxembourg', 'latitude' => 49.75, 'longitude' => 6.17], + 'LV' => ['name' => 'latitudevia', 'latitude' => 57.0, 'longitude' => 25.0], + 'MA' => ['name' => 'Morocco', 'latitude' => 32.0, 'longitude' => -5.0], + 'MC' => ['name' => 'Monaco', 'latitude' => 43.73, 'longitude' => 7.4], + 'MD' => ['name' => 'Moldova', 'latitude' => 47.0, 'longitude' => 29.0], + 'MG' => ['name' => 'Madagascar', 'latitude' => -20.0, 'longitude' => 47.0], + 'MV' => ['name' => 'Maldives', 'latitude' => 3.25, 'longitude' => 73.0], + 'MX' => ['name' => 'Mexico', 'latitude' => 23.0, 'longitude' => -102.0], + 'MH' => ['name' => 'Marshall Islands', 'latitude' => 9.0, 'longitude' => 168.0], + 'MK' => ['name' => 'North Macedonia', 'latitude' => 41.83, 'longitude' => 22.0], + 'ML' => ['name' => 'Mali', 'latitude' => 17.0, 'longitude' => -4.0], + 'MT' => ['name' => 'Malta', 'latitude' => 35.83, 'longitude' => 14.58], + 'MM' => ['name' => 'Myanmar', 'latitude' => 22.0, 'longitude' => 98.0], + 'ME' => ['name' => 'Montenegro', 'latitude' => 42.5, 'longitude' => 19.3], + 'MN' => ['name' => 'Mongolia', 'latitude' => 46.0, 'longitude' => 105.0], + 'MZ' => ['name' => 'Mozambique', 'latitude' => -18.25, 'longitude' => 35.0], + 'MR' => ['name' => 'Mauritania', 'latitude' => 20.0, 'longitude' => -12.0], + 'MU' => ['name' => 'Mauritius', 'latitude' => -20.28, 'longitude' => 57.55], + 'MW' => ['name' => 'Malawi', 'latitude' => -13.5, 'longitude' => 34.0], + 'MY' => ['name' => 'Malaysia', 'latitude' => 2.5, 'longitude' => 112.5], + 'NA' => ['name' => 'Namibia', 'latitude' => -22.0, 'longitude' => 17.0], + 'NE' => ['name' => 'Niger', 'latitude' => 16.0, 'longitude' => 8.0], + 'NG' => ['name' => 'Nigeria', 'latitude' => 10.0, 'longitude' => 8.0], + 'NI' => ['name' => 'Nicaragua', 'latitude' => 13.0, 'longitude' => -85.0], + 'NL' => ['name' => 'Netherlands', 'latitude' => 52.5, 'longitude' => 5.75], + 'NO' => ['name' => 'Norway', 'latitude' => 62.0, 'longitude' => 10.0], + 'NP' => ['name' => 'Nepal', 'latitude' => 28.0, 'longitude' => 84.0], + 'NR' => ['name' => 'Nauru', 'latitude' => -0.53, 'longitude' => 166.92], + 'NZ' => ['name' => 'New Zealand', 'latitude' => -41.0, 'longitude' => 174.0], + 'OM' => ['name' => 'Oman', 'latitude' => 21.0, 'longitude' => 57.0], + 'PK' => ['name' => 'Pakistan', 'latitude' => 30.0, 'longitude' => 70.0], + 'PS' => ['name' => 'Palestine', 'latitude' => 31.9, 'longitude' => 35.2], + 'PA' => ['name' => 'Panama', 'latitude' => 9.0, 'longitude' => -80.0], + 'PE' => ['name' => 'Peru', 'latitude' => -10.0, 'longitude' => -76.0], + 'PH' => ['name' => 'Philippines', 'latitude' => 13.0, 'longitude' => 122.0], + 'PW' => ['name' => 'Palau', 'latitude' => 7.5, 'longitude' => 134.5], + 'PG' => ['name' => 'Papua New Guinea', 'latitude' => -6.0, 'longitude' => 147.0], + 'PL' => ['name' => 'Poland', 'latitude' => 52.0, 'longitude' => 20.0], + 'KP' => ['name' => 'North Korea', 'latitude' => 40.0, 'longitude' => 127.0], + 'PT' => ['name' => 'Portugal', 'latitude' => 39.5, 'longitude' => -8.0], + 'PY' => ['name' => 'Paraguay', 'latitude' => -23.0, 'longitude' => -58.0], + 'QA' => ['name' => 'Qatar', 'latitude' => 25.5, 'longitude' => 51.25], + 'RO' => ['name' => 'Romania', 'latitude' => 46.0, 'longitude' => 25.0], + 'RU' => ['name' => 'Russia', 'latitude' => 60.0, 'longitude' => 100.0], + 'RW' => ['name' => 'Rwanda', 'latitude' => -2.0, 'longitude' => 30.0], + 'SA' => ['name' => 'Saudi Arabia', 'latitude' => 25.0, 'longitude' => 45.0], + 'SD' => ['name' => 'Sudan', 'latitude' => 15.0, 'longitude' => 30.0], + 'SN' => ['name' => 'Senegal', 'latitude' => 14.0, 'longitude' => -14.0], + 'SG' => ['name' => 'Singapore', 'latitude' => 1.37, 'longitude' => 103.8], + 'SB' => ['name' => 'Solomon Islands', 'latitude' => -8.0, 'longitude' => 159.0], + 'SL' => ['name' => 'Sierra Leone', 'latitude' => 8.5, 'longitude' => -11.5], + 'SV' => ['name' => 'El Salvador', 'latitude' => 13.83, 'longitude' => -88.92], + 'SM' => ['name' => 'San Marino', 'latitude' => 43.77, 'longitude' => 12.42], + 'SO' => ['name' => 'Somalia', 'latitude' => 10.0, 'longitude' => 49.0], + 'RS' => ['name' => 'Serbia', 'latitude' => 44.0, 'longitude' => 21.0], + 'SS' => ['name' => 'South Sudan', 'latitude' => 8.0, 'longitude' => 30.0], + 'ST' => ['name' => 'São Tomé and Príncipe', 'latitude' => 1.0, 'longitude' => 7.0], + 'SR' => ['name' => 'Suriname', 'latitude' => 4.0, 'longitude' => -56.0], + 'SK' => ['name' => 'Slovakia', 'latitude' => 48.67, 'longitude' => 19.5], + 'SI' => ['name' => 'Slovenia', 'latitude' => 46.0, 'longitude' => 15.0], + 'SE' => ['name' => 'Sweden', 'latitude' => 62.0, 'longitude' => 15.0], + 'SZ' => ['name' => 'Eswatini', 'latitude' => -26.5, 'longitude' => 31.5], + 'SC' => ['name' => 'Seychelles', 'latitude' => -4.58, 'longitude' => 55.67], + 'SY' => ['name' => 'Syria', 'latitude' => 35.0, 'longitude' => 38.0], + 'TD' => ['name' => 'Chad', 'latitude' => 15.0, 'longitude' => 19.0], + 'TG' => ['name' => 'Togo', 'latitude' => 8.0, 'longitude' => 1.17], + 'TH' => ['name' => 'Thailand', 'latitude' => 15.0, 'longitude' => 100.0], + 'TJ' => ['name' => 'Tajikistan', 'latitude' => 39.0, 'longitude' => 71.0], + 'TM' => ['name' => 'Turkmenistan', 'latitude' => 40.0, 'longitude' => 60.0], + 'TL' => ['name' => 'Timor-Leste', 'latitude' => -8.83, 'longitude' => 125.92], + 'TO' => ['name' => 'Tonga', 'latitude' => -20.0, 'longitude' => -175.0], + 'TT' => ['name' => 'Trinidad and Tobago', 'latitude' => 11.0, 'longitude' => -61.0], + 'TN' => ['name' => 'Tunisia', 'latitude' => 34.0, 'longitude' => 9.0], + 'TR' => ['name' => 'Turkey', 'latitude' => 39.0, 'longitude' => 35.0], + 'TV' => ['name' => 'Tuvalu', 'latitude' => -8.0, 'longitude' => 178.0], + 'TZ' => ['name' => 'Tanzania', 'latitude' => -6.0, 'longitude' => 35.0], + 'TW' => ['name' => 'Taiwan', 'latitude' => 23.5, 'longitude' => 121.0], + 'UG' => ['name' => 'Uganda', 'latitude' => 1.0, 'longitude' => 32.0], + 'UA' => ['name' => 'Ukraine', 'latitude' => 49.0, 'longitude' => 32.0], + 'UY' => ['name' => 'Uruguay', 'latitude' => -33.0, 'longitude' => -56.0], + 'US' => ['name' => 'United States', 'latitude' => 38.0, 'longitude' => -97.0], + 'UZ' => ['name' => 'Uzbekistan', 'latitude' => 41.0, 'longitude' => 64.0], + 'VA' => ['name' => 'Vatican City', 'latitude' => 41.9, 'longitude' => 12.45], + 'VC' => ['name' => 'Saint Vincent and the Grenadines', 'latitude' => 13.25, 'longitude' => -61.2], + 'VE' => ['name' => 'Venezuela', 'latitude' => 8.0, 'longitude' => -66.0], + 'VN' => ['name' => 'Vietnam', 'latitude' => 16.0, 'longitude' => 106.0], + 'VU' => ['name' => 'Vanuatu', 'latitude' => -16.0, 'longitude' => 167.0], + 'WS' => ['name' => 'Samoa', 'latitude' => -13.58, 'longitude' => -172.33], + 'YE' => ['name' => 'Yemen', 'latitude' => 15.0, 'longitude' => 48.0], + 'ZA' => ['name' => 'South Africa', 'latitude' => -29.0, 'longitude' => 24.0], + 'ZM' => ['name' => 'Zambia', 'latitude' => -15.0, 'longitude' => 30.0], + 'ZW' => ['name' => 'Zimbabwe', 'latitude' => -20.0, 'longitude' => 30.0], ]; diff --git a/app/controllers/api/locale.php b/app/controllers/api/locale.php index 5b4c1ac47f..523ac3976b 100644 --- a/app/controllers/api/locale.php +++ b/app/controllers/api/locale.php @@ -118,7 +118,7 @@ App::get('/v1/locale/countries') ->inject('response') ->inject('locale') ->action(function (Response $response, Locale $locale) { - $list = Config::getParam('locale-countries'); /* @var $list array */ + $list = array_keys(Config::getParam('locale-countries')); /* @var $list array */ $output = []; foreach ($list as $value) { @@ -229,7 +229,7 @@ App::get('/v1/locale/continents') ->inject('response') ->inject('locale') ->action(function (Response $response, Locale $locale) { - $list = Config::getParam('locale-continents'); + $list = array_keys(Config::getParam('locale-continents')); foreach ($list as $value) { $output[] = new Document([ From 3a2babc906f0ca2a65a70ae42b4620907b2805e3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Feb 2025 21:34:46 +1300 Subject: [PATCH 070/275] Update date --- app/controllers/api/account.php | 2 +- app/controllers/api/databases.php | 6 +++--- app/controllers/api/messaging.php | 8 ++++---- app/controllers/api/teams.php | 2 +- app/controllers/api/users.php | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 438a036274..1ba3204134 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2717,7 +2717,7 @@ App::get('/v1/account/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 3cc9c110c0..ffe9142251 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -672,7 +672,7 @@ App::get('/v1/databases/:databaseId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -1068,7 +1068,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -3721,7 +3721,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 2e89863ec3..05bb461faf 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -996,7 +996,7 @@ App::get('/v1/messaging/providers/:providerId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -2226,7 +2226,7 @@ App::get('/v1/messaging/topics/:topicId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -2638,7 +2638,7 @@ App::get('/v1/messaging/subscribers/:subscriberId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -3405,7 +3405,7 @@ App::get('/v1/messaging/messages/:messageId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 815bd6db1d..5fbc1a4e08 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1364,7 +1364,7 @@ App::get('/v1/teams/:teamId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 5e58a610cf..0bea1a07f9 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -864,7 +864,7 @@ App::get('/v1/users/:userId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T00:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); From 542e34bc943d35344e7294ecd8af4c3de1664858 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 25 Feb 2025 12:28:05 +0000 Subject: [PATCH 071/275] chore: bump docker-base --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2bb9f80d9e..8ef6c204c7 100755 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN composer install --ignore-platform-reqs --optimize-autoloader \ --no-plugins --no-scripts --prefer-dist \ `if [ "$TESTING" != "true" ]; then echo "--no-dev"; fi` -FROM appwrite/base:0.9.5 AS final +FROM appwrite/base:0.10.0 AS final LABEL maintainer="team@appwrite.io" From e8fb2f8ae11777fd22ff845261f229fa0a41924a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 25 Feb 2025 20:25:54 +0100 Subject: [PATCH 072/275] Changed config format --- app/config/locale/continents.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/locale/continents.php b/app/config/locale/continents.php index 4033c9ff5a..611c725ef1 100644 --- a/app/config/locale/continents.php +++ b/app/config/locale/continents.php @@ -2,7 +2,7 @@ /** * Continent codes with names and approximate central coordinates - * + * * Coordinates represent approximate geographical centers of each continent * Note: These are simplified centroids and may not represent the exact geographical center */ From 7dfa3193c933e33b04011a0aad8f2b5565ae448b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Feb 2025 13:45:49 +1300 Subject: [PATCH 073/275] Test increase batch --- src/Appwrite/Platform/Workers/Audits.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 977aff3cfb..becc6f0d4e 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -17,7 +17,7 @@ use Utopia\System\System; class Audits extends Action { - private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development + private const BATCH_SIZE_DEVELOPMENT = 3; // smaller batch size for development private const BATCH_SIZE_PRODUCTION = 5_000; private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds From 34d91a6518fd5c37203e7c57328e7b9091ede14e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Feb 2025 13:51:41 +1300 Subject: [PATCH 074/275] Revert "Test increase batch" This reverts commit 7dfa3193c933e33b04011a0aad8f2b5565ae448b. --- src/Appwrite/Platform/Workers/Audits.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index becc6f0d4e..977aff3cfb 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -17,7 +17,7 @@ use Utopia\System\System; class Audits extends Action { - private const BATCH_SIZE_DEVELOPMENT = 3; // smaller batch size for development + private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development private const BATCH_SIZE_PRODUCTION = 5_000; private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds From 6e82bfe8e7d92d2f7c4424cfaf64a6e9c9f855ea Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Feb 2025 14:00:16 +1300 Subject: [PATCH 075/275] Update date --- app/controllers/api/account.php | 2 +- app/controllers/api/databases.php | 6 +++--- app/controllers/api/messaging.php | 8 ++++---- app/controllers/api/teams.php | 2 +- app/controllers/api/users.php | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 1ba3204134..a634618e6e 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2717,7 +2717,7 @@ App::get('/v1/account/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index ffe9142251..013e4639cc 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -672,7 +672,7 @@ App::get('/v1/databases/:databaseId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -1068,7 +1068,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -3721,7 +3721,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 05bb461faf..9393f1fbfe 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -996,7 +996,7 @@ App::get('/v1/messaging/providers/:providerId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -2226,7 +2226,7 @@ App::get('/v1/messaging/topics/:topicId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -2638,7 +2638,7 @@ App::get('/v1/messaging/subscribers/:subscriberId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); @@ -3405,7 +3405,7 @@ App::get('/v1/messaging/messages/:messageId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 5fbc1a4e08..b45c9fd3b9 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1364,7 +1364,7 @@ App::get('/v1/teams/:teamId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 0bea1a07f9..4a551b7478 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -864,7 +864,7 @@ App::get('/v1/users/:userId/logs') // Temp fix for logs $queries[] = Query::or([ - Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-25T09:00+00:00'))), + Query::greaterThan('$createdAt', DateTime::format(new \DateTime('2025-02-26T01:30+00:00'))), Query::lessThan('$createdAt', DateTime::format(new \DateTime('2025-02-13T00:00+00:00'))), ]); From 7ea108f015d8b71ed25bb0b48dc3bda99841ec7b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 26 Feb 2025 08:37:36 +0545 Subject: [PATCH 076/275] Fix: stats resources only queue projects accessed in last 3 hours --- src/Appwrite/Platform/Tasks/StatsResources.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/StatsResources.php b/src/Appwrite/Platform/Tasks/StatsResources.php index ac3b9ead73..c318f92cf1 100644 --- a/src/Appwrite/Platform/Tasks/StatsResources.php +++ b/src/Appwrite/Platform/Tasks/StatsResources.php @@ -62,7 +62,7 @@ class StatsResources extends Action Authorization::disable(); Authorization::setDefaultStatus(false); - $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours')); + $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('3 hours')); /** * For each project that were accessed in last 24 hours */ From ae78e793dd3d026e376def042d2b88fbec9ecd80 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 26 Feb 2025 08:52:06 +0000 Subject: [PATCH 077/275] chore: added blocking to file preview endpoint --- app/config/errors.php | 5 +++++ app/controllers/api/storage.php | 18 ++++++++++-------- src/Appwrite/Extend/Exception.php | 1 + 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index 7c7f6dc9ec..d8a5c798d3 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -480,6 +480,11 @@ return [ 'description' => 'The requested file is not publicly readable.', 'code' => 403, ], + Exception::STORAGE_FILE_PREVIEW_BLOCKED => [ + 'name' => Exception::STORAGE_FILE_PREVIEW_BLOCKED, + 'description' => 'File preview is not available on your pricing current tier.', + 'code' => 403, + ], /** VCS */ Exception::INSTALLATION_NOT_FOUND => [ diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 972232167b..38e74ad2ee 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -935,15 +935,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->param('rotation', 0, new Range(-360, 360), 'Preview image rotation in degrees. Pass an integer between -360 and 360.', true) ->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true) ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true) - ->inject('request') ->inject('response') - ->inject('project') + ->inject('plan') ->inject('dbForProject') - ->inject('mode') ->inject('deviceForFiles') ->inject('deviceForLocal') ->inject('queueForStatsUsage') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal, StatsUsage $queueForStatsUsage) { + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Response $response, array $plan, Database $dbForProject, Device $deviceForFiles, Device $deviceForLocal, StatsUsage $queueForStatsUsage) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -965,6 +963,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') throw new Exception(Exception::USER_UNAUTHORIZED); } + if (isset($plan['imageTransformations'])) { + if ($plan['imageTransformations'] === -1) { + throw new Exception(Exception::STORAGE_FILE_PREVIEW_BLOCKED); + } + } + if ($fileSecurity && !$valid) { $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); } else { @@ -1073,8 +1077,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $queueForStatsUsage ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) - ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) - ; + ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1); $transformedAt = $file->getAttribute('transformedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { @@ -1085,8 +1088,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $response ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days ->setContentType($contentType) - ->file($data) - ; + ->file($data); unset($image); }); diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index d4f47ca177..7f601018fe 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -143,6 +143,7 @@ class Exception extends \Exception public const STORAGE_INVALID_RANGE = 'storage_invalid_range'; public const STORAGE_INVALID_APPWRITE_ID = 'storage_invalid_appwrite_id'; public const STORAGE_FILE_NOT_PUBLIC = 'storage_file_not_public'; + public const STORAGE_FILE_PREVIEW_BLOCKED = 'storage_file_preview_blocked'; /** VCS */ public const INSTALLATION_NOT_FOUND = 'installation_not_found'; From ad56782b49a3939a6eb6df2606d5e0caf78f736b Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Wed, 26 Feb 2025 10:19:51 +0100 Subject: [PATCH 078/275] Fixed tests --- tests/e2e/Services/Locale/LocaleBase.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Services/Locale/LocaleBase.php b/tests/e2e/Services/Locale/LocaleBase.php index 0e2928004d..ee731a99e5 100644 --- a/tests/e2e/Services/Locale/LocaleBase.php +++ b/tests/e2e/Services/Locale/LocaleBase.php @@ -228,8 +228,8 @@ trait LocaleBase * Test for SUCCESS */ $languages = require(__DIR__ . '/../../../../app/config/locale/codes.php'); - $defaultCountries = require(__DIR__ . '/../../../../app/config/locale/countries.php'); - $defaultContinents = require(__DIR__ . '/../../../../app/config/locale/continents.php'); + $defaultCountries = array_keys(require(__DIR__ . '/../../../../app/config/locale/countries.php')); + $defaultContinents = array_keys(require(__DIR__ . '/../../../../app/config/locale/continents.php')); foreach ($languages as $lang) { $response = $this->client->call(Client::METHOD_GET, '/locale/countries', [ From bb5ed1ccc68546176840410b957bcd742f473f34 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 26 Feb 2025 10:14:32 +0000 Subject: [PATCH 079/275] chore: bump base to 0.10.1 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8ef6c204c7..41592ef9fb 100755 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN composer install --ignore-platform-reqs --optimize-autoloader \ --no-plugins --no-scripts --prefer-dist \ `if [ "$TESTING" != "true" ]; then echo "--no-dev"; fi` -FROM appwrite/base:0.10.0 AS final +FROM appwrite/base:0.10.1 AS final LABEL maintainer="team@appwrite.io" From 0ddcb9b14c9a07a647a5eb3e5bf95f41b3e2b891 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 26 Feb 2025 16:24:05 +0000 Subject: [PATCH 080/275] chore: refactor adapter --- src/Appwrite/Event/Realtime.php | 42 ++++++++++++--------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index e04bbfffb4..e7a61ea545 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -82,33 +82,21 @@ class Realtime extends Event bucket: $bucket, ); - if (!empty($this->getTargets())) { - foreach ($this->getTargets() as $targetProjectId) { - RealtimeAdapter::send( - projectId: $targetProjectId, - payload: $this->getRealtimePayload(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'permissionsChanged' => $target['permissionsChanged'], - 'userId' => $this->getParam('userId') - ] - ); - } - } else { - RealtimeAdapter::send( - projectId: $target['projectId'] ?? $this->getProject()->getId(), - payload: $this->getRealtimePayload(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'permissionsChanged' => $target['permissionsChanged'], - 'userId' => $this->getParam('userId') - ] - ); - } + $projectIds = !empty($this->getTargets()) + ? $this->getTargets() + : [$target['projectId'] ?? $this->getProject()->getId()]; + + RealtimeAdapter::send( + projectId: $projectIds, + payload: $this->getRealtimePayload(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'], + options: [ + 'permissionsChanged' => $target['permissionsChanged'], + 'userId' => $this->getParam('userId') + ] + ); return true; } From f5947de50880f1568967db56097cc2629075c5b4 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Feb 2025 16:38:05 +0000 Subject: [PATCH 081/275] chore: bump cache 0.12.x --- composer.json | 4 ++-- composer.lock | 48 +++++++++++++++++++++++++----------------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/composer.json b/composer.json index 6180773b24..0988810b47 100644 --- a/composer.json +++ b/composer.json @@ -48,7 +48,7 @@ "utopia-php/abuse": "0.51.*", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.54.0", - "utopia-php/cache": "0.11.*", + "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.60.*", @@ -71,7 +71,7 @@ "utopia-php/swoole": "0.8.*", "utopia-php/system": "0.9.*", "utopia-php/telemetry": "0.1.*", - "utopia-php/vcs": "0.8.*", + "utopia-php/vcs": "0.9.3", "utopia-php/websocket": "0.1.*", "matomo/device-detector": "6.1.*", "dragonmantank/cron-expression": "3.3.2", diff --git a/composer.lock b/composer.lock index 841d5b51c5..ad1ed0f14a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "87a12bb6f58c5dad7488229590e9a3f5", + "content-hash": "7b90c906ef7ef71246bbed87f0b45955", "packages": [ { "name": "adhocore/jwt", @@ -3521,23 +3521,24 @@ }, { "name": "utopia-php/cache", - "version": "0.11.0", + "version": "0.12.0", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "8ebcab5aac7606331cef69b0081f6c9eff2e58bc" + "reference": "646038f1d470b759c129348be8fc14da3c00bbd9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/8ebcab5aac7606331cef69b0081f6c9eff2e58bc", - "reference": "8ebcab5aac7606331cef69b0081f6c9eff2e58bc", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/646038f1d470b759c129348be8fc14da3c00bbd9", + "reference": "646038f1d470b759c129348be8fc14da3c00bbd9", "shasum": "" }, "require": { "ext-json": "*", "ext-memcached": "*", "ext-redis": "*", - "php": ">=8.0" + "php": ">=8.0", + "utopia-php/telemetry": "0.1.*" }, "require-dev": { "laravel/pint": "1.2.*", @@ -3565,9 +3566,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/0.11.0" + "source": "https://github.com/utopia-php/cache/tree/0.12.0" }, - "time": "2024-11-05T16:53:58+00:00" + "time": "2025-02-25T09:09:21+00:00" }, { "name": "utopia-php/cli", @@ -3717,23 +3718,23 @@ }, { "name": "utopia-php/database", - "version": "0.60.3", + "version": "0.60.4", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "c4bc4af3f09a91aea76aac75b4b78fa06598c61d" + "reference": "3a034f8eb275cd92d088f1e492a7cd00fd021427" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/c4bc4af3f09a91aea76aac75b4b78fa06598c61d", - "reference": "c4bc4af3f09a91aea76aac75b4b78fa06598c61d", + "url": "https://api.github.com/repos/utopia-php/database/zipball/3a034f8eb275cd92d088f1e492a7cd00fd021427", + "reference": "3a034f8eb275cd92d088f1e492a7cd00fd021427", "shasum": "" }, "require": { "ext-mbstring": "*", "ext-pdo": "*", "php": ">=8.1", - "utopia-php/cache": "0.11.*", + "utopia-php/cache": "0.12.*", "utopia-php/framework": "0.33.*", "utopia-php/mongo": "0.3.*" }, @@ -3767,9 +3768,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.60.3" + "source": "https://github.com/utopia-php/database/tree/0.60.4" }, - "time": "2025-02-17T12:46:59+00:00" + "time": "2025-02-25T23:09:18+00:00" }, { "name": "utopia-php/domains", @@ -4820,23 +4821,24 @@ }, { "name": "utopia-php/vcs", - "version": "0.8.6", + "version": "0.9.3", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "b10225f54d5670f09f83e82e09de9d820ada6931" + "reference": "865a00c67e81a20938b883f9aa802303790dd3b5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/b10225f54d5670f09f83e82e09de9d820ada6931", - "reference": "b10225f54d5670f09f83e82e09de9d820ada6931", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/865a00c67e81a20938b883f9aa802303790dd3b5", + "reference": "865a00c67e81a20938b883f9aa802303790dd3b5", "shasum": "" }, "require": { "adhocore/jwt": "^1.1", "php": ">=8.0", - "utopia-php/cache": "^0.11.0", - "utopia-php/framework": "0.*.*" + "utopia-php/cache": "0.12.*", + "utopia-php/framework": "0.*.*", + "utopia-php/system": "0.9.*" }, "require-dev": { "laravel/pint": "1.2.*", @@ -4863,9 +4865,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/0.8.6" + "source": "https://github.com/utopia-php/vcs/tree/0.9.3" }, - "time": "2024-12-10T13:13:23+00:00" + "time": "2025-02-26T16:33:35+00:00" }, { "name": "utopia-php/websocket", From 31d6fe24233f26b7851f3cc956a7d2153006215a Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Feb 2025 16:40:19 +0000 Subject: [PATCH 082/275] chore: bump cache 0.12.x --- composer.json | 2 +- composer.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 0988810b47..e096f9e4e7 100644 --- a/composer.json +++ b/composer.json @@ -71,7 +71,7 @@ "utopia-php/swoole": "0.8.*", "utopia-php/system": "0.9.*", "utopia-php/telemetry": "0.1.*", - "utopia-php/vcs": "0.9.3", + "utopia-php/vcs": "0.9.*", "utopia-php/websocket": "0.1.*", "matomo/device-detector": "6.1.*", "dragonmantank/cron-expression": "3.3.2", diff --git a/composer.lock b/composer.lock index ad1ed0f14a..61eee9b8dc 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7b90c906ef7ef71246bbed87f0b45955", + "content-hash": "52270716c94a941bef634c33d85df905", "packages": [ { "name": "adhocore/jwt", From 7687784515ab4e0f78cf8f7d024281a968dff341 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 26 Feb 2025 16:41:26 +0000 Subject: [PATCH 083/275] chore: added fallback to old values in update --- app/controllers/api/databases.php | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index a4b5683512..e14c504234 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -299,6 +299,9 @@ function updateAttribute( switch ($attribute->getAttribute('format')) { case APP_DATABASE_ATTRIBUTE_INT_RANGE: case APP_DATABASE_ATTRIBUTE_FLOAT_RANGE: + $min ??= $attribute->getAttribute('formatOptions')['min']; + $max ??= $attribute->getAttribute('formatOptions')['max']; + if ($min > $max) { throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Minimum value must be lesser than maximum value'); } @@ -1550,8 +1553,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?int $min, ?int $max, ?int $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { // Ensure attribute default is within range - $min = \is_null($min) ? PHP_INT_MIN : $min; - $max = \is_null($max) ? PHP_INT_MAX : $max; + $min ??= PHP_INT_MIN; + $max ??= PHP_INT_MAX; if ($min > $max) { throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Minimum value must be lesser than maximum value'); @@ -1627,8 +1630,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float' ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?float $min, ?float $max, ?float $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { // Ensure attribute default is within range - $min = \is_null($min) ? -PHP_FLOAT_MAX : $min; - $max = \is_null($max) ? PHP_FLOAT_MAX : $max; + $min ??= -PHP_FLOAT_MAX; + $max ??= PHP_FLOAT_MAX; if ($min > $max) { throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Minimum value must be lesser than maximum value'); @@ -2347,11 +2350,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/integ ->inject('dbForProject') ->inject('queueForEvents') ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?int $min, ?int $max, ?int $default, ?string $newKey, Response $response, Database $dbForProject, Event $queueForEvents) { - - // Ensure attribute default is within range - $min = \is_null($min) ? PHP_INT_MIN : $min; - $max = \is_null($max) ? PHP_INT_MAX : $max; - $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, @@ -2411,11 +2409,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/float ->inject('dbForProject') ->inject('queueForEvents') ->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?float $min, ?float $max, ?float $default, ?string $newKey, Response $response, Database $dbForProject, Event $queueForEvents) { - - // Ensure attribute default is within range - $min = \is_null($min) ? -PHP_FLOAT_MAX : $min; - $max = \is_null($max) ? PHP_FLOAT_MAX : $max; - $attribute = updateAttribute( databaseId: $databaseId, collectionId: $collectionId, From 28896081d78cf12a0a7c06838f7fb2987f823bb1 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 26 Feb 2025 16:50:12 +0000 Subject: [PATCH 084/275] chore: fix forloop --- src/Appwrite/Event/Realtime.php | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index e7a61ea545..324f140165 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -86,17 +86,19 @@ class Realtime extends Event ? $this->getTargets() : [$target['projectId'] ?? $this->getProject()->getId()]; - RealtimeAdapter::send( - projectId: $projectIds, - payload: $this->getRealtimePayload(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'permissionsChanged' => $target['permissionsChanged'], - 'userId' => $this->getParam('userId') - ] - ); + foreach ($projectIds as $projectId) { + RealtimeAdapter::send( + projectId: $projectId, + payload: $this->getRealtimePayload(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'], + options: [ + 'permissionsChanged' => $target['permissionsChanged'], + 'userId' => $this->getParam('userId') + ] + ); + } return true; } From c79ed286b1cc79ba94778acda18983e8052da288 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 26 Feb 2025 17:35:06 +0000 Subject: [PATCH 085/275] chore: update tests --- app/controllers/api/databases.php | 4 ++-- .../e2e/Services/Databases/DatabasesCustomServerTest.php | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index e14c504234..876c6bc82a 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -242,8 +242,8 @@ function updateAttribute( string $filter = null, string|bool|int|float $default = null, bool $required = null, - int|float $min = null, - int|float $max = null, + int|float|null $min = null, + int|float|null $max = null, array $elements = null, array $options = [], string $newKey = null, diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 75526b9928..57e0b93634 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -2327,8 +2327,8 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'required' => false, - 'default' => -10, - 'max' => 0, + 'default' => 10, + 'max' => 100, ]); $this->assertEquals(200, $update['headers']['status-code']); @@ -2588,8 +2588,8 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'required' => false, - 'default' => -123.456, - 'max' => 0.0, + 'default' => 23.456, + 'max' => 100.0, ]); $this->assertEquals(200, $update['headers']['status-code']); From 57c7ebb53def528b366126e08b418a0d9fae6582 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 26 Feb 2025 17:51:00 +0000 Subject: [PATCH 086/275] chore: bump sdk --- composer.lock | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/composer.lock b/composer.lock index d23d9acc99..e1d1ac16e3 100644 --- a/composer.lock +++ b/composer.lock @@ -279,16 +279,16 @@ }, { "name": "brick/math", - "version": "0.12.1", + "version": "0.12.2", "source": { "type": "git", "url": "https://github.com/brick/math.git", - "reference": "f510c0a40911935b77b86859eb5223d58d660df1" + "reference": "901eddb1e45a8e0f689302e40af871c181ecbe40" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/brick/math/zipball/f510c0a40911935b77b86859eb5223d58d660df1", - "reference": "f510c0a40911935b77b86859eb5223d58d660df1", + "url": "https://api.github.com/repos/brick/math/zipball/901eddb1e45a8e0f689302e40af871c181ecbe40", + "reference": "901eddb1e45a8e0f689302e40af871c181ecbe40", "shasum": "" }, "require": { @@ -297,7 +297,7 @@ "require-dev": { "php-coveralls/php-coveralls": "^2.2", "phpunit/phpunit": "^10.1", - "vimeo/psalm": "5.16.0" + "vimeo/psalm": "6.8.8" }, "type": "library", "autoload": { @@ -327,7 +327,7 @@ ], "support": { "issues": "https://github.com/brick/math/issues", - "source": "https://github.com/brick/math/tree/0.12.1" + "source": "https://github.com/brick/math/tree/0.12.2" }, "funding": [ { @@ -335,7 +335,7 @@ "type": "github" } ], - "time": "2023-11-29T23:19:16+00:00" + "time": "2025-02-26T10:21:45+00:00" }, { "name": "chillerlan/php-qrcode", @@ -2694,16 +2694,16 @@ }, { "name": "symfony/http-client", - "version": "v7.2.3", + "version": "v7.2.4", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "7ce6078c79a4a7afff931c413d2959d3bffbfb8d" + "reference": "78981a2ffef6437ed92d4d7e2a86a82f256c6dc6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/7ce6078c79a4a7afff931c413d2959d3bffbfb8d", - "reference": "7ce6078c79a4a7afff931c413d2959d3bffbfb8d", + "url": "https://api.github.com/repos/symfony/http-client/zipball/78981a2ffef6437ed92d4d7e2a86a82f256c6dc6", + "reference": "78981a2ffef6437ed92d4d7e2a86a82f256c6dc6", "shasum": "" }, "require": { @@ -2769,7 +2769,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.3" + "source": "https://github.com/symfony/http-client/tree/v7.2.4" }, "funding": [ { @@ -2785,7 +2785,7 @@ "type": "tidelift" } ], - "time": "2025-01-28T15:51:35+00:00" + "time": "2025-02-13T10:27:23+00:00" }, { "name": "symfony/http-client-contracts", @@ -5052,16 +5052,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.0", + "version": "0.40.1", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "d2880132c900f64108d3e4484a6c1ed1bed2303c" + "reference": "df180676b6fbde7832ae1495af3e2f3e8f700837" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/d2880132c900f64108d3e4484a6c1ed1bed2303c", - "reference": "d2880132c900f64108d3e4484a6c1ed1bed2303c", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/df180676b6fbde7832ae1495af3e2f3e8f700837", + "reference": "df180676b6fbde7832ae1495af3e2f3e8f700837", "shasum": "" }, "require": { @@ -5097,9 +5097,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.0" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.1" }, - "time": "2025-02-04T12:47:33+00:00" + "time": "2025-02-26T07:07:10+00:00" }, { "name": "doctrine/annotations", @@ -8035,16 +8035,16 @@ }, { "name": "symfony/process", - "version": "v7.2.0", + "version": "v7.2.4", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "d34b22ba9390ec19d2dd966c40aa9e8462f27a7e" + "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/d34b22ba9390ec19d2dd966c40aa9e8462f27a7e", - "reference": "d34b22ba9390ec19d2dd966c40aa9e8462f27a7e", + "url": "https://api.github.com/repos/symfony/process/zipball/d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", + "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", "shasum": "" }, "require": { @@ -8076,7 +8076,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.2.0" + "source": "https://github.com/symfony/process/tree/v7.2.4" }, "funding": [ { @@ -8092,7 +8092,7 @@ "type": "tidelift" } ], - "time": "2024-11-06T14:24:19+00:00" + "time": "2025-02-05T08:33:46+00:00" }, { "name": "symfony/string", From 611f2cad6a579b623c71735c77757e4ae03f4e44 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Feb 2025 22:02:55 +0000 Subject: [PATCH 087/275] chore: bump cache 0.12.x --- composer.lock | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/composer.lock b/composer.lock index 61eee9b8dc..3d145438e2 100644 --- a/composer.lock +++ b/composer.lock @@ -279,16 +279,16 @@ }, { "name": "brick/math", - "version": "0.12.1", + "version": "0.12.2", "source": { "type": "git", "url": "https://github.com/brick/math.git", - "reference": "f510c0a40911935b77b86859eb5223d58d660df1" + "reference": "901eddb1e45a8e0f689302e40af871c181ecbe40" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/brick/math/zipball/f510c0a40911935b77b86859eb5223d58d660df1", - "reference": "f510c0a40911935b77b86859eb5223d58d660df1", + "url": "https://api.github.com/repos/brick/math/zipball/901eddb1e45a8e0f689302e40af871c181ecbe40", + "reference": "901eddb1e45a8e0f689302e40af871c181ecbe40", "shasum": "" }, "require": { @@ -297,7 +297,7 @@ "require-dev": { "php-coveralls/php-coveralls": "^2.2", "phpunit/phpunit": "^10.1", - "vimeo/psalm": "5.16.0" + "vimeo/psalm": "6.8.8" }, "type": "library", "autoload": { @@ -327,7 +327,7 @@ ], "support": { "issues": "https://github.com/brick/math/issues", - "source": "https://github.com/brick/math/tree/0.12.1" + "source": "https://github.com/brick/math/tree/0.12.2" }, "funding": [ { @@ -335,7 +335,7 @@ "type": "github" } ], - "time": "2023-11-29T23:19:16+00:00" + "time": "2025-02-26T10:21:45+00:00" }, { "name": "chillerlan/php-qrcode", @@ -2694,16 +2694,16 @@ }, { "name": "symfony/http-client", - "version": "v7.2.3", + "version": "v7.2.4", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "7ce6078c79a4a7afff931c413d2959d3bffbfb8d" + "reference": "78981a2ffef6437ed92d4d7e2a86a82f256c6dc6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/7ce6078c79a4a7afff931c413d2959d3bffbfb8d", - "reference": "7ce6078c79a4a7afff931c413d2959d3bffbfb8d", + "url": "https://api.github.com/repos/symfony/http-client/zipball/78981a2ffef6437ed92d4d7e2a86a82f256c6dc6", + "reference": "78981a2ffef6437ed92d4d7e2a86a82f256c6dc6", "shasum": "" }, "require": { @@ -2769,7 +2769,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.3" + "source": "https://github.com/symfony/http-client/tree/v7.2.4" }, "funding": [ { @@ -2785,7 +2785,7 @@ "type": "tidelift" } ], - "time": "2025-01-28T15:51:35+00:00" + "time": "2025-02-13T10:27:23+00:00" }, { "name": "symfony/http-client-contracts", @@ -5054,16 +5054,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.0", + "version": "0.40.1", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "d2880132c900f64108d3e4484a6c1ed1bed2303c" + "reference": "df180676b6fbde7832ae1495af3e2f3e8f700837" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/d2880132c900f64108d3e4484a6c1ed1bed2303c", - "reference": "d2880132c900f64108d3e4484a6c1ed1bed2303c", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/df180676b6fbde7832ae1495af3e2f3e8f700837", + "reference": "df180676b6fbde7832ae1495af3e2f3e8f700837", "shasum": "" }, "require": { @@ -5099,9 +5099,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.0" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.1" }, - "time": "2025-02-04T12:47:33+00:00" + "time": "2025-02-26T07:07:10+00:00" }, { "name": "doctrine/annotations", @@ -8037,16 +8037,16 @@ }, { "name": "symfony/process", - "version": "v7.2.0", + "version": "v7.2.4", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "d34b22ba9390ec19d2dd966c40aa9e8462f27a7e" + "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/d34b22ba9390ec19d2dd966c40aa9e8462f27a7e", - "reference": "d34b22ba9390ec19d2dd966c40aa9e8462f27a7e", + "url": "https://api.github.com/repos/symfony/process/zipball/d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", + "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", "shasum": "" }, "require": { @@ -8078,7 +8078,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.2.0" + "source": "https://github.com/symfony/process/tree/v7.2.4" }, "funding": [ { @@ -8094,7 +8094,7 @@ "type": "tidelift" } ], - "time": "2024-11-06T14:24:19+00:00" + "time": "2025-02-05T08:33:46+00:00" }, { "name": "symfony/string", From 0bbcdf8268b5aac551f94387dc559e3ba9ee4faf Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Feb 2025 22:05:09 +0000 Subject: [PATCH 088/275] chore: bump cache 0.12.x --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index e2b25321a2..76f61e44d7 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "52270716c94a941bef634c33d85df905", + "content-hash": "58ad2e1375ec47d944b96864d4aae63f", "packages": [ { "name": "adhocore/jwt", From 6ad34a7bf7a20f04896cc8c072ff50fc01bf4a2c Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 27 Feb 2025 05:05:31 +0000 Subject: [PATCH 089/275] chore: sdk versions and corresponding docs --- app/config/platforms.php | 14 +++++++------- .../examples/account/update-mfa-challenge.md | 2 +- .../examples/account/update-mfa-challenge.md | 2 +- .../examples/account/update-mfa-challenge.md | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/config/platforms.php b/app/config/platforms.php index 92e325bd44..4b6a0d2da6 100644 --- a/app/config/platforms.php +++ b/app/config/platforms.php @@ -11,7 +11,7 @@ return [ [ 'key' => 'web', 'name' => 'Web', - 'version' => '16.1.0', + 'version' => '17.0.1', 'url' => 'https://github.com/appwrite/sdk-for-web', 'package' => 'https://www.npmjs.com/package/appwrite', 'enabled' => true, @@ -59,7 +59,7 @@ return [ [ 'key' => 'flutter', 'name' => 'Flutter', - 'version' => '13.1.1', + 'version' => '14.0.1', 'url' => 'https://github.com/appwrite/sdk-for-flutter', 'package' => 'https://pub.dev/packages/appwrite', 'enabled' => true, @@ -77,7 +77,7 @@ return [ [ 'key' => 'apple', 'name' => 'Apple', - 'version' => '7.1.0', + 'version' => '9.0.0', 'url' => 'https://github.com/appwrite/sdk-for-apple', 'package' => 'https://github.com/appwrite/sdk-for-apple', 'enabled' => true, @@ -217,7 +217,7 @@ return [ [ 'key' => 'cli', 'name' => 'Command Line', - 'version' => '6.2.0', + 'version' => '6.2.1', 'url' => 'https://github.com/appwrite/sdk-for-cli', 'package' => 'https://www.npmjs.com/package/appwrite-cli', 'enabled' => true, @@ -245,7 +245,7 @@ return [ [ 'key' => 'nodejs', 'name' => 'Node.js', - 'version' => '14.2.0', + 'version' => '15.0.1', 'url' => 'https://github.com/appwrite/sdk-for-node', 'package' => 'https://www.npmjs.com/package/node-appwrite', 'enabled' => true, @@ -371,7 +371,7 @@ return [ [ 'key' => 'dart', 'name' => 'Dart', - 'version' => '12.2.0', + 'version' => '13.0.1', 'url' => 'https://github.com/appwrite/sdk-for-dart', 'package' => 'https://pub.dev/packages/dart_appwrite', 'enabled' => true, @@ -411,7 +411,7 @@ return [ [ 'key' => 'swift', 'name' => 'Swift', - 'version' => '6.2.0', + 'version' => '8.0.0', 'url' => 'https://github.com/appwrite/sdk-for-swift', 'package' => 'https://github.com/appwrite/sdk-for-swift', 'enabled' => true, diff --git a/docs/examples/1.6.x/client-apple/examples/account/update-mfa-challenge.md b/docs/examples/1.6.x/client-apple/examples/account/update-mfa-challenge.md index a237537ae3..1c5874f784 100644 --- a/docs/examples/1.6.x/client-apple/examples/account/update-mfa-challenge.md +++ b/docs/examples/1.6.x/client-apple/examples/account/update-mfa-challenge.md @@ -6,7 +6,7 @@ let client = Client() let account = Account(client) -let result = try await account.updateMfaChallenge( +let session = try await account.updateMfaChallenge( challengeId: "", otp: "" ) diff --git a/docs/examples/1.6.x/client-flutter/examples/account/update-mfa-challenge.md b/docs/examples/1.6.x/client-flutter/examples/account/update-mfa-challenge.md index c8e1af7e90..bbe7c03470 100644 --- a/docs/examples/1.6.x/client-flutter/examples/account/update-mfa-challenge.md +++ b/docs/examples/1.6.x/client-flutter/examples/account/update-mfa-challenge.md @@ -6,7 +6,7 @@ Client client = Client() Account account = Account(client); - result = await account.updateMfaChallenge( +Session result = await account.updateMfaChallenge( challengeId: '', otp: '', ); diff --git a/docs/examples/1.6.x/server-dart/examples/account/update-mfa-challenge.md b/docs/examples/1.6.x/server-dart/examples/account/update-mfa-challenge.md index fd64c61cf9..2843d2f1b4 100644 --- a/docs/examples/1.6.x/server-dart/examples/account/update-mfa-challenge.md +++ b/docs/examples/1.6.x/server-dart/examples/account/update-mfa-challenge.md @@ -7,7 +7,7 @@ Client client = Client() Account account = Account(client); - result = await account.updateMfaChallenge( +Session result = await account.updateMfaChallenge( challengeId: '', otp: '', ); From 388affa3d0f9f0c231f731b30b86cc4127d2c129 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 27 Feb 2025 07:23:13 +0000 Subject: [PATCH 090/275] chore: updated naming for subscribers --- app/controllers/api/functions.php | 2 +- src/Appwrite/Event/Realtime.php | 20 +++++++++---------- src/Appwrite/Platform/Workers/Builds.php | 8 ++++---- .../Platform/Workers/Certificates.php | 2 +- src/Appwrite/Platform/Workers/Databases.php | 6 +++--- src/Appwrite/Platform/Workers/Functions.php | 2 +- src/Appwrite/Platform/Workers/Migrations.php | 2 +- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index a36313b7ab..583468f6c1 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -411,7 +411,7 @@ App::post('/v1/functions') /** Trigger Realtime Events */ $queueForRealtime ->from($ruleCreate) - ->setTargets(['console', $project->getId()]) + ->setSubscribers(['console', $project->getId()]) ->trigger(); } diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index 324f140165..28a1bb6a6d 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -7,7 +7,7 @@ use Utopia\Database\Document; class Realtime extends Event { - protected array $targets = []; + protected array $subscribers = []; public function __construct() { @@ -32,25 +32,25 @@ class Realtime extends Event } /** - * Set targets for this realtime event. + * Set subscribers for this realtime event. * - * @param array $targets + * @param array $subscribers * @return array */ - public function setTargets(array $targets): self + public function setSubscribers(array $subscribers): self { - $this->targets = $targets; + $this->subscribers = $subscribers; return $this; } /** - * Get targets for this realtime event. + * Get subscribers for this realtime event. * * @return array */ - public function getTargets(): array + public function getSubscribers(): array { - return $this->targets; + return $this->subscribers; } /** @@ -82,8 +82,8 @@ class Realtime extends Event bucket: $bucket, ); - $projectIds = !empty($this->getTargets()) - ? $this->getTargets() + $projectIds = !empty($this->getSubscribers()) + ? $this->getSubscribers() : [$target['projectId'] ?? $this->getProject()->getId()]; foreach ($projectIds as $projectId) { diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index c3853a1dde..6f26e9a80c 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -383,7 +383,7 @@ class Builds extends Action */ $queueForRealtime ->setProject($project) - ->setTargets(['console']) + ->setSubscribers(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) @@ -457,7 +457,7 @@ class Builds extends Action /** Trigger Realtime Event */ $queueForRealtime ->setProject($project) - ->setTargets(['console']) + ->setSubscribers(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) @@ -589,7 +589,7 @@ class Builds extends Action */ $queueForRealtime ->setProject($project) - ->setTargets(['console']) + ->setSubscribers(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) @@ -686,7 +686,7 @@ class Builds extends Action */ $queueForRealtime ->setProject($project) - ->setTargets(['console']) + ->setSubscribers(['console']) ->setEvent($event) ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()) diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 009ac24021..093e6fda5a 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -394,7 +394,7 @@ class Certificates extends Action /** Trigger Realtime Events */ $queueForRealtime ->from($queueForEvents) - ->setTargets(['console', $projectId]) + ->setSubscribers(['console', $projectId]) ->trigger(); } } diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 74101dd1eb..4abd035599 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -601,17 +601,17 @@ class Databases extends Action ): void { $queueForRealtime ->setProject($project) - ->setTargets(['console']) + ->setSubscribers(['console']) ->setEvent($event) ->setParam('databaseId', $database->getId()) ->setParam('collectionId', $collection->getId()); - if ($attribute !== null) { + if ($attribute !== null && !empty($attribute)) { $queueForRealtime ->setParam('attributeId', $attribute->getId()) ->setPayload($attribute->getArrayCopy()); } - if ($index !== null) { + if ($index !== null && !empty($index)) { $queueForRealtime ->setParam('indexId', $index->getId()) ->setPayload($index->getArrayCopy()); diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index 73b02be85e..a7caa3207f 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -600,7 +600,7 @@ class Functions extends Action /** Trigger Realtime Events */ $queueForRealtime ->from($queueForEvents) - ->setTargets(['console', $project->getId()]) + ->setSubscribers(['console', $project->getId()]) ->trigger(); if (!empty($error)) { diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 08c75d934a..f21a846a0d 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -160,7 +160,7 @@ class Migrations extends Action /** Trigger Realtime Events */ $queueForRealtime ->setProject($project) - ->setTargets(['console', $project->getId()]) + ->setSubscribers(['console', $project->getId()]) ->setEvent('migrations.[migrationId].update') ->setParam('migrationId', $migration->getId()) ->setPayload($migration->getArrayCopy()) From f31acd214ec73385d1f7668ed80787dfd4be85ba Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 27 Feb 2025 09:52:27 +0000 Subject: [PATCH 091/275] chore: remove plan blocking --- app/config/errors.php | 5 ----- app/controllers/api/storage.php | 9 +-------- src/Appwrite/Extend/Exception.php | 1 - 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index d8a5c798d3..7c7f6dc9ec 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -480,11 +480,6 @@ return [ 'description' => 'The requested file is not publicly readable.', 'code' => 403, ], - Exception::STORAGE_FILE_PREVIEW_BLOCKED => [ - 'name' => Exception::STORAGE_FILE_PREVIEW_BLOCKED, - 'description' => 'File preview is not available on your pricing current tier.', - 'code' => 403, - ], /** VCS */ Exception::INSTALLATION_NOT_FOUND => [ diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 38e74ad2ee..bee245b64f 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -936,12 +936,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true) ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true) ->inject('response') - ->inject('plan') ->inject('dbForProject') ->inject('deviceForFiles') ->inject('deviceForLocal') ->inject('queueForStatsUsage') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Response $response, array $plan, Database $dbForProject, Device $deviceForFiles, Device $deviceForLocal, StatsUsage $queueForStatsUsage) { + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Response $response, Database $dbForProject, Device $deviceForFiles, Device $deviceForLocal, StatsUsage $queueForStatsUsage) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -963,12 +962,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') throw new Exception(Exception::USER_UNAUTHORIZED); } - if (isset($plan['imageTransformations'])) { - if ($plan['imageTransformations'] === -1) { - throw new Exception(Exception::STORAGE_FILE_PREVIEW_BLOCKED); - } - } - if ($fileSecurity && !$valid) { $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); } else { diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 7f601018fe..d4f47ca177 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -143,7 +143,6 @@ class Exception extends \Exception public const STORAGE_INVALID_RANGE = 'storage_invalid_range'; public const STORAGE_INVALID_APPWRITE_ID = 'storage_invalid_appwrite_id'; public const STORAGE_FILE_NOT_PUBLIC = 'storage_file_not_public'; - public const STORAGE_FILE_PREVIEW_BLOCKED = 'storage_file_preview_blocked'; /** VCS */ public const INSTALLATION_NOT_FOUND = 'installation_not_found'; From e9d8d13fbdf79dfe03fd6bd61474348d1f877e01 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 28 Feb 2025 08:28:21 +0000 Subject: [PATCH 092/275] chore: added imageTransformations to project usage --- app/controllers/api/project.php | 2 ++ src/Appwrite/Utopia/Response/Model/UsageProject.php | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index ea2cd4436d..cf2a128939 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -63,6 +63,7 @@ App::get('/v1/project/usage') METRIC_BUILDS_STORAGE, METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, + METRIC_FILES_IMAGES_TRANSFORMED, ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -363,6 +364,7 @@ App::get('/v1/project/usage') 'authPhoneTotal' => $authPhoneTotal, 'authPhoneEstimate' => $authPhoneEstimate, 'authPhoneCountryBreakdown' => $authPhoneCountryBreakdown, + 'imageTransformations' => $total[METRIC_FILES_IMAGES_TRANSFORMED], ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 1006276b56..6f74ec561b 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -197,6 +197,13 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('imageTransformations', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of aggregated number of image transformations.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } From d5cfdc570f25dbab1a591df1f8e32e800fe216fa Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 28 Feb 2025 08:41:53 +0000 Subject: [PATCH 093/275] chore: updated specs --- app/config/specs/open-api3-latest-client.json | 10 +- .../specs/open-api3-latest-console.json | 211 ++++++++++------- app/config/specs/open-api3-latest-server.json | 178 ++++++++------- app/config/specs/swagger2-latest-client.json | 10 +- app/config/specs/swagger2-latest-console.json | 215 +++++++++++------- app/config/specs/swagger2-latest-server.json | 180 ++++++++------- 6 files changed, 460 insertions(+), 344 deletions(-) diff --git a/app/config/specs/open-api3-latest-client.json b/app/config/specs/open-api3-latest-client.json index 820b1f55e0..316fe13116 100644 --- a/app/config/specs/open-api3-latest-client.json +++ b/app/config/specs/open-api3-latest-client.json @@ -3383,7 +3383,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "schema": { "type": "string", @@ -3404,7 +3404,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3423,7 +3424,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ] }, "in": "path" @@ -4365,7 +4367,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 7f57dfc437..5b1d6f0d1e 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -3387,7 +3387,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "schema": { "type": "string", @@ -3408,7 +3408,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3427,7 +3428,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ] }, "in": "path" @@ -7823,7 +7825,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", @@ -11585,7 +11587,7 @@ }, "x-appwrite": { "method": "getCertificate", - "weight": 134, + "weight": 133, "cookies": false, "type": "", "deprecated": false, @@ -11692,7 +11694,7 @@ }, "x-appwrite": { "method": "getPubSub", - "weight": 130, + "weight": 129, "cookies": false, "type": "", "deprecated": false, @@ -11718,54 +11720,6 @@ ] } }, - "\/health\/queue": { - "get": { - "summary": "Get queue", - "operationId": "healthGetQueue", - "tags": [ - "health" - ], - "description": "Check the Appwrite queue messaging servers are up and connection is successful.", - "responses": { - "200": { - "description": "Health Status", - "content": { - "application\/json": { - "schema": { - "$ref": "#\/components\/schemas\/healthStatus" - } - } - } - } - }, - "x-appwrite": { - "method": "getQueue", - "weight": 129, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "health\/get-queue.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "health.read", - "platforms": [ - "server" - ], - "packaging": false, - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [], - "Key": [] - } - ] - } - }, "\/health\/queue\/builds": { "get": { "summary": "Get builds queue", @@ -11788,7 +11742,7 @@ }, "x-appwrite": { "method": "getQueueBuilds", - "weight": 136, + "weight": 135, "cookies": false, "type": "", "deprecated": false, @@ -11849,7 +11803,7 @@ }, "x-appwrite": { "method": "getQueueCertificates", - "weight": 135, + "weight": 134, "cookies": false, "type": "", "deprecated": false, @@ -11910,7 +11864,7 @@ }, "x-appwrite": { "method": "getQueueDatabases", - "weight": 137, + "weight": 136, "cookies": false, "type": "", "deprecated": false, @@ -11982,7 +11936,7 @@ }, "x-appwrite": { "method": "getQueueDeletes", - "weight": 138, + "weight": 137, "cookies": false, "type": "", "deprecated": false, @@ -12081,8 +12035,9 @@ "v1-audits", "v1-mails", "v1-functions", - "v1-usage", - "v1-usage-dump", + "v1-stats-resources", + "v1-stats-usage", + "v1-stats-usage-dump", "v1-webhooks", "v1-certificates", "v1-builds", @@ -12130,7 +12085,7 @@ }, "x-appwrite": { "method": "getQueueFunctions", - "weight": 142, + "weight": 141, "cookies": false, "type": "", "deprecated": false, @@ -12191,7 +12146,7 @@ }, "x-appwrite": { "method": "getQueueLogs", - "weight": 133, + "weight": 132, "cookies": false, "type": "", "deprecated": false, @@ -12252,7 +12207,7 @@ }, "x-appwrite": { "method": "getQueueMails", - "weight": 139, + "weight": 138, "cookies": false, "type": "", "deprecated": false, @@ -12313,7 +12268,7 @@ }, "x-appwrite": { "method": "getQueueMessaging", - "weight": 140, + "weight": 139, "cookies": false, "type": "", "deprecated": false, @@ -12374,7 +12329,7 @@ }, "x-appwrite": { "method": "getQueueMigrations", - "weight": 141, + "weight": 140, "cookies": false, "type": "", "deprecated": false, @@ -12413,14 +12368,14 @@ ] } }, - "\/health\/queue\/usage": { + "\/health\/queue\/stats-resources": { "get": { - "summary": "Get usage queue", - "operationId": "healthGetQueueUsage", + "summary": "Get stats resources queue", + "operationId": "healthGetQueueStatsResources", "tags": [ "health" ], - "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "description": "Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.", "responses": { "200": { "description": "Health Queue", @@ -12434,13 +12389,13 @@ } }, "x-appwrite": { - "method": "getQueueUsage", - "weight": 143, + "method": "getQueueStatsResources", + "weight": 142, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage.md", + "demo": "health\/get-queue-stats-resources.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-resources.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -12474,10 +12429,71 @@ ] } }, - "\/health\/queue\/usage-dump": { + "\/health\/queue\/stats-usage": { + "get": { + "summary": "Get stats usage queue", + "operationId": "healthGetQueueUsage", + "tags": [ + "health" + ], + "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "responses": { + "200": { + "description": "Health Queue", + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/healthQueue" + } + } + } + } + }, + "x-appwrite": { + "method": "getQueueUsage", + "weight": 143, + "cookies": false, + "type": "", + "deprecated": false, + "demo": "health\/get-queue-usage.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage.md", + "rate-limit": 0, + "rate-time": 3600, + "rate-key": "url:{url},ip:{ip}", + "scope": "health.read", + "platforms": [ + "server" + ], + "packaging": false, + "auth": { + "Project": [] + } + }, + "security": [ + { + "Project": [], + "Key": [] + } + ], + "parameters": [ + { + "name": "threshold", + "description": "Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.", + "required": false, + "schema": { + "type": "integer", + "format": "int32", + "default": 5000 + }, + "in": "query" + } + ] + } + }, + "\/health\/queue\/stats-usage-dump": { "get": { "summary": "Get usage dump queue", - "operationId": "healthGetQueueUsageDump", + "operationId": "healthGetQueueStatsUsageDump", "tags": [ "health" ], @@ -12495,13 +12511,13 @@ } }, "x-appwrite": { - "method": "getQueueUsageDump", + "method": "getQueueStatsUsageDump", "weight": 144, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage-dump.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage-dump.md", + "demo": "health\/get-queue-stats-usage-dump.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage-dump.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -12557,7 +12573,7 @@ }, "x-appwrite": { "method": "getQueueWebhooks", - "weight": 132, + "weight": 131, "cookies": false, "type": "", "deprecated": false, @@ -12714,7 +12730,7 @@ }, "x-appwrite": { "method": "getTime", - "weight": 131, + "weight": 130, "cookies": false, "type": "", "deprecated": false, @@ -17609,17 +17625,17 @@ }, "endpoint": { "type": "string", - "description": "Source's Appwrite Endpoint", + "description": "Source Appwrite endpoint", "x-example": "https:\/\/example.com" }, "projectId": { "type": "string", - "description": "Source's Project ID", + "description": "Source Project ID", "x-example": "" }, "apiKey": { "type": "string", - "description": "Source's API Key", + "description": "Source API Key", "x-example": "" } }, @@ -36052,6 +36068,20 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "imageTransformations": { + "type": "array", + "description": "Aggregated number of files transformations per period.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] + }, + "imageTransformationsTotal": { + "type": "integer", + "description": "Total aggregated number of files transformations.", + "x-example": 0, + "format": "int32" } }, "required": [ @@ -36059,7 +36089,9 @@ "filesTotal", "filesStorageTotal", "files", - "storage" + "storage", + "imageTransformations", + "imageTransformationsTotal" ] }, "usageFunctions": { @@ -36597,6 +36629,14 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "imageTransformations": { + "type": "array", + "description": "An array of aggregated number of image transformations.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] } }, "required": [ @@ -36628,7 +36668,8 @@ "authPhoneEstimate", "authPhoneCountryBreakdown", "databasesReads", - "databasesWrites" + "databasesWrites", + "imageTransformations" ] }, "headers": { diff --git a/app/config/specs/open-api3-latest-server.json b/app/config/specs/open-api3-latest-server.json index 68d408762a..280c080514 100644 --- a/app/config/specs/open-api3-latest-server.json +++ b/app/config/specs/open-api3-latest-server.json @@ -3077,7 +3077,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "schema": { "type": "string", @@ -3098,7 +3098,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3117,7 +3118,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ] }, "in": "path" @@ -7381,7 +7383,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", @@ -10461,7 +10463,7 @@ }, "x-appwrite": { "method": "getCertificate", - "weight": 134, + "weight": 133, "cookies": false, "type": "", "deprecated": false, @@ -10570,7 +10572,7 @@ }, "x-appwrite": { "method": "getPubSub", - "weight": 130, + "weight": 129, "cookies": false, "type": "", "deprecated": false, @@ -10597,55 +10599,6 @@ ] } }, - "\/health\/queue": { - "get": { - "summary": "Get queue", - "operationId": "healthGetQueue", - "tags": [ - "health" - ], - "description": "Check the Appwrite queue messaging servers are up and connection is successful.", - "responses": { - "200": { - "description": "Health Status", - "content": { - "application\/json": { - "schema": { - "$ref": "#\/components\/schemas\/healthStatus" - } - } - } - } - }, - "x-appwrite": { - "method": "getQueue", - "weight": 129, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "health\/get-queue.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "health.read", - "platforms": [ - "server" - ], - "packaging": false, - "auth": { - "Project": [], - "Key": [] - } - }, - "security": [ - { - "Project": [], - "Key": [] - } - ] - } - }, "\/health\/queue\/builds": { "get": { "summary": "Get builds queue", @@ -10668,7 +10621,7 @@ }, "x-appwrite": { "method": "getQueueBuilds", - "weight": 136, + "weight": 135, "cookies": false, "type": "", "deprecated": false, @@ -10730,7 +10683,7 @@ }, "x-appwrite": { "method": "getQueueCertificates", - "weight": 135, + "weight": 134, "cookies": false, "type": "", "deprecated": false, @@ -10792,7 +10745,7 @@ }, "x-appwrite": { "method": "getQueueDatabases", - "weight": 137, + "weight": 136, "cookies": false, "type": "", "deprecated": false, @@ -10865,7 +10818,7 @@ }, "x-appwrite": { "method": "getQueueDeletes", - "weight": 138, + "weight": 137, "cookies": false, "type": "", "deprecated": false, @@ -10966,8 +10919,9 @@ "v1-audits", "v1-mails", "v1-functions", - "v1-usage", - "v1-usage-dump", + "v1-stats-resources", + "v1-stats-usage", + "v1-stats-usage-dump", "v1-webhooks", "v1-certificates", "v1-builds", @@ -11015,7 +10969,7 @@ }, "x-appwrite": { "method": "getQueueFunctions", - "weight": 142, + "weight": 141, "cookies": false, "type": "", "deprecated": false, @@ -11077,7 +11031,7 @@ }, "x-appwrite": { "method": "getQueueLogs", - "weight": 133, + "weight": 132, "cookies": false, "type": "", "deprecated": false, @@ -11139,7 +11093,7 @@ }, "x-appwrite": { "method": "getQueueMails", - "weight": 139, + "weight": 138, "cookies": false, "type": "", "deprecated": false, @@ -11201,7 +11155,7 @@ }, "x-appwrite": { "method": "getQueueMessaging", - "weight": 140, + "weight": 139, "cookies": false, "type": "", "deprecated": false, @@ -11263,7 +11217,7 @@ }, "x-appwrite": { "method": "getQueueMigrations", - "weight": 141, + "weight": 140, "cookies": false, "type": "", "deprecated": false, @@ -11303,14 +11257,14 @@ ] } }, - "\/health\/queue\/usage": { + "\/health\/queue\/stats-resources": { "get": { - "summary": "Get usage queue", - "operationId": "healthGetQueueUsage", + "summary": "Get stats resources queue", + "operationId": "healthGetQueueStatsResources", "tags": [ "health" ], - "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "description": "Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.", "responses": { "200": { "description": "Health Queue", @@ -11324,13 +11278,13 @@ } }, "x-appwrite": { - "method": "getQueueUsage", - "weight": 143, + "method": "getQueueStatsResources", + "weight": 142, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage.md", + "demo": "health\/get-queue-stats-resources.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-resources.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -11365,10 +11319,72 @@ ] } }, - "\/health\/queue\/usage-dump": { + "\/health\/queue\/stats-usage": { + "get": { + "summary": "Get stats usage queue", + "operationId": "healthGetQueueUsage", + "tags": [ + "health" + ], + "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "responses": { + "200": { + "description": "Health Queue", + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/healthQueue" + } + } + } + } + }, + "x-appwrite": { + "method": "getQueueUsage", + "weight": 143, + "cookies": false, + "type": "", + "deprecated": false, + "demo": "health\/get-queue-usage.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage.md", + "rate-limit": 0, + "rate-time": 3600, + "rate-key": "url:{url},ip:{ip}", + "scope": "health.read", + "platforms": [ + "server" + ], + "packaging": false, + "auth": { + "Project": [], + "Key": [] + } + }, + "security": [ + { + "Project": [], + "Key": [] + } + ], + "parameters": [ + { + "name": "threshold", + "description": "Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.", + "required": false, + "schema": { + "type": "integer", + "format": "int32", + "default": 5000 + }, + "in": "query" + } + ] + } + }, + "\/health\/queue\/stats-usage-dump": { "get": { "summary": "Get usage dump queue", - "operationId": "healthGetQueueUsageDump", + "operationId": "healthGetQueueStatsUsageDump", "tags": [ "health" ], @@ -11386,13 +11402,13 @@ } }, "x-appwrite": { - "method": "getQueueUsageDump", + "method": "getQueueStatsUsageDump", "weight": 144, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage-dump.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage-dump.md", + "demo": "health\/get-queue-stats-usage-dump.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage-dump.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -11449,7 +11465,7 @@ }, "x-appwrite": { "method": "getQueueWebhooks", - "weight": 132, + "weight": 131, "cookies": false, "type": "", "deprecated": false, @@ -11609,7 +11625,7 @@ }, "x-appwrite": { "method": "getTime", - "weight": 131, + "weight": 130, "cookies": false, "type": "", "deprecated": false, diff --git a/app/config/specs/swagger2-latest-client.json b/app/config/specs/swagger2-latest-client.json index c0980c44ce..8960bfaa5c 100644 --- a/app/config/specs/swagger2-latest-client.json +++ b/app/config/specs/swagger2-latest-client.json @@ -3557,7 +3557,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "type": "string", "x-example": "amex", @@ -3577,7 +3577,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3596,7 +3597,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ], "in": "path" }, @@ -4547,7 +4549,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index 94b0d55199..f13db98150 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -3577,7 +3577,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "type": "string", "x-example": "amex", @@ -3597,7 +3597,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3616,7 +3617,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ], "in": "path" }, @@ -8012,7 +8014,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", @@ -11810,7 +11812,7 @@ }, "x-appwrite": { "method": "getCertificate", - "weight": 134, + "weight": 133, "cookies": false, "type": "", "deprecated": false, @@ -11919,7 +11921,7 @@ }, "x-appwrite": { "method": "getPubSub", - "weight": 130, + "weight": 129, "cookies": false, "type": "", "deprecated": false, @@ -11945,56 +11947,6 @@ ] } }, - "\/health\/queue": { - "get": { - "summary": "Get queue", - "operationId": "healthGetQueue", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "health" - ], - "description": "Check the Appwrite queue messaging servers are up and connection is successful.", - "responses": { - "200": { - "description": "Health Status", - "schema": { - "$ref": "#\/definitions\/healthStatus" - } - } - }, - "x-appwrite": { - "method": "getQueue", - "weight": 129, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "health\/get-queue.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "health.read", - "platforms": [ - "server" - ], - "packaging": false, - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [], - "Key": [] - } - ] - } - }, "\/health\/queue\/builds": { "get": { "summary": "Get builds queue", @@ -12019,7 +11971,7 @@ }, "x-appwrite": { "method": "getQueueBuilds", - "weight": 136, + "weight": 135, "cookies": false, "type": "", "deprecated": false, @@ -12080,7 +12032,7 @@ }, "x-appwrite": { "method": "getQueueCertificates", - "weight": 135, + "weight": 134, "cookies": false, "type": "", "deprecated": false, @@ -12141,7 +12093,7 @@ }, "x-appwrite": { "method": "getQueueDatabases", - "weight": 137, + "weight": 136, "cookies": false, "type": "", "deprecated": false, @@ -12211,7 +12163,7 @@ }, "x-appwrite": { "method": "getQueueDeletes", - "weight": 138, + "weight": 137, "cookies": false, "type": "", "deprecated": false, @@ -12309,8 +12261,9 @@ "v1-audits", "v1-mails", "v1-functions", - "v1-usage", - "v1-usage-dump", + "v1-stats-resources", + "v1-stats-usage", + "v1-stats-usage-dump", "v1-webhooks", "v1-certificates", "v1-builds", @@ -12357,7 +12310,7 @@ }, "x-appwrite": { "method": "getQueueFunctions", - "weight": 142, + "weight": 141, "cookies": false, "type": "", "deprecated": false, @@ -12418,7 +12371,7 @@ }, "x-appwrite": { "method": "getQueueLogs", - "weight": 133, + "weight": 132, "cookies": false, "type": "", "deprecated": false, @@ -12479,7 +12432,7 @@ }, "x-appwrite": { "method": "getQueueMails", - "weight": 139, + "weight": 138, "cookies": false, "type": "", "deprecated": false, @@ -12540,7 +12493,7 @@ }, "x-appwrite": { "method": "getQueueMessaging", - "weight": 140, + "weight": 139, "cookies": false, "type": "", "deprecated": false, @@ -12601,7 +12554,7 @@ }, "x-appwrite": { "method": "getQueueMigrations", - "weight": 141, + "weight": 140, "cookies": false, "type": "", "deprecated": false, @@ -12638,10 +12591,10 @@ ] } }, - "\/health\/queue\/usage": { + "\/health\/queue\/stats-resources": { "get": { - "summary": "Get usage queue", - "operationId": "healthGetQueueUsage", + "summary": "Get stats resources queue", + "operationId": "healthGetQueueStatsResources", "consumes": [ "application\/json" ], @@ -12651,7 +12604,7 @@ "tags": [ "health" ], - "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "description": "Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.", "responses": { "200": { "description": "Health Queue", @@ -12661,13 +12614,13 @@ } }, "x-appwrite": { - "method": "getQueueUsage", - "weight": 143, + "method": "getQueueStatsResources", + "weight": 142, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage.md", + "demo": "health\/get-queue-stats-resources.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-resources.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -12699,10 +12652,71 @@ ] } }, - "\/health\/queue\/usage-dump": { + "\/health\/queue\/stats-usage": { + "get": { + "summary": "Get stats usage queue", + "operationId": "healthGetQueueUsage", + "consumes": [ + "application\/json" + ], + "produces": [ + "application\/json" + ], + "tags": [ + "health" + ], + "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "responses": { + "200": { + "description": "Health Queue", + "schema": { + "$ref": "#\/definitions\/healthQueue" + } + } + }, + "x-appwrite": { + "method": "getQueueUsage", + "weight": 143, + "cookies": false, + "type": "", + "deprecated": false, + "demo": "health\/get-queue-usage.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage.md", + "rate-limit": 0, + "rate-time": 3600, + "rate-key": "url:{url},ip:{ip}", + "scope": "health.read", + "platforms": [ + "server" + ], + "packaging": false, + "auth": { + "Project": [] + } + }, + "security": [ + { + "Project": [], + "Key": [] + } + ], + "parameters": [ + { + "name": "threshold", + "description": "Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.", + "required": false, + "type": "integer", + "format": "int32", + "default": 5000, + "in": "query" + } + ] + } + }, + "\/health\/queue\/stats-usage-dump": { "get": { "summary": "Get usage dump queue", - "operationId": "healthGetQueueUsageDump", + "operationId": "healthGetQueueStatsUsageDump", "consumes": [ "application\/json" ], @@ -12722,13 +12736,13 @@ } }, "x-appwrite": { - "method": "getQueueUsageDump", + "method": "getQueueStatsUsageDump", "weight": 144, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage-dump.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage-dump.md", + "demo": "health\/get-queue-stats-usage-dump.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage-dump.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -12784,7 +12798,7 @@ }, "x-appwrite": { "method": "getQueueWebhooks", - "weight": 132, + "weight": 131, "cookies": false, "type": "", "deprecated": false, @@ -12945,7 +12959,7 @@ }, "x-appwrite": { "method": "getTime", - "weight": 131, + "weight": 130, "cookies": false, "type": "", "deprecated": false, @@ -18070,19 +18084,19 @@ }, "endpoint": { "type": "string", - "description": "Source's Appwrite Endpoint", + "description": "Source Appwrite endpoint", "default": null, "x-example": "https:\/\/example.com" }, "projectId": { "type": "string", - "description": "Source's Project ID", + "description": "Source Project ID", "default": null, "x-example": "" }, "apiKey": { "type": "string", - "description": "Source's API Key", + "description": "Source API Key", "default": null, "x-example": "" } @@ -36608,6 +36622,21 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "imageTransformations": { + "type": "array", + "description": "Aggregated number of files transformations per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] + }, + "imageTransformationsTotal": { + "type": "integer", + "description": "Total aggregated number of files transformations.", + "x-example": 0, + "format": "int32" } }, "required": [ @@ -36615,7 +36644,9 @@ "filesTotal", "filesStorageTotal", "files", - "storage" + "storage", + "imageTransformations", + "imageTransformationsTotal" ] }, "usageFunctions": { @@ -37185,6 +37216,15 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "imageTransformations": { + "type": "array", + "description": "An array of aggregated number of image transformations.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] } }, "required": [ @@ -37216,7 +37256,8 @@ "authPhoneEstimate", "authPhoneCountryBreakdown", "databasesReads", - "databasesWrites" + "databasesWrites", + "imageTransformations" ] }, "headers": { diff --git a/app/config/specs/swagger2-latest-server.json b/app/config/specs/swagger2-latest-server.json index e38495629c..749f87553b 100644 --- a/app/config/specs/swagger2-latest-server.json +++ b/app/config/specs/swagger2-latest-server.json @@ -3261,7 +3261,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "type": "string", "x-example": "amex", @@ -3281,7 +3281,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3300,7 +3301,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ], "in": "path" }, @@ -7552,7 +7554,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", @@ -10689,7 +10691,7 @@ }, "x-appwrite": { "method": "getCertificate", - "weight": 134, + "weight": 133, "cookies": false, "type": "", "deprecated": false, @@ -10800,7 +10802,7 @@ }, "x-appwrite": { "method": "getPubSub", - "weight": 130, + "weight": 129, "cookies": false, "type": "", "deprecated": false, @@ -10827,57 +10829,6 @@ ] } }, - "\/health\/queue": { - "get": { - "summary": "Get queue", - "operationId": "healthGetQueue", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "health" - ], - "description": "Check the Appwrite queue messaging servers are up and connection is successful.", - "responses": { - "200": { - "description": "Health Status", - "schema": { - "$ref": "#\/definitions\/healthStatus" - } - } - }, - "x-appwrite": { - "method": "getQueue", - "weight": 129, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "health\/get-queue.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "health.read", - "platforms": [ - "server" - ], - "packaging": false, - "auth": { - "Project": [], - "Key": [] - } - }, - "security": [ - { - "Project": [], - "Key": [] - } - ] - } - }, "\/health\/queue\/builds": { "get": { "summary": "Get builds queue", @@ -10902,7 +10853,7 @@ }, "x-appwrite": { "method": "getQueueBuilds", - "weight": 136, + "weight": 135, "cookies": false, "type": "", "deprecated": false, @@ -10964,7 +10915,7 @@ }, "x-appwrite": { "method": "getQueueCertificates", - "weight": 135, + "weight": 134, "cookies": false, "type": "", "deprecated": false, @@ -11026,7 +10977,7 @@ }, "x-appwrite": { "method": "getQueueDatabases", - "weight": 137, + "weight": 136, "cookies": false, "type": "", "deprecated": false, @@ -11097,7 +11048,7 @@ }, "x-appwrite": { "method": "getQueueDeletes", - "weight": 138, + "weight": 137, "cookies": false, "type": "", "deprecated": false, @@ -11197,8 +11148,9 @@ "v1-audits", "v1-mails", "v1-functions", - "v1-usage", - "v1-usage-dump", + "v1-stats-resources", + "v1-stats-usage", + "v1-stats-usage-dump", "v1-webhooks", "v1-certificates", "v1-builds", @@ -11245,7 +11197,7 @@ }, "x-appwrite": { "method": "getQueueFunctions", - "weight": 142, + "weight": 141, "cookies": false, "type": "", "deprecated": false, @@ -11307,7 +11259,7 @@ }, "x-appwrite": { "method": "getQueueLogs", - "weight": 133, + "weight": 132, "cookies": false, "type": "", "deprecated": false, @@ -11369,7 +11321,7 @@ }, "x-appwrite": { "method": "getQueueMails", - "weight": 139, + "weight": 138, "cookies": false, "type": "", "deprecated": false, @@ -11431,7 +11383,7 @@ }, "x-appwrite": { "method": "getQueueMessaging", - "weight": 140, + "weight": 139, "cookies": false, "type": "", "deprecated": false, @@ -11493,7 +11445,7 @@ }, "x-appwrite": { "method": "getQueueMigrations", - "weight": 141, + "weight": 140, "cookies": false, "type": "", "deprecated": false, @@ -11531,10 +11483,10 @@ ] } }, - "\/health\/queue\/usage": { + "\/health\/queue\/stats-resources": { "get": { - "summary": "Get usage queue", - "operationId": "healthGetQueueUsage", + "summary": "Get stats resources queue", + "operationId": "healthGetQueueStatsResources", "consumes": [ "application\/json" ], @@ -11544,7 +11496,7 @@ "tags": [ "health" ], - "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "description": "Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.", "responses": { "200": { "description": "Health Queue", @@ -11554,13 +11506,13 @@ } }, "x-appwrite": { - "method": "getQueueUsage", - "weight": 143, + "method": "getQueueStatsResources", + "weight": 142, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage.md", + "demo": "health\/get-queue-stats-resources.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-resources.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -11593,10 +11545,72 @@ ] } }, - "\/health\/queue\/usage-dump": { + "\/health\/queue\/stats-usage": { + "get": { + "summary": "Get stats usage queue", + "operationId": "healthGetQueueUsage", + "consumes": [ + "application\/json" + ], + "produces": [ + "application\/json" + ], + "tags": [ + "health" + ], + "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "responses": { + "200": { + "description": "Health Queue", + "schema": { + "$ref": "#\/definitions\/healthQueue" + } + } + }, + "x-appwrite": { + "method": "getQueueUsage", + "weight": 143, + "cookies": false, + "type": "", + "deprecated": false, + "demo": "health\/get-queue-usage.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage.md", + "rate-limit": 0, + "rate-time": 3600, + "rate-key": "url:{url},ip:{ip}", + "scope": "health.read", + "platforms": [ + "server" + ], + "packaging": false, + "auth": { + "Project": [], + "Key": [] + } + }, + "security": [ + { + "Project": [], + "Key": [] + } + ], + "parameters": [ + { + "name": "threshold", + "description": "Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.", + "required": false, + "type": "integer", + "format": "int32", + "default": 5000, + "in": "query" + } + ] + } + }, + "\/health\/queue\/stats-usage-dump": { "get": { "summary": "Get usage dump queue", - "operationId": "healthGetQueueUsageDump", + "operationId": "healthGetQueueStatsUsageDump", "consumes": [ "application\/json" ], @@ -11616,13 +11630,13 @@ } }, "x-appwrite": { - "method": "getQueueUsageDump", + "method": "getQueueStatsUsageDump", "weight": 144, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage-dump.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage-dump.md", + "demo": "health\/get-queue-stats-usage-dump.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage-dump.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -11679,7 +11693,7 @@ }, "x-appwrite": { "method": "getQueueWebhooks", - "weight": 132, + "weight": 131, "cookies": false, "type": "", "deprecated": false, @@ -11843,7 +11857,7 @@ }, "x-appwrite": { "method": "getTime", - "weight": 131, + "weight": 130, "cookies": false, "type": "", "deprecated": false, From df3746271684ad08c1b1379bf9d784122bdd5262 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 28 Feb 2025 13:20:55 +0000 Subject: [PATCH 094/275] chore: fix typing and tests --- app/config/specs/open-api3-latest-console.json | 10 ++++------ app/config/specs/swagger2-latest-console.json | 11 ++++------- src/Appwrite/Utopia/Response/Model/UsageProject.php | 9 ++++----- tests/e2e/General/UsageTest.php | 2 +- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 5b1d6f0d1e..5b31a62c23 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -36631,12 +36631,10 @@ "x-example": [] }, "imageTransformations": { - "type": "array", - "description": "An array of aggregated number of image transformations.", - "items": { - "$ref": "#\/components\/schemas\/metric" - }, - "x-example": [] + "type": "integer", + "description": "Total aggregated number of image transformations.", + "x-example": 0, + "format": "int32" } }, "required": [ diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index f13db98150..5c2dabd318 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -37218,13 +37218,10 @@ "x-example": [] }, "imageTransformations": { - "type": "array", - "description": "An array of aggregated number of image transformations.", - "items": { - "type": "object", - "$ref": "#\/definitions\/metric" - }, - "x-example": [] + "type": "integer", + "description": "Total aggregated number of image transformations.", + "x-example": 0, + "format": "int32" } }, "required": [ diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 6f74ec561b..bdb7d6a897 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -198,11 +198,10 @@ class UsageProject extends Model 'array' => true ]) ->addRule('imageTransformations', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'An array of aggregated number of image transformations.', - 'default' => [], - 'example' => [], - 'array' => true + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of image transformations.', + 'default' => 0, + 'example' => 0, ]) ; } diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index c0d0c80eb1..9061f95a96 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -150,7 +150,7 @@ class UsageTest extends Scope ); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(29, count($response['body'])); + $this->assertEquals(30, count($response['body'])); $this->validateDates($response['body']['network']); $this->validateDates($response['body']['requests']); $this->validateDates($response['body']['users']); From 02de0d2432750728768fe6efb0a81cd21eff54d9 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 28 Feb 2025 13:29:32 +0000 Subject: [PATCH 095/275] chore: fix stats count --- tests/e2e/General/UsageTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 9061f95a96..2e13dcdb98 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -331,7 +331,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(29, count($response['body'])); + $this->assertEquals(30, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); $this->validateDates($response['body']['requests']); @@ -552,7 +552,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(29, count($response['body'])); + $this->assertEquals(30, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals(1, count($response['body']['network'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); From 712c33064a3bd4d7af83278dab9ad13a17ff8759 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 1 Mar 2025 17:18:44 +0000 Subject: [PATCH 096/275] chore: fix sending of transformation stats --- app/controllers/api/project.php | 4 +++- src/Appwrite/Utopia/Response/Model/UsageProject.php | 6 ++++++ tests/e2e/General/UsageTest.php | 6 +++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index cf2a128939..986264e3c5 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -76,6 +76,7 @@ App::get('/v1/project/usage') METRIC_BUILDS_MB_SECONDS, METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, + METRIC_FILES_IMAGES_TRANSFORMED, ] ]; @@ -364,7 +365,8 @@ App::get('/v1/project/usage') 'authPhoneTotal' => $authPhoneTotal, 'authPhoneEstimate' => $authPhoneEstimate, 'authPhoneCountryBreakdown' => $authPhoneCountryBreakdown, - 'imageTransformations' => $total[METRIC_FILES_IMAGES_TRANSFORMED], + 'imageTransformations' => $usage[METRIC_FILES_IMAGES_TRANSFORMED], + 'imageTransformationsTotal' => $total[METRIC_FILES_IMAGES_TRANSFORMED], ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index bdb7d6a897..6ac8830ac5 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -197,6 +197,12 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('imageTransformationsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'An array of aggregated number of image transformations.', + 'default' => 0, + 'example' => 0, + ]) ->addRule('imageTransformations', [ 'type' => self::TYPE_INTEGER, 'description' => 'Total aggregated number of image transformations.', diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 2e13dcdb98..33a1408704 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -150,7 +150,7 @@ class UsageTest extends Scope ); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(30, count($response['body'])); + $this->assertEquals(31, count($response['body'])); $this->validateDates($response['body']['network']); $this->validateDates($response['body']['requests']); $this->validateDates($response['body']['users']); @@ -331,7 +331,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(30, count($response['body'])); + $this->assertEquals(31, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); $this->validateDates($response['body']['requests']); @@ -552,7 +552,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(30, count($response['body'])); + $this->assertEquals(31, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals(1, count($response['body']['network'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); From aaee99e08b950affc760ee193776e0ec4d8df767 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 1 Mar 2025 17:19:55 +0000 Subject: [PATCH 097/275] chore: generate specs --- app/config/specs/open-api3-latest-console.json | 7 +++++++ app/config/specs/swagger2-latest-console.json | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 5b31a62c23..bc216226fb 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -36630,6 +36630,12 @@ }, "x-example": [] }, + "imageTransformationsTotal": { + "type": "integer", + "description": "An array of aggregated number of image transformations.", + "x-example": 0, + "format": "int32" + }, "imageTransformations": { "type": "integer", "description": "Total aggregated number of image transformations.", @@ -36667,6 +36673,7 @@ "authPhoneCountryBreakdown", "databasesReads", "databasesWrites", + "imageTransformationsTotal", "imageTransformations" ] }, diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index 5c2dabd318..2dddc72802 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -37217,6 +37217,12 @@ }, "x-example": [] }, + "imageTransformationsTotal": { + "type": "integer", + "description": "An array of aggregated number of image transformations.", + "x-example": 0, + "format": "int32" + }, "imageTransformations": { "type": "integer", "description": "Total aggregated number of image transformations.", @@ -37254,6 +37260,7 @@ "authPhoneCountryBreakdown", "databasesReads", "databasesWrites", + "imageTransformationsTotal", "imageTransformations" ] }, From 6900b717da261b71bfbb928c96e8fa264f6d5ea4 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 1 Mar 2025 17:45:20 +0000 Subject: [PATCH 098/275] chore: updated to use logsdb --- app/controllers/api/project.php | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 986264e3c5..b267e8e51e 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -40,14 +40,18 @@ App::get('/v1/project/usage') ->param('endDate', '', new DateTimeValidator(), 'End date for the usage') ->param('period', '1d', new WhiteList(['1h', '1d']), 'Period used', true) ->inject('response') + ->inject('project') ->inject('dbForProject') + ->inject('getLogsDB') ->inject('smsRates') - ->action(function (string $startDate, string $endDate, string $period, Response $response, Database $dbForProject, array $smsRates) { + ->action(function (string $startDate, string $endDate, string $period, Response $response, Document $project, Database $dbForProject, callable $getLogsDB, array $smsRates) { $stats = $total = $usage = []; $format = 'Y-m-d 00:00:00'; $firstDay = (new DateTime($startDate))->format($format); $lastDay = (new DateTime($endDate))->format($format); + $dbForLogs = call_user_func($getLogsDB, $project); + $metrics = [ 'total' => [ METRIC_EXECUTIONS, @@ -95,9 +99,11 @@ App::get('/v1/project/usage') '1d' => 'Y-m-d\T00:00:00.000P', }; - Authorization::skip(function () use ($dbForProject, $firstDay, $lastDay, $period, $metrics, $limit, &$total, &$stats) { + Authorization::skip(function () use ($dbForProject, $dbForLogs, $firstDay, $lastDay, $period, $metrics, $limit, &$total, &$stats) { foreach ($metrics['total'] as $metric) { - $result = $dbForProject->findOne('stats', [ + $db = ($metric === METRIC_FILES_IMAGES_TRANSFORMED) ? $dbForLogs : $dbForProject; + + $result = $db->findOne('stats', [ Query::equal('metric', [$metric]), Query::equal('period', ['inf']) ]); @@ -105,7 +111,9 @@ App::get('/v1/project/usage') } foreach ($metrics['period'] as $metric) { - $results = $dbForProject->find('stats', [ + $db = ($metric === METRIC_FILES_IMAGES_TRANSFORMED) ? $dbForLogs : $dbForProject; + + $results = $db->find('stats', [ Query::equal('metric', [$metric]), Query::equal('period', [$period]), Query::greaterThanEqual('time', $firstDay), From 5becd0a5dc9553fb0e120ad5ee52e0785f55a28f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 3 Mar 2025 16:04:31 +0545 Subject: [PATCH 099/275] Revert "Fix: stats resources only queue projects accessed in last 3 hours" --- src/Appwrite/Platform/Tasks/StatsResources.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/StatsResources.php b/src/Appwrite/Platform/Tasks/StatsResources.php index c318f92cf1..ac3b9ead73 100644 --- a/src/Appwrite/Platform/Tasks/StatsResources.php +++ b/src/Appwrite/Platform/Tasks/StatsResources.php @@ -62,7 +62,7 @@ class StatsResources extends Action Authorization::disable(); Authorization::setDefaultStatus(false); - $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('3 hours')); + $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours')); /** * For each project that were accessed in last 24 hours */ From b314ae82084fea574ac63d0a9f18bb2990ef88b1 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 3 Mar 2025 15:19:49 +0200 Subject: [PATCH 100/275] disable transformedAt update for console users --- app/controllers/api/storage.php | 20 ++++++++------------ app/controllers/shared/api.php | 14 ++++++++------ 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f180c22acf..5bf52f995f 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -6,7 +6,6 @@ use Appwrite\Auth\Auth; use Appwrite\ClamAV\Network; use Appwrite\Event\Delete; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\OpenSSL\OpenSSL; use Appwrite\SDK\AuthType; @@ -942,8 +941,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->inject('mode') ->inject('deviceForFiles') ->inject('deviceForLocal') - ->inject('queueForStatsUsage') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal, StatsUsage $queueForStatsUsage) { + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -1071,15 +1069,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; - $queueForStatsUsage - ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) - ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) - ; - - $transformedAt = $file->getAttribute('transformedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { - $file->setAttribute('transformedAt', DateTime::now()); - Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $file->getAttribute('bucketInternalId'), $file->getId(), $file)); + //Do not update transformedAt if it's a console user + if (!Auth::isPrivilegedUser(Authorization::getRoles())) { + $transformedAt = $file->getAttribute('transformedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { + $file->setAttribute('transformedAt', DateTime::now()); + Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $file->getAttribute('bucketInternalId'), $file->getId(), $file)); + } } $response diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 7f7b73ab0c..1015400a12 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -534,7 +534,7 @@ App::init() if ($type === 'bucket') { $bucketId = $parts[1] ?? null; - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAppUser && !$isPrivilegedUser)) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); @@ -560,11 +560,13 @@ App::init() if ($file->isEmpty()) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); } - - $transformedAt = $file->getAttribute('transformedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { - $file->setAttribute('transformedAt', DateTime::now()); - Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $file->getAttribute('bucketInternalId'), $file->getId(), $file)); + //Do not update transformedAt if it's a console user + if (!Auth::isPrivilegedUser(Authorization::getRoles())) { + $transformedAt = $file->getAttribute('transformedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { + $file->setAttribute('transformedAt', DateTime::now()); + Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $file->getAttribute('bucketInternalId'), $file->getId(), $file)); + } } } From c6b8e649fef5108bcbe70bc54270b01d851cd09b Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 3 Mar 2025 14:34:10 +0000 Subject: [PATCH 101/275] chore: added maxtimeout to testChannelExecutions --- tests/e2e/Services/Realtime/RealtimeCustomClientTest.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index dda524fc7c..c72dbf85ec 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -1311,6 +1311,8 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals($deployment['headers']['status-code'], 202); $this->assertNotEmpty($deployment['body']['$id']); + $maxTimeSeconds = 10; + $startTime = time(); // Poll until deployment is built while (true) { $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ @@ -1326,6 +1328,10 @@ class RealtimeCustomClientTest extends Scope break; } + if (time() - $startTime >= $maxTimeSeconds) { + throw new \Exception('Deployment timed out after ' . $maxTimeSeconds . ' seconds'); + } + \sleep(1); } From e865ca74d775393d12af377b4f120fd087a4d32b Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 3 Mar 2025 14:41:11 +0000 Subject: [PATCH 102/275] chore: added timeout to webhooks base test --- tests/e2e/Services/Webhooks/WebhooksBase.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/e2e/Services/Webhooks/WebhooksBase.php b/tests/e2e/Services/Webhooks/WebhooksBase.php index 2ef41003ee..ffcb1ce980 100644 --- a/tests/e2e/Services/Webhooks/WebhooksBase.php +++ b/tests/e2e/Services/Webhooks/WebhooksBase.php @@ -14,6 +14,8 @@ trait WebhooksBase { protected function awaitDeploymentIsBuilt($functionId, $deploymentId, $checkForSuccess = true): void { + $maxTimeSeconds = 10; + $startTime = time(); while (true) { $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/deployments/' . $deploymentId, [ 'content-type' => 'application/json', @@ -28,6 +30,10 @@ trait WebhooksBase break; } + if (time() - $startTime >= $maxTimeSeconds) { + throw new \Exception('Deployment timed out after ' . $maxTimeSeconds . ' seconds'); + } + \sleep(1); } From fc05a4f1e30e164c54e1330ef2cdcb77cd3cc36b Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 3 Mar 2025 16:07:33 +0000 Subject: [PATCH 103/275] chore: updated to use assertEventually --- .../Realtime/RealtimeCustomClientTest.php | 19 ++----------- tests/e2e/Services/Webhooks/WebhooksBase.php | 28 ++++++------------- 2 files changed, 11 insertions(+), 36 deletions(-) diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index c72dbf85ec..e356397408 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -1311,29 +1311,16 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals($deployment['headers']['status-code'], 202); $this->assertNotEmpty($deployment['body']['$id']); - $maxTimeSeconds = 10; - $startTime = time(); // Poll until deployment is built - while (true) { + $this->assertEventually(function () use ($function, $deploymentId) { $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'], ]); - if ( - $deployment['headers']['status-code'] >= 400 - || \in_array($deployment['body']['status'], ['ready', 'failed']) - ) { - break; - } - - if (time() - $startTime >= $maxTimeSeconds) { - throw new \Exception('Deployment timed out after ' . $maxTimeSeconds . ' seconds'); - } - - \sleep(1); - } + $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); + }); $response = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([ 'content-type' => 'application/json', diff --git a/tests/e2e/Services/Webhooks/WebhooksBase.php b/tests/e2e/Services/Webhooks/WebhooksBase.php index ffcb1ce980..b7985b5be1 100644 --- a/tests/e2e/Services/Webhooks/WebhooksBase.php +++ b/tests/e2e/Services/Webhooks/WebhooksBase.php @@ -2,6 +2,7 @@ namespace Tests\E2E\Services\Webhooks; +use Appwrite\Tests\Async; use Appwrite\Tests\Retry; use CURLFile; use Tests\E2E\Client; @@ -12,35 +13,22 @@ use Utopia\Database\Validator\Datetime as DatetimeValidator; trait WebhooksBase { + use Async; + protected function awaitDeploymentIsBuilt($functionId, $deploymentId, $checkForSuccess = true): void { - $maxTimeSeconds = 10; - $startTime = time(); - while (true) { + $this->assertEventually(function () use ($functionId, $deploymentId, $checkForSuccess) { $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/deployments/' . $deploymentId, [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'], ]); - if ( - $deployment['headers']['status-code'] >= 400 - || \in_array($deployment['body']['status'], ['ready', 'failed']) - ) { - break; + if ($checkForSuccess) { + $this->assertEquals(200, $deployment['headers']['status-code']); + $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); } - - if (time() - $startTime >= $maxTimeSeconds) { - throw new \Exception('Deployment timed out after ' . $maxTimeSeconds . ' seconds'); - } - - \sleep(1); - } - - if ($checkForSuccess) { - $this->assertEquals(200, $deployment['headers']['status-code']); - $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); - } + }); } public static function getWebhookSignature(array $webhook, string $signatureKey): string From fe7aeefabf9f2f1523df07ebfa3a060099688d70 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 3 Mar 2025 16:20:32 +0000 Subject: [PATCH 104/275] chore: remove redundant param --- tests/e2e/Services/Webhooks/WebhooksBase.php | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/e2e/Services/Webhooks/WebhooksBase.php b/tests/e2e/Services/Webhooks/WebhooksBase.php index b7985b5be1..c743810feb 100644 --- a/tests/e2e/Services/Webhooks/WebhooksBase.php +++ b/tests/e2e/Services/Webhooks/WebhooksBase.php @@ -15,19 +15,17 @@ trait WebhooksBase { use Async; - protected function awaitDeploymentIsBuilt($functionId, $deploymentId, $checkForSuccess = true): void + protected function awaitDeploymentIsBuilt($functionId, $deploymentId): void { - $this->assertEventually(function () use ($functionId, $deploymentId, $checkForSuccess) { + $this->assertEventually(function () use ($functionId, $deploymentId) { $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/deployments/' . $deploymentId, [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'], ]); - if ($checkForSuccess) { - $this->assertEquals(200, $deployment['headers']['status-code']); - $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); - } + $this->assertEquals(200, $deployment['headers']['status-code']); + $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); }); } From 9a6de87f7dcbdeff0d6c70f18cb1923ed87c2a6b Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 3 Mar 2025 16:31:41 +0000 Subject: [PATCH 105/275] chore: usagetest and migrationsbase --- tests/e2e/General/UsageTest.php | 67 ++++++++----------- .../Services/Migrations/MigrationsBase.php | 23 +++---- 2 files changed, 37 insertions(+), 53 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index df780d8f47..27d521aa6a 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1142,49 +1142,38 @@ class UsageTest extends Scope $tries = 0; - while (true) { - try { - // Compare new values with old values - $response = $this->client->call( - Client::METHOD_GET, - '/functions/' . $functionId . '/usage?range=30d', - $this->getConsoleHeaders() - ); + $this->assertEventually(function () use ($functionId, $functionsMetrics, $projectMetrics) { + // Compare new values with old values + $response = $this->client->call( + Client::METHOD_GET, + '/functions/' . $functionId . '/usage?range=30d', + $this->getConsoleHeaders() + ); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(19, count($response['body'])); - $this->assertEquals('30d', $response['body']['range']); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(19, count($response['body'])); + $this->assertEquals('30d', $response['body']['range']); - // Check if the new values are greater than the old values - $this->assertEquals($functionsMetrics['executionsTotal'] + 1, $response['body']['executionsTotal']); - $this->assertGreaterThan($functionsMetrics['executionsTimeTotal'], $response['body']['executionsTimeTotal']); - $this->assertGreaterThan($functionsMetrics['executionsMbSecondsTotal'], $response['body']['executionsMbSecondsTotal']); + // Check if the new values are greater than the old values + $this->assertEquals($functionsMetrics['executionsTotal'] + 1, $response['body']['executionsTotal']); + $this->assertGreaterThan($functionsMetrics['executionsTimeTotal'], $response['body']['executionsTimeTotal']); + $this->assertGreaterThan($functionsMetrics['executionsMbSecondsTotal'], $response['body']['executionsMbSecondsTotal']); - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals($projectMetrics['executionsTotal'] + 1, $response['body']['executionsTotal']); - $this->assertGreaterThan($projectMetrics['executionsMbSecondsTotal'], $response['body']['executionsMbSecondsTotal']); - - break; - } catch (ExpectationFailedException $th) { - if ($tries >= 5) { - throw $th; - } else { - $tries++; - sleep(5); - } - } - } + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($projectMetrics['executionsTotal'] + 1, $response['body']['executionsTotal']); + $this->assertGreaterThan($projectMetrics['executionsMbSecondsTotal'], $response['body']['executionsMbSecondsTotal']); + }); } public function tearDown(): void diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 381706f5ee..6c468ee730 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -53,8 +53,9 @@ trait MigrationsBase $this->assertNotEmpty($migration['body']); $this->assertNotEmpty($migration['body']['$id']); - $attempts = 0; - while ($attempts < 5) { + $migrationResult = []; + + $this->assertEventually(function () use ($migration, &$migrationResult) { $response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migration['body']['$id'], [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getDestinationProject()['$id'], @@ -66,24 +67,18 @@ trait MigrationsBase $this->assertNotEmpty($response['body']['$id']); if ($response['body']['status'] === 'failed') { - $this->fail('Migration failed', json_encode($response['body'], JSON_PRETTY_PRINT)); + $this->fail('Migration failed' . json_encode($response['body'], JSON_PRETTY_PRINT)); } $this->assertNotEquals('failed', $response['body']['status']); + $this->assertEquals('completed', $response['body']['status']); - if ($response['body']['status'] === 'completed') { - return $response['body']; - } + $migrationResult = $response['body']; - if ($attempts === 4) { - $this->assertEquals('completed', $response['body']['status']); - } + return true; + }); - $attempts++; - sleep(5); - } - - return []; + return $migrationResult; } /** From 5d6154444462ca8b49f0c8b1f97ec3e15e1a1e1b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 4 Mar 2025 05:31:35 +0000 Subject: [PATCH 106/275] Fix: use receivedAt date when available --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 5d7240f4b5..292fdffc00 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -142,7 +142,11 @@ class StatsUsageDump extends Action } foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); + $time = null; + + if ($period !== 'inf') { + $time = $receivedAt !== 'NONE' ? (new \DateTime($receivedAt))->format($format) : date($format, time()); + } $id = \md5("{$time}_{$period}_{$key}"); $document = new Document([ From dd5dd4cc43c26404a8e73a7e78e7063bac3464c2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 4 Mar 2025 05:34:02 +0000 Subject: [PATCH 107/275] Same for database metrics --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 292fdffc00..294269d586 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -134,7 +134,7 @@ class StatsUsageDump extends Action if (str_contains($key, METRIC_DATABASES_STORAGE)) { try { - $this->handleDatabaseStorage($key, $dbForProject, $project); + $this->handleDatabaseStorage($key, $dbForProject, $project, $receivedAt); } catch (\Exception $e) { console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); } @@ -175,12 +175,12 @@ class StatsUsageDump extends Action } } - private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project): void + private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project, string $receivedAt): void { $data = explode('.', $key); $start = microtime(true); - $updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) { + $updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) use ($receivedAt) { $id = \md5("{$time}_{$period}_{$key}"); $document = new Document([ @@ -201,7 +201,11 @@ class StatsUsageDump extends Action }; foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); + $time = null; + + if ($period !== 'inf') { + $time = $receivedAt !== 'NONE' ? (new \DateTime($receivedAt))->format($format) : date($format, time()); + } $id = \md5("{$time}_{$period}_{$key}"); $value = 0; From 4d5d30ba7bbab63508467788ecf5fd7a04df222e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 4 Mar 2025 05:50:21 +0000 Subject: [PATCH 108/275] Fix default value --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 294269d586..e03bb86b58 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -117,7 +117,7 @@ class StatsUsageDump extends Action $project = new Document($stats['project'] ?? []); $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; - $receivedAt = $stats['receivedAt'] ?? 'NONE'; + $receivedAt = $stats['receivedAt'] ?? null; if ($numberOfKeys === 0) { continue; } @@ -145,7 +145,7 @@ class StatsUsageDump extends Action $time = null; if ($period !== 'inf') { - $time = $receivedAt !== 'NONE' ? (new \DateTime($receivedAt))->format($format) : date($format, time()); + $time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time()); } $id = \md5("{$time}_{$period}_{$key}"); @@ -204,7 +204,7 @@ class StatsUsageDump extends Action $time = null; if ($period !== 'inf') { - $time = $receivedAt !== 'NONE' ? (new \DateTime($receivedAt))->format($format) : date($format, time()); + $time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time()); } $id = \md5("{$time}_{$period}_{$key}"); From 0659dce9357584dcf3e90283b739303a2fa6fdee Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 4 Mar 2025 06:22:58 +0000 Subject: [PATCH 109/275] Fix: disable dual writing --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index e03bb86b58..81acd4e4b0 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -328,7 +328,7 @@ class StatsUsageDump extends Action protected function writeToLogsDB(Document $project, Document $document): void { - if (!System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', false)) { + if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') { Console::log('Dual Writing is disabled. Skipping...'); return; } From 19ad2088a094b60e6aee4e5eb622b11316afbc89 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 10:08:07 +0200 Subject: [PATCH 110/275] update region --- app/controllers/api/functions.php | 2 +- app/controllers/api/projects.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 4 ++-- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 583468f6c1..0e2d8a0103 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -295,7 +295,7 @@ App::post('/v1/functions') $schedule = Authorization::skip( fn () => $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), // Todo replace with projects region + 'region' => $project->getAttribute('region'), 'resourceType' => 'function', 'resourceId' => $function->getId(), 'resourceInternalId' => $function->getInternalId(), diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 48d20cd17f..5099393e68 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -81,7 +81,7 @@ App::post('/v1/projects') ->param('projectId', '', new ProjectId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, and hyphen. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Project name. Max length: 128 chars.') ->param('teamId', '', new UID(), 'Team unique ID.') - ->param('region', System::getEnv('_APP_REGION', 'default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) + ->param('region', System::getEnv('_APP_REGION'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) ->param('description', '', new Text(256), 'Project description. Max length: 256 chars.', true) ->param('logo', '', new Text(1024), 'Project logo.', true) ->param('url', '', new URL(), 'Project URL.', true) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index dad2db0d9a..b62b95b066 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -104,7 +104,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', [System::getEnv('_APP_REGION')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); @@ -154,7 +154,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', [System::getEnv('_APP_REGION')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index e11181d199..9cb8f21a49 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -183,7 +183,7 @@ class Deletes extends Action $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', [System::getEnv('_APP_REGION')]), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], From 2a5b3317bf05bed6d57e2278d6dd2904ede59feb Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 11:36:45 +0200 Subject: [PATCH 111/275] update region --- .env | 1 + app/config/variables.php | 9 +++++++++ app/views/install/compose.phtml | 22 +++++++++++++++++++++- docker-compose.yml | 21 +++++++++++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 1893e023ba..9b4aef4ddd 100644 --- a/.env +++ b/.env @@ -1,6 +1,7 @@ _APP_ENV=development _APP_EDITION=self-hosted _APP_LOCALE=en +_APP_REGION=fra _APP_WORKER_PER_CORE=6 _APP_COMPRESSION_MIN_SIZE_BYTES=1024 _APP_CONSOLE_WHITELIST_ROOT=disabled diff --git a/app/config/variables.php b/app/config/variables.php index 98dd9ffec1..2f7a10221d 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -7,6 +7,15 @@ return [ 'category' => 'General', 'description' => '', 'variables' => [ + [ + 'name' => '_APP_REGION', + 'description' => 'Set your server running geo region. By default, the var is set to \'fra\'.', + 'introduction' => '', + 'default' => 'fra', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_ENV', 'description' => 'Set your server running environment. By default, the var is set to \'development\'. When deploying to production, change it to: \'production\'.', diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 62fcd03624..c85fbcafcf 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -164,7 +164,7 @@ $image = $this->getParam('image', ''); - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_ASSISTANT_OPENAI_API_KEY - + - _APP_REGION appwrite-console: <<: *x-logging container_name: appwrite-console @@ -229,6 +229,7 @@ $image = $this->getParam('image', ''); - _APP_DB_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-audits: image: /: @@ -255,6 +256,7 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-webhooks: image: /: @@ -283,6 +285,7 @@ $image = $this->getParam('image', ''); - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-deletes: image: /: @@ -342,6 +345,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_EXECUTION + - _APP_REGION appwrite-worker-databases: image: /: @@ -368,6 +372,7 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-builds: image: /: @@ -432,6 +437,7 @@ $image = $this->getParam('image', ''); - _APP_STORAGE_WASABI_SECRET - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET + - _APP_REGION appwrite-worker-certificates: image: /: @@ -465,6 +471,7 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-functions: image: /: @@ -503,6 +510,7 @@ $image = $this->getParam('image', ''); - _APP_DOCKER_HUB_USERNAME - _APP_DOCKER_HUB_PASSWORD - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-mails: image: /: @@ -537,6 +545,7 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_DOMAIN - _APP_OPTIONS_FORCE_HTTPS + - _APP_REGION appwrite-worker-messaging: image: /: @@ -588,6 +597,7 @@ $image = $this->getParam('image', ''); - _APP_STORAGE_WASABI_SECRET - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET + - _APP_REGION appwrite-worker-migrations: image: /: @@ -618,6 +628,7 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET + - _APP_REGION appwrite-task-maintenance: image: /: @@ -652,6 +663,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES + - _APP_REGION appwrite-task-stats-resources: image: /: @@ -683,6 +695,7 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL + - _APP_REGION appwrite-worker-stats-resources: image: /: @@ -711,6 +724,7 @@ $image = $this->getParam('image', ''); - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_STATS_RESOURCES_INTERVAL + - _APP_REGION appwrite-worker-stats-usage: image: /: @@ -739,6 +753,7 @@ $image = $this->getParam('image', ''); - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_REGION appwrite-worker-stats-usage-dump: image: /: @@ -767,6 +782,7 @@ $image = $this->getParam('image', ''); - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_REGION appwrite-task-scheduler-functions: image: /: @@ -792,6 +808,7 @@ $image = $this->getParam('image', ''); - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REGION appwrite-task-scheduler-executions: image: /: @@ -817,6 +834,7 @@ $image = $this->getParam('image', ''); - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REGION appwrite-task-scheduler-messages: image: /: @@ -842,6 +860,7 @@ $image = $this->getParam('image', ''); - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REGION appwrite-assistant: image: appwrite/assistant:0.4.0 @@ -877,6 +896,7 @@ $image = $this->getParam('image', ''); - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_ENV=$_APP_ENV + - OPR_EXECUTOR_REGION=$_APP_REGION - OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG diff --git a/docker-compose.yml b/docker-compose.yml index facf0e6db9..51689fd872 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -268,6 +268,7 @@ services: - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-audits: entrypoint: worker-audits @@ -297,6 +298,7 @@ services: - _APP_DB_PASS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-webhooks: entrypoint: worker-webhooks @@ -329,6 +331,7 @@ services: - _APP_LOGGING_CONFIG - _APP_WEBHOOK_MAX_FAILED_ATTEMPTS - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-deletes: entrypoint: worker-deletes @@ -389,6 +392,7 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES + - _APP_REGION appwrite-worker-databases: entrypoint: worker-databases @@ -420,6 +424,7 @@ services: - _APP_WORKERS_NUM - _APP_QUEUE_NAME - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-builds: entrypoint: worker-builds @@ -486,6 +491,7 @@ services: - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-certificates: entrypoint: worker-certificates @@ -521,6 +527,7 @@ services: - _APP_DB_PASS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-functions: entrypoint: worker-functions @@ -563,6 +570,7 @@ services: - _APP_LOGGING_CONFIG - _APP_LOGGING_PROVIDER - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-mails: entrypoint: worker-mails @@ -597,6 +605,7 @@ services: - _APP_DOMAIN - _APP_OPTIONS_FORCE_HTTPS - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-messaging: entrypoint: worker-messaging @@ -652,6 +661,7 @@ services: - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-migrations: entrypoint: worker-migrations @@ -687,6 +697,7 @@ services: - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-task-maintenance: entrypoint: maintenance @@ -725,6 +736,7 @@ services: - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_MAINTENANCE_DELAY - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-task-stats-resources: container_name: appwrite-task-stats-resources @@ -756,6 +768,7 @@ services: - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL + - _APP_REGION appwrite-worker-stats-resources: entrypoint: worker-stats-resources @@ -787,6 +800,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-stats-usage: entrypoint: worker-stats-usage @@ -818,6 +832,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-stats-usage-dump: entrypoint: worker-stats-usage-dump @@ -850,6 +865,7 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - _APP_STATS_USAGE_DUAL_WRITING_DBS + - _APP_REGION appwrite-task-scheduler-functions: entrypoint: schedule-functions @@ -878,6 +894,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-task-scheduler-executions: entrypoint: schedule-executions @@ -905,6 +922,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REGION appwrite-task-scheduler-messages: entrypoint: schedule-messages @@ -933,6 +951,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-assistant: container_name: appwrite-assistant @@ -966,6 +985,7 @@ services: - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_ENV=$_APP_ENV + - OPR_EXECUTOR_REGION=$_APP_REGION - OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - OPR_EXECUTOR_RUNTIME_VERSIONS=v2,v4 @@ -1005,6 +1025,7 @@ services: environment: - OPR_PROXY_WORKER_PER_CORE=$_APP_WORKER_PER_CORE - OPR_PROXY_ENV=$_APP_ENV + - OPR_PROXY_REGION=$_APP_REGION - OPR_PROXY_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - OPR_PROXY_SECRET=$_APP_EXECUTOR_SECRET - OPR_PROXY_LOGGING_CONFIG=$_APP_LOGGING_CONFIG From 8b4bc7e4a6b1a39af6b171bc3362bf7b5e1cafca Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 18:19:23 +0200 Subject: [PATCH 112/275] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 977aff3cfb..2d48750e9d 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -17,15 +17,15 @@ use Utopia\System\System; class Audits extends Action { - private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development - private const BATCH_SIZE_PRODUCTION = 5_000; - private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds + protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development + protected const BATCH_SIZE_PRODUCTION = 5_000; + protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds - private int $lastTriggeredTime = 0; + protected int $lastTriggeredTime = 0; - private array $logs = []; + protected array $logs = []; - private function getBatchSize(): int + protected function getBatchSize(): int { return System::getEnv('_APP_ENV', 'development') === 'development' ? self::BATCH_SIZE_DEVELOPMENT From f761829d6053580defc8eada81cb72061c42b07f Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 18:49:31 +0200 Subject: [PATCH 113/275] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 2d48750e9d..2a43a9e24f 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -17,7 +17,7 @@ use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development + protected const BATCH_SIZE_DEVELOPMENT = 2; // smaller batch size for development protected const BATCH_SIZE_PRODUCTION = 5_000; protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds From 76dc8c29d5e5bf8c2f8149e4a186331753082acc Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 19:07:45 +0200 Subject: [PATCH 114/275] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 2a43a9e24f..f22df17d31 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -21,7 +21,7 @@ class Audits extends Action protected const BATCH_SIZE_PRODUCTION = 5_000; protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds - protected int $lastTriggeredTime = 0; + private int $lastTriggeredTime = 0; protected array $logs = []; From ce054e7088db0966c3cbf0d7527b284b273af3c6 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 19:12:20 +0200 Subject: [PATCH 115/275] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index f22df17d31..85fe18ce9e 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -23,7 +23,7 @@ class Audits extends Action private int $lastTriggeredTime = 0; - protected array $logs = []; + private array $logs = []; protected function getBatchSize(): int { From 09aa383cf338fe53053725c1ade509a249ed9f32 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 19:15:00 +0200 Subject: [PATCH 116/275] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 85fe18ce9e..3221010a07 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -25,6 +25,7 @@ class Audits extends Action private array $logs = []; + protected function getBatchSize(): int { return System::getEnv('_APP_ENV', 'development') === 'development' From 1838b9c4e879f2eca2fd775dcd32df9821e35db5 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 19:49:10 +0200 Subject: [PATCH 117/275] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 3221010a07..c663ffc6d4 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -17,9 +17,9 @@ use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 2; // smaller batch size for development + protected const BATCH_SIZE_DEVELOPMENT = 26; // smaller batch size for development protected const BATCH_SIZE_PRODUCTION = 5_000; - protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds + protected const BATCH_AGGREGATION_INTERVAL = 300; // in seconds private int $lastTriggeredTime = 0; From c443c8fdab744b6c532196145ee046ac98612b1c Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 19:49:37 +0200 Subject: [PATCH 118/275] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index c663ffc6d4..3367bc1268 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -17,9 +17,9 @@ use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 26; // smaller batch size for development + protected const BATCH_SIZE_DEVELOPMENT = 6; // smaller batch size for development protected const BATCH_SIZE_PRODUCTION = 5_000; - protected const BATCH_AGGREGATION_INTERVAL = 300; // in seconds + protected const BATCH_AGGREGATION_INTERVAL = 120; // in seconds private int $lastTriggeredTime = 0; From f2ad3a501ed4a538f655592df4aa0b31e3105399 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 20:34:56 +0200 Subject: [PATCH 119/275] updating audits workers var scope --- src/Appwrite/Platform/Workers/Audits.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 3367bc1268..269b29c4c8 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -17,9 +17,9 @@ use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 6; // smaller batch size for development + protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development protected const BATCH_SIZE_PRODUCTION = 5_000; - protected const BATCH_AGGREGATION_INTERVAL = 120; // in seconds + protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds private int $lastTriggeredTime = 0; From 1d2883f47250b45837d87905c8ffcf0d4b480f5f Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 5 Mar 2025 04:46:06 +0000 Subject: [PATCH 120/275] chore: fix model for image transformations in usage project --- src/Appwrite/Utopia/Response/Model/UsageProject.php | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 6ac8830ac5..6737cbf6ef 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -197,18 +197,19 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('imageTransformations', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of aggregated number of image transformations.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ->addRule('imageTransformationsTotal', [ 'type' => self::TYPE_INTEGER, 'description' => 'An array of aggregated number of image transformations.', 'default' => 0, 'example' => 0, ]) - ->addRule('imageTransformations', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of image transformations.', - 'default' => 0, - 'example' => 0, - ]) ; } From 2c6f731a3820333a18fbcafbd43f478349f0a0bd Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 5 Mar 2025 04:49:34 +0000 Subject: [PATCH 121/275] fix: description --- src/Appwrite/Utopia/Response/Model/UsageProject.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 6737cbf6ef..395b19b7cd 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -206,7 +206,7 @@ class UsageProject extends Model ]) ->addRule('imageTransformationsTotal', [ 'type' => self::TYPE_INTEGER, - 'description' => 'An array of aggregated number of image transformations.', + 'description' => 'Total aggregated number of image transformations.', 'default' => 0, 'example' => 0, ]) From 6024d01e7ba7413ce462f09064b9a2abfd502a88 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 5 Mar 2025 05:25:50 +0000 Subject: [PATCH 122/275] Feat: calculate database storage in stats-resources - Calculate database storage in stats-resources worker --- .../Platform/Workers/StatsResources.php | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 62046bd186..969d43e895 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -182,7 +182,7 @@ class StatsResources extends Action } try { - $this->countForDatabase($dbForProject, $dbForLogs, $region); + $this->countForDatabase($dbForProject, $region); } catch (Throwable $th) { call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]); } @@ -242,42 +242,54 @@ class StatsResources extends Action $this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalImageTransformations); } - protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region) + protected function countForDatabase(Database $dbForProject, string $region) { $totalCollections = 0; $totalDocuments = 0; - $this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $dbForLogs, $region, &$totalCollections, &$totalDocuments) { + $totalDatabaseStorage = 0; + + $this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $region, &$totalCollections, &$totalDocuments, &$totalDatabaseStorage) { $collections = $dbForProject->count('database_' . $database->getInternalId()); $metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS); $this->createStatsDocuments($region, $metric, $collections); - $documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region); + [$documents, $storage] = $this->countForCollections($dbForProject, $database, $region); + $totalDatabaseStorage += $storage; $totalDocuments += $documents; $totalCollections += $collections; }); $this->createStatsDocuments($region, METRIC_COLLECTIONS, $totalCollections); $this->createStatsDocuments($region, METRIC_DOCUMENTS, $totalDocuments); + $this->createStatsDocuments($region, METRIC_DATABASES_STORAGE, $totalDatabaseStorage); } - protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int + protected function countForCollections(Database $dbForProject, Document $database, string $region): array { $databaseDocuments = 0; - $this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $dbForLogs, $database, $region, &$totalCollections, &$databaseDocuments) { + $databaseStorage = 0; + $this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $database, $region, &$databaseStorage, &$databaseDocuments) { $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - $metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS); $this->createStatsDocuments($region, $metric, $documents); - $databaseDocuments += $documents; + + $collectionStorage = $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); + $metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE); + $this->createStatsDocuments($region, $metric, $collectionStorage); + $databaseStorage += $collectionStorage; + }); $metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS); $this->createStatsDocuments($region, $metric, $databaseDocuments); - return $databaseDocuments; + $metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE); + $this->createStatsDocuments($region, $metric, $databaseStorage); + + return [$databaseDocuments, $databaseStorage]; } protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region) From 2afb8a14ac7f541eda336cd85c71e76e31e4b83d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 5 Mar 2025 05:27:56 +0000 Subject: [PATCH 123/275] Skip in dual writing --- src/Appwrite/Platform/Workers/StatsUsageDump.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 81acd4e4b0..119a9e7288 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -48,6 +48,7 @@ class StatsUsageDump extends Action METRIC_BUILDS => true, METRIC_COLLECTIONS => true, METRIC_DOCUMENTS => true, + METRIC_DATABASES_STORAGE => true, ]; /** @@ -63,6 +64,7 @@ class StatsUsageDump extends Action '.deployments.storage', '.builds', '.builds.storage', + '.databases.storage' ]; /** From 0ce4555f70312b0d9f1de93d91f96f2ed70d3192 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 5 Mar 2025 17:42:43 +0000 Subject: [PATCH 124/275] chore: added auth group to create phone token --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index a634618e6e..20f64496ac 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2400,7 +2400,7 @@ App::put('/v1/account/sessions/phone') App::post('/v1/account/tokens/phone') ->alias('/v1/account/sessions/phone') ->desc('Create phone token') - ->groups(['api', 'account']) + ->groups(['api', 'account', 'auth']) ->label('scope', 'sessions.write') ->label('auth.type', 'phone') ->label('audits.event', 'session.create') From 36b047529843dc23f3b94d2b23cb2fb6c2f0b36b Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 5 Mar 2025 18:40:41 +0000 Subject: [PATCH 125/275] chore: update tests --- .../Account/AccountCustomClientTest.php | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 439fa24fb6..daa5bcbff8 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -2419,6 +2419,33 @@ class AccountCustomClientTest extends Scope $message = $smsRequest['data']['message']; $token = substr($message, 0, 6); + /** + * Test for FAILURE + */ + + // disable phone sessions + $response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $this->getProject()['$id'] . '/auth/phone', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ]), [ + 'status' => false, + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(false, $response['body']['authPhone']); + + $response = $this->client->call(Client::METHOD_POST, '/account/verification/phone', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ])); + + $this->assertEquals(501, $response['headers']['status-code']); + $this->assertEquals("Phone authentication is disabled for this project", $response['body']['message']); + return \array_merge($data, [ 'token' => \substr($smsRequest['data']['message'], 0, 6) ]); From a0605b14cb46b541cc62d9ba4920189cdf9102fd Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 7 Mar 2025 04:14:45 +0000 Subject: [PATCH 126/275] chore: regen specs and update vers --- app/config/platforms.php | 6 +- .../specs/open-api3-latest-console.json | 22 ++- app/config/specs/open-api3-latest-server.json | 4 - app/config/specs/swagger2-latest-console.json | 23 ++- app/config/specs/swagger2-latest-server.json | 4 - composer.lock | 137 ++++++++---------- .../examples/account/update-mfa-challenge.md | 2 +- .../users/delete-mfa-authenticator.md | 2 +- 8 files changed, 89 insertions(+), 111 deletions(-) diff --git a/app/config/platforms.php b/app/config/platforms.php index 4b6a0d2da6..4fd4decc5b 100644 --- a/app/config/platforms.php +++ b/app/config/platforms.php @@ -59,7 +59,7 @@ return [ [ 'key' => 'flutter', 'name' => 'Flutter', - 'version' => '14.0.1', + 'version' => '15.0.0', 'url' => 'https://github.com/appwrite/sdk-for-flutter', 'package' => 'https://pub.dev/packages/appwrite', 'enabled' => true, @@ -217,7 +217,7 @@ return [ [ 'key' => 'cli', 'name' => 'Command Line', - 'version' => '6.2.1', + 'version' => '6.2.2', 'url' => 'https://github.com/appwrite/sdk-for-cli', 'package' => 'https://www.npmjs.com/package/appwrite-cli', 'enabled' => true, @@ -371,7 +371,7 @@ return [ [ 'key' => 'dart', 'name' => 'Dart', - 'version' => '13.0.1', + 'version' => '14.0.0', 'url' => 'https://github.com/appwrite/sdk-for-dart', 'package' => 'https://pub.dev/packages/dart_appwrite', 'enabled' => true, diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index bc216226fb..54161c4262 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -6409,8 +6409,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -6646,8 +6644,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -36630,13 +36626,15 @@ }, "x-example": [] }, - "imageTransformationsTotal": { - "type": "integer", - "description": "An array of aggregated number of image transformations.", - "x-example": 0, - "format": "int32" - }, "imageTransformations": { + "type": "array", + "description": "An array of aggregated number of image transformations.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] + }, + "imageTransformationsTotal": { "type": "integer", "description": "Total aggregated number of image transformations.", "x-example": 0, @@ -36673,8 +36671,8 @@ "authPhoneCountryBreakdown", "databasesReads", "databasesWrites", - "imageTransformationsTotal", - "imageTransformations" + "imageTransformations", + "imageTransformationsTotal" ] }, "headers": { diff --git a/app/config/specs/open-api3-latest-server.json b/app/config/specs/open-api3-latest-server.json index 280c080514..3d32d3e978 100644 --- a/app/config/specs/open-api3-latest-server.json +++ b/app/config/specs/open-api3-latest-server.json @@ -5953,8 +5953,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -6192,8 +6190,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index 2dddc72802..8fc7e7daf3 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -6609,8 +6609,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -6847,8 +6845,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -37217,13 +37213,16 @@ }, "x-example": [] }, - "imageTransformationsTotal": { - "type": "integer", - "description": "An array of aggregated number of image transformations.", - "x-example": 0, - "format": "int32" - }, "imageTransformations": { + "type": "array", + "description": "An array of aggregated number of image transformations.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] + }, + "imageTransformationsTotal": { "type": "integer", "description": "Total aggregated number of image transformations.", "x-example": 0, @@ -37260,8 +37259,8 @@ "authPhoneCountryBreakdown", "databasesReads", "databasesWrites", - "imageTransformationsTotal", - "imageTransformations" + "imageTransformations", + "imageTransformationsTotal" ] }, "headers": { diff --git a/app/config/specs/swagger2-latest-server.json b/app/config/specs/swagger2-latest-server.json index 749f87553b..83757c94f4 100644 --- a/app/config/specs/swagger2-latest-server.json +++ b/app/config/specs/swagger2-latest-server.json @@ -6135,8 +6135,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -6375,8 +6373,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } diff --git a/composer.lock b/composer.lock index e1d1ac16e3..8930e956a3 100644 --- a/composer.lock +++ b/composer.lock @@ -279,16 +279,16 @@ }, { "name": "brick/math", - "version": "0.12.2", + "version": "0.12.3", "source": { "type": "git", "url": "https://github.com/brick/math.git", - "reference": "901eddb1e45a8e0f689302e40af871c181ecbe40" + "reference": "866551da34e9a618e64a819ee1e01c20d8a588ba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/brick/math/zipball/901eddb1e45a8e0f689302e40af871c181ecbe40", - "reference": "901eddb1e45a8e0f689302e40af871c181ecbe40", + "url": "https://api.github.com/repos/brick/math/zipball/866551da34e9a618e64a819ee1e01c20d8a588ba", + "reference": "866551da34e9a618e64a819ee1e01c20d8a588ba", "shasum": "" }, "require": { @@ -327,7 +327,7 @@ ], "support": { "issues": "https://github.com/brick/math/issues", - "source": "https://github.com/brick/math/tree/0.12.2" + "source": "https://github.com/brick/math/tree/0.12.3" }, "funding": [ { @@ -335,7 +335,7 @@ "type": "github" } ], - "time": "2025-02-26T10:21:45+00:00" + "time": "2025-02-28T13:11:00+00:00" }, { "name": "chillerlan/php-qrcode", @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.29.3", + "version": "v4.30.0", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7" + "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7", - "reference": "ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/e1d66682f6836aa87820400f0aa07d9eb566feb6", + "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.3" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.0" }, - "time": "2025-01-08T21:00:13+00:00" + "time": "2025-03-04T22:54:49+00:00" }, { "name": "jean85/pretty-package-versions", @@ -1237,16 +1237,16 @@ }, { "name": "open-telemetry/api", - "version": "1.2.2", + "version": "1.2.3", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "8b925df3047628968bc5be722468db1b98b82d51" + "reference": "199d7ddda88f5f5619fa73463f1a5a7149ccd1f1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/8b925df3047628968bc5be722468db1b98b82d51", - "reference": "8b925df3047628968bc5be722468db1b98b82d51", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/199d7ddda88f5f5619fa73463f1a5a7149ccd1f1", + "reference": "199d7ddda88f5f5619fa73463f1a5a7149ccd1f1", "shasum": "" }, "require": { @@ -1303,7 +1303,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-02-03T21:49:11+00:00" + "time": "2025-03-05T21:42:54+00:00" }, { "name": "open-telemetry/context", @@ -2371,16 +2371,16 @@ }, { "name": "ramsey/collection", - "version": "2.0.0", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/ramsey/collection.git", - "reference": "a4b48764bfbb8f3a6a4d1aeb1a35bb5e9ecac4a5" + "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ramsey/collection/zipball/a4b48764bfbb8f3a6a4d1aeb1a35bb5e9ecac4a5", - "reference": "a4b48764bfbb8f3a6a4d1aeb1a35bb5e9ecac4a5", + "url": "https://api.github.com/repos/ramsey/collection/zipball/3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", + "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", "shasum": "" }, "require": { @@ -2388,25 +2388,22 @@ }, "require-dev": { "captainhook/plugin-composer": "^5.3", - "ergebnis/composer-normalize": "^2.28.3", - "fakerphp/faker": "^1.21", + "ergebnis/composer-normalize": "^2.45", + "fakerphp/faker": "^1.24", "hamcrest/hamcrest-php": "^2.0", - "jangregor/phpstan-prophecy": "^1.0", - "mockery/mockery": "^1.5", + "jangregor/phpstan-prophecy": "^2.1", + "mockery/mockery": "^1.6", "php-parallel-lint/php-console-highlighter": "^1.0", - "php-parallel-lint/php-parallel-lint": "^1.3", - "phpcsstandards/phpcsutils": "^1.0.0-rc1", - "phpspec/prophecy-phpunit": "^2.0", - "phpstan/extension-installer": "^1.2", - "phpstan/phpstan": "^1.9", - "phpstan/phpstan-mockery": "^1.1", - "phpstan/phpstan-phpunit": "^1.3", - "phpunit/phpunit": "^9.5", - "psalm/plugin-mockery": "^1.1", - "psalm/plugin-phpunit": "^0.18.4", - "ramsey/coding-standard": "^2.0.3", - "ramsey/conventional-commits": "^1.3", - "vimeo/psalm": "^5.4" + "php-parallel-lint/php-parallel-lint": "^1.4", + "phpspec/prophecy-phpunit": "^2.3", + "phpstan/extension-installer": "^1.4", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-mockery": "^2.0", + "phpstan/phpstan-phpunit": "^2.0", + "phpunit/phpunit": "^10.5", + "ramsey/coding-standard": "^2.3", + "ramsey/conventional-commits": "^1.6", + "roave/security-advisories": "dev-latest" }, "type": "library", "extra": { @@ -2444,19 +2441,9 @@ ], "support": { "issues": "https://github.com/ramsey/collection/issues", - "source": "https://github.com/ramsey/collection/tree/2.0.0" + "source": "https://github.com/ramsey/collection/tree/2.1.0" }, - "funding": [ - { - "url": "https://github.com/ramsey", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/ramsey/collection", - "type": "tidelift" - } - ], - "time": "2022-12-31T21:50:55+00:00" + "time": "2025-03-02T04:48:29+00:00" }, { "name": "ramsey/uuid", @@ -3880,16 +3867,16 @@ }, { "name": "utopia-php/fetch", - "version": "0.3.0", + "version": "0.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/fetch.git", - "reference": "02b12c05aec13399dcc2da8d51f908e328ab63f4" + "reference": "524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/fetch/zipball/02b12c05aec13399dcc2da8d51f908e328ab63f4", - "reference": "02b12c05aec13399dcc2da8d51f908e328ab63f4", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0", + "reference": "524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0", "shasum": "" }, "require": { @@ -3913,22 +3900,22 @@ "description": "A simple library that provides an interface for making HTTP Requests.", "support": { "issues": "https://github.com/utopia-php/fetch/issues", - "source": "https://github.com/utopia-php/fetch/tree/0.3.0" + "source": "https://github.com/utopia-php/fetch/tree/0.3.1" }, - "time": "2025-01-17T06:11:10+00:00" + "time": "2025-03-05T18:08:55+00:00" }, { "name": "utopia-php/framework", - "version": "0.33.17", + "version": "0.33.19", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644" + "reference": "64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/73fac6fbce9f56282dba4e52a58cf836ec434644", - "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644", + "url": "https://api.github.com/repos/utopia-php/http/zipball/64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0", + "reference": "64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0", "shasum": "" }, "require": { @@ -3960,9 +3947,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.17" + "source": "https://github.com/utopia-php/http/tree/0.33.19" }, - "time": "2025-02-24T17:35:48+00:00" + "time": "2025-03-06T11:37:49+00:00" }, { "name": "utopia-php/image", @@ -4608,22 +4595,24 @@ }, { "name": "utopia-php/storage", - "version": "0.18.9", + "version": "0.18.10", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c" + "reference": "76f31158f4251abb207f7a9b16f7cb0bfdb3b39e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/1cf455404e8700b3093fd73d74a38d41cdced90c", - "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/76f31158f4251abb207f7a9b16f7cb0bfdb3b39e", + "reference": "76f31158f4251abb207f7a9b16f7cb0bfdb3b39e", "shasum": "" }, "require": { "ext-brotli": "*", + "ext-curl": "*", "ext-fileinfo": "*", "ext-lz4": "*", + "ext-simplexml": "*", "ext-snappy": "*", "ext-xz": "*", "ext-zlib": "*", @@ -4657,9 +4646,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.9" + "source": "https://github.com/utopia-php/storage/tree/0.18.10" }, - "time": "2025-02-11T13:10:40+00:00" + "time": "2025-03-03T10:47:54+00:00" }, { "name": "utopia-php/swoole", @@ -5052,16 +5041,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.1", + "version": "0.40.2", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "df180676b6fbde7832ae1495af3e2f3e8f700837" + "reference": "56f09482d9e2f223911277ab887f197402708049" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/df180676b6fbde7832ae1495af3e2f3e8f700837", - "reference": "df180676b6fbde7832ae1495af3e2f3e8f700837", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/56f09482d9e2f223911277ab887f197402708049", + "reference": "56f09482d9e2f223911277ab887f197402708049", "shasum": "" }, "require": { @@ -5097,9 +5086,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.1" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.2" }, - "time": "2025-02-26T07:07:10+00:00" + "time": "2025-03-06T16:31:03+00:00" }, { "name": "doctrine/annotations", diff --git a/docs/examples/1.6.x/server-swift/examples/account/update-mfa-challenge.md b/docs/examples/1.6.x/server-swift/examples/account/update-mfa-challenge.md index eed3bfade7..fee76bf0dd 100644 --- a/docs/examples/1.6.x/server-swift/examples/account/update-mfa-challenge.md +++ b/docs/examples/1.6.x/server-swift/examples/account/update-mfa-challenge.md @@ -7,7 +7,7 @@ let client = Client() let account = Account(client) -let result = try await account.updateMfaChallenge( +let session = try await account.updateMfaChallenge( challengeId: "", otp: "" ) diff --git a/docs/examples/1.6.x/server-swift/examples/users/delete-mfa-authenticator.md b/docs/examples/1.6.x/server-swift/examples/users/delete-mfa-authenticator.md index 902d0c904c..5f1d6a0eeb 100644 --- a/docs/examples/1.6.x/server-swift/examples/users/delete-mfa-authenticator.md +++ b/docs/examples/1.6.x/server-swift/examples/users/delete-mfa-authenticator.md @@ -8,7 +8,7 @@ let client = Client() let users = Users(client) -let user = try await users.deleteMfaAuthenticator( +let result = try await users.deleteMfaAuthenticator( userId: "", type: .totp ) From b791e596721071413eff91ef0a9ec438c2ebdb68 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 7 Mar 2025 10:59:16 +0000 Subject: [PATCH 127/275] chore: bump composer --- composer.lock | 159 +++++++++++++++++++++++--------------------------- 1 file changed, 74 insertions(+), 85 deletions(-) diff --git a/composer.lock b/composer.lock index 76f61e44d7..e89d6b53b5 100644 --- a/composer.lock +++ b/composer.lock @@ -279,16 +279,16 @@ }, { "name": "brick/math", - "version": "0.12.2", + "version": "0.12.3", "source": { "type": "git", "url": "https://github.com/brick/math.git", - "reference": "901eddb1e45a8e0f689302e40af871c181ecbe40" + "reference": "866551da34e9a618e64a819ee1e01c20d8a588ba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/brick/math/zipball/901eddb1e45a8e0f689302e40af871c181ecbe40", - "reference": "901eddb1e45a8e0f689302e40af871c181ecbe40", + "url": "https://api.github.com/repos/brick/math/zipball/866551da34e9a618e64a819ee1e01c20d8a588ba", + "reference": "866551da34e9a618e64a819ee1e01c20d8a588ba", "shasum": "" }, "require": { @@ -327,7 +327,7 @@ ], "support": { "issues": "https://github.com/brick/math/issues", - "source": "https://github.com/brick/math/tree/0.12.2" + "source": "https://github.com/brick/math/tree/0.12.3" }, "funding": [ { @@ -335,7 +335,7 @@ "type": "github" } ], - "time": "2025-02-26T10:21:45+00:00" + "time": "2025-02-28T13:11:00+00:00" }, { "name": "chillerlan/php-qrcode", @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.29.3", + "version": "v4.30.0", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7" + "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7", - "reference": "ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/e1d66682f6836aa87820400f0aa07d9eb566feb6", + "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.3" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.0" }, - "time": "2025-01-08T21:00:13+00:00" + "time": "2025-03-04T22:54:49+00:00" }, { "name": "jean85/pretty-package-versions", @@ -1237,16 +1237,16 @@ }, { "name": "open-telemetry/api", - "version": "1.2.2", + "version": "1.2.3", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "8b925df3047628968bc5be722468db1b98b82d51" + "reference": "199d7ddda88f5f5619fa73463f1a5a7149ccd1f1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/8b925df3047628968bc5be722468db1b98b82d51", - "reference": "8b925df3047628968bc5be722468db1b98b82d51", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/199d7ddda88f5f5619fa73463f1a5a7149ccd1f1", + "reference": "199d7ddda88f5f5619fa73463f1a5a7149ccd1f1", "shasum": "" }, "require": { @@ -1303,7 +1303,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-02-03T21:49:11+00:00" + "time": "2025-03-05T21:42:54+00:00" }, { "name": "open-telemetry/context", @@ -1366,16 +1366,16 @@ }, { "name": "open-telemetry/exporter-otlp", - "version": "1.2.0", + "version": "1.2.1", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/exporter-otlp.git", - "reference": "243d9657c44a06f740cf384f486afe954c2b725f" + "reference": "b7580440b7481a98da97aceabeb46e1b276c8747" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/243d9657c44a06f740cf384f486afe954c2b725f", - "reference": "243d9657c44a06f740cf384f486afe954c2b725f", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/b7580440b7481a98da97aceabeb46e1b276c8747", + "reference": "b7580440b7481a98da97aceabeb46e1b276c8747", "shasum": "" }, "require": { @@ -1426,7 +1426,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-08T23:50:03+00:00" + "time": "2025-03-06T23:21:56+00:00" }, { "name": "open-telemetry/gen-otlp-protobuf", @@ -2371,16 +2371,16 @@ }, { "name": "ramsey/collection", - "version": "2.0.0", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/ramsey/collection.git", - "reference": "a4b48764bfbb8f3a6a4d1aeb1a35bb5e9ecac4a5" + "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ramsey/collection/zipball/a4b48764bfbb8f3a6a4d1aeb1a35bb5e9ecac4a5", - "reference": "a4b48764bfbb8f3a6a4d1aeb1a35bb5e9ecac4a5", + "url": "https://api.github.com/repos/ramsey/collection/zipball/3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", + "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", "shasum": "" }, "require": { @@ -2388,25 +2388,22 @@ }, "require-dev": { "captainhook/plugin-composer": "^5.3", - "ergebnis/composer-normalize": "^2.28.3", - "fakerphp/faker": "^1.21", + "ergebnis/composer-normalize": "^2.45", + "fakerphp/faker": "^1.24", "hamcrest/hamcrest-php": "^2.0", - "jangregor/phpstan-prophecy": "^1.0", - "mockery/mockery": "^1.5", + "jangregor/phpstan-prophecy": "^2.1", + "mockery/mockery": "^1.6", "php-parallel-lint/php-console-highlighter": "^1.0", - "php-parallel-lint/php-parallel-lint": "^1.3", - "phpcsstandards/phpcsutils": "^1.0.0-rc1", - "phpspec/prophecy-phpunit": "^2.0", - "phpstan/extension-installer": "^1.2", - "phpstan/phpstan": "^1.9", - "phpstan/phpstan-mockery": "^1.1", - "phpstan/phpstan-phpunit": "^1.3", - "phpunit/phpunit": "^9.5", - "psalm/plugin-mockery": "^1.1", - "psalm/plugin-phpunit": "^0.18.4", - "ramsey/coding-standard": "^2.0.3", - "ramsey/conventional-commits": "^1.3", - "vimeo/psalm": "^5.4" + "php-parallel-lint/php-parallel-lint": "^1.4", + "phpspec/prophecy-phpunit": "^2.3", + "phpstan/extension-installer": "^1.4", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-mockery": "^2.0", + "phpstan/phpstan-phpunit": "^2.0", + "phpunit/phpunit": "^10.5", + "ramsey/coding-standard": "^2.3", + "ramsey/conventional-commits": "^1.6", + "roave/security-advisories": "dev-latest" }, "type": "library", "extra": { @@ -2444,19 +2441,9 @@ ], "support": { "issues": "https://github.com/ramsey/collection/issues", - "source": "https://github.com/ramsey/collection/tree/2.0.0" + "source": "https://github.com/ramsey/collection/tree/2.1.0" }, - "funding": [ - { - "url": "https://github.com/ramsey", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/ramsey/collection", - "type": "tidelift" - } - ], - "time": "2022-12-31T21:50:55+00:00" + "time": "2025-03-02T04:48:29+00:00" }, { "name": "ramsey/uuid", @@ -3718,16 +3705,16 @@ }, { "name": "utopia-php/database", - "version": "0.60.4", + "version": "0.60.6", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "3a034f8eb275cd92d088f1e492a7cd00fd021427" + "reference": "f3c9aa964b39c6205069f038a26e709a15541406" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/3a034f8eb275cd92d088f1e492a7cd00fd021427", - "reference": "3a034f8eb275cd92d088f1e492a7cd00fd021427", + "url": "https://api.github.com/repos/utopia-php/database/zipball/f3c9aa964b39c6205069f038a26e709a15541406", + "reference": "f3c9aa964b39c6205069f038a26e709a15541406", "shasum": "" }, "require": { @@ -3768,9 +3755,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.60.4" + "source": "https://github.com/utopia-php/database/tree/0.60.6" }, - "time": "2025-02-25T23:09:18+00:00" + "time": "2025-03-05T01:23:14+00:00" }, { "name": "utopia-php/domains", @@ -3881,16 +3868,16 @@ }, { "name": "utopia-php/fetch", - "version": "0.3.0", + "version": "0.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/fetch.git", - "reference": "02b12c05aec13399dcc2da8d51f908e328ab63f4" + "reference": "524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/fetch/zipball/02b12c05aec13399dcc2da8d51f908e328ab63f4", - "reference": "02b12c05aec13399dcc2da8d51f908e328ab63f4", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0", + "reference": "524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0", "shasum": "" }, "require": { @@ -3914,22 +3901,22 @@ "description": "A simple library that provides an interface for making HTTP Requests.", "support": { "issues": "https://github.com/utopia-php/fetch/issues", - "source": "https://github.com/utopia-php/fetch/tree/0.3.0" + "source": "https://github.com/utopia-php/fetch/tree/0.3.1" }, - "time": "2025-01-17T06:11:10+00:00" + "time": "2025-03-05T18:08:55+00:00" }, { "name": "utopia-php/framework", - "version": "0.33.17", + "version": "0.33.19", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644" + "reference": "64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/73fac6fbce9f56282dba4e52a58cf836ec434644", - "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644", + "url": "https://api.github.com/repos/utopia-php/http/zipball/64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0", + "reference": "64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0", "shasum": "" }, "require": { @@ -3961,9 +3948,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.17" + "source": "https://github.com/utopia-php/http/tree/0.33.19" }, - "time": "2025-02-24T17:35:48+00:00" + "time": "2025-03-06T11:37:49+00:00" }, { "name": "utopia-php/image", @@ -4609,22 +4596,24 @@ }, { "name": "utopia-php/storage", - "version": "0.18.9", + "version": "0.18.10", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c" + "reference": "76f31158f4251abb207f7a9b16f7cb0bfdb3b39e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/1cf455404e8700b3093fd73d74a38d41cdced90c", - "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/76f31158f4251abb207f7a9b16f7cb0bfdb3b39e", + "reference": "76f31158f4251abb207f7a9b16f7cb0bfdb3b39e", "shasum": "" }, "require": { "ext-brotli": "*", + "ext-curl": "*", "ext-fileinfo": "*", "ext-lz4": "*", + "ext-simplexml": "*", "ext-snappy": "*", "ext-xz": "*", "ext-zlib": "*", @@ -4658,9 +4647,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.9" + "source": "https://github.com/utopia-php/storage/tree/0.18.10" }, - "time": "2025-02-11T13:10:40+00:00" + "time": "2025-03-03T10:47:54+00:00" }, { "name": "utopia-php/swoole", @@ -5054,16 +5043,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.1", + "version": "0.40.2", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "df180676b6fbde7832ae1495af3e2f3e8f700837" + "reference": "56f09482d9e2f223911277ab887f197402708049" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/df180676b6fbde7832ae1495af3e2f3e8f700837", - "reference": "df180676b6fbde7832ae1495af3e2f3e8f700837", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/56f09482d9e2f223911277ab887f197402708049", + "reference": "56f09482d9e2f223911277ab887f197402708049", "shasum": "" }, "require": { @@ -5099,9 +5088,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.1" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.2" }, - "time": "2025-02-26T07:07:10+00:00" + "time": "2025-03-06T16:31:03+00:00" }, { "name": "doctrine/annotations", From d9fc6b70ad3bbd501e56458befc7b0f2d4fbaaa6 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 7 Mar 2025 13:18:20 +0000 Subject: [PATCH 128/275] chore: bump react native --- app/config/platforms.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/platforms.php b/app/config/platforms.php index 4fd4decc5b..477d2a47b2 100644 --- a/app/config/platforms.php +++ b/app/config/platforms.php @@ -134,7 +134,7 @@ return [ [ 'key' => 'react-native', 'name' => 'React Native', - 'version' => '0.6.0', + 'version' => '0.7.1', 'url' => 'https://github.com/appwrite/sdk-for-react-native', 'package' => 'https://npmjs.com/package/react-native-appwrite', 'enabled' => true, From 493f536a6e228a45eb7f82989d767385e727bebb Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 7 Mar 2025 14:04:52 +0000 Subject: [PATCH 129/275] chore: added logsdb for deletes worker --- src/Appwrite/Platform/Tasks/Maintenance.php | 1 - src/Appwrite/Platform/Workers/Deletes.php | 30 +++++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 2d37bdbf70..53aa4f1273 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -52,7 +52,6 @@ class Maintenance extends Action ->setProject($project) ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) ->trigger(); - }); $this->notifyDeleteConnections($queueForDeletes); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index e11181d199..643fc4d010 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -47,7 +47,7 @@ class Deletes extends Action ->inject('project') ->inject('dbForPlatform') ->inject('getProjectDB') - ->inject('timelimit') + ->inject('getLogsDB') ->inject('deviceForFiles') ->inject('deviceForFunctions') ->inject('deviceForBuilds') @@ -57,8 +57,8 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $project, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -131,7 +131,7 @@ class Deletes extends Action $this->deleteExpiredSessions($project, $getProjectDB); break; case DELETE_TYPE_USAGE: - $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime); + $this->deleteUsageStats($project, $getProjectDB, $getLogsDB, $hourlyUsageRetentionDatetime); break; case DELETE_TYPE_CACHE_BY_RESOURCE: $this->deleteCacheByResource($project, $getProjectDB, $resource, $resourceType); @@ -158,7 +158,7 @@ class Deletes extends Action $this->deleteExpiredTargets($project, $getProjectDB); $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); $this->deleteAuditLogs($project, $getProjectDB, $auditRetention); - $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime); + $this->deleteUsageStats($project, $getProjectDB, $getLogsDB, $hourlyUsageRetentionDatetime); $this->deleteExpiredSessions($project, $getProjectDB); break; default: @@ -412,14 +412,22 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteUsageStats(Document $project, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void + private function deleteUsageStats(Document $project, callable $getProjectDB, callable $getLogsDB, string $hourlyUsageRetentionDatetime): void { $dbForProject = $getProjectDB($project); - // Delete Usage stats + $dbForLogs = $getLogsDB($project); + + // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::equal('period', ['1h']), ], $dbForProject); + + // Delete Usage stats from logsDB + $this->deleteByGroup('stats', [ + Query::lessThan('time', $hourlyUsageRetentionDatetime), + Query::equal('period', ['1h']), + ], $dbForLogs); } /** @@ -851,7 +859,7 @@ class Deletes extends Action } else { Console::error('Failed to delete deployment files: ' . $deploymentPath); } - } catch (\Throwable $th) { + } catch (Throwable $th) { Console::error('Failed to delete deployment files: ' . $deploymentPath); Console::error('[Error] Type: ' . get_class($th)); Console::error('[Error] Message: ' . $th->getMessage()); @@ -881,7 +889,7 @@ class Deletes extends Action } else { Console::error('Failed to delete build files: ' . $buildPath); } - } catch (\Throwable $th) { + } catch (Throwable $th) { Console::error('Failed to delete deployment files: ' . $buildPath); Console::error('[Error] Type: ' . get_class($th)); Console::error('[Error] Message: ' . $th->getMessage()); @@ -947,7 +955,7 @@ class Deletes extends Action try { $documents = $database->deleteDocuments($collection, $queries); - } catch (\Throwable $th) { + } catch (Throwable $th) { Console::error('Failed to delete documents for collection ' . $collection . ': ' . $th->getMessage()); return; } From f6b5deda6da2e93c68693547994b381d5fcce8d6 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Fri, 7 Mar 2025 12:35:18 -0800 Subject: [PATCH 130/275] fix: add missing _APP_EMAIL_CERTIFICATES env var to deletes worker The deletes worker uses the certificates resource to delete certificates, but the certificates resource requires the _APP_EMAIL_CERTIFICATES env var to be set or it will throw an exception and not process any delete jobs. This commit adds the missing env var to the deletes worker. Also add _APP_SYSTEM_SECURITY_EMAIL_ADDRESS as a fallback. --- app/views/install/compose.phtml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 62fcd03624..e50c70de5b 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -342,6 +342,8 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_EXECUTION + - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS + - _APP_EMAIL_CERTIFICATES appwrite-worker-databases: image: /: From 0fbd2c642518cbdf63a11b320fc8ec90016df80e Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Fri, 7 Mar 2025 12:54:28 -0800 Subject: [PATCH 131/275] fix: prevent warning if no logging config is set If no logging config is set, there's a warning saying: > Using deprecated logging configuration. However, if they didn't set any config, it's not deprecated. --- app/init.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/init.php b/app/init.php index 1311a1429f..1d7f734723 100644 --- a/app/init.php +++ b/app/init.php @@ -817,6 +817,10 @@ $register->set('logger', function () { $providerName = System::getEnv('_APP_LOGGING_PROVIDER', ''); $providerConfig = System::getEnv('_APP_LOGGING_CONFIG', ''); + if (empty($providerConfig)) { + return; + } + try { $loggingProvider = new DSN($providerConfig ?? ''); From 59a8172e96d7956b376d993950eec06228d85100 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Fri, 7 Mar 2025 12:59:11 -0800 Subject: [PATCH 132/275] fix: prevent "Failed to initialize logging provider" errors If the _APP_EXPERIMENT_LOGGING_CONFIG env var isn't set, every 4XX error outputs this warning: Failed to initialize logging provider: Unable to parse DSN: scheme is required To reduce this unnecessary clutter when _APP_EXPERIMENT_LOGGING_CONFIG isn't set, this PR adds a check to only attempt to initialize the logging provider if the env var is set. --- app/controllers/general.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 65979d3475..d93766a5e7 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -859,11 +859,9 @@ App::error() $publish = $error->getCode() === 0 || $error->getCode() >= 500; } - if ($error->getCode() >= 400 && $error->getCode() < 500) { + $providerConfig = System::getEnv('_APP_EXPERIMENT_LOGGING_CONFIG', ''); + if (!empty($providerConfig) && $error->getCode() >= 400 && $error->getCode() < 500) { // Register error logger - $providerName = System::getEnv('_APP_EXPERIMENT_LOGGING_PROVIDER', ''); - $providerConfig = System::getEnv('_APP_EXPERIMENT_LOGGING_CONFIG', ''); - try { $loggingProvider = new DSN($providerConfig ?? ''); $providerName = $loggingProvider->getScheme(); From 6a6d99a9d0f87dabac610e7027b341ad0c9ac719 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 10 Mar 2025 08:54:20 +0000 Subject: [PATCH 133/275] chore: queue console project for maintenance delete --- app/cli.php | 45 +++++++++++++++++++++ src/Appwrite/Platform/Tasks/Maintenance.php | 14 +++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/app/cli.php b/app/cli.php index 0b2cb884e6..dbbdfdee4d 100644 --- a/app/cli.php +++ b/app/cli.php @@ -2,11 +2,13 @@ require_once __DIR__ . '/init.php'; +use Appwrite\Auth\Auth; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; use Appwrite\Event\Func; use Appwrite\Event\StatsResources; use Appwrite\Event\StatsUsage; +use Appwrite\Network\Validator\Origin; use Appwrite\Platform\Appwrite; use Appwrite\Runtimes\Runtimes; use Utopia\Cache\Adapter\Sharding; @@ -16,6 +18,7 @@ use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Helpers\ID; use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; use Utopia\Logger\Log; @@ -99,6 +102,48 @@ CLI::setResource('dbForPlatform', function ($pools, $cache) { return $dbForPlatform; }, ['pools', 'cache']); +CLI::setResource('console', function () { + return new Document([ + '$id' => ID::custom('console'), + '$internalId' => ID::custom('console'), + 'name' => 'Appwrite', + '$collection' => ID::custom('projects'), + 'description' => 'Appwrite core engine', + 'logo' => '', + 'teamId' => null, + 'webhooks' => [], + 'keys' => [], + 'platforms' => [ + [ + '$collection' => ID::custom('platforms'), + 'name' => 'Localhost', + 'type' => Origin::CLIENT_TYPE_WEB, + 'hostname' => 'localhost', + ], // Current host is added on app init + ], + 'legalName' => '', + 'legalCountry' => '', + 'legalState' => '', + 'legalCity' => '', + 'legalAddress' => '', + 'legalTaxId' => '', + 'auths' => [ + 'mockNumbers' => [], + 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', + 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user + 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds + 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' + ], + 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], + 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], + 'oAuthProviders' => [ + 'githubEnabled' => true, + 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), + 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') + ], + ]); +}, []); + CLI::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 2d37bdbf70..98a3f4d295 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -24,12 +24,13 @@ class Maintenance extends Action $this ->desc('Schedules maintenance tasks and publishes them to our queues') ->inject('dbForPlatform') + ->inject('console') ->inject('queueForCertificates') ->inject('queueForDeletes') - ->callback(fn (Database $dbForPlatform, Certificate $queueForCertificates, Delete $queueForDeletes) => $this->action($dbForPlatform, $queueForCertificates, $queueForDeletes)); + ->callback(fn (Database $dbForPlatform, Document $console, Certificate $queueForCertificates, Delete $queueForDeletes) => $this->action($dbForPlatform, $console, $queueForCertificates, $queueForDeletes)); } - public function action(Database $dbForPlatform, Certificate $queueForCertificates, Delete $queueForDeletes): void + public function action(Database $dbForPlatform, Document $console, Certificate $queueForCertificates, Delete $queueForDeletes): void { Console::title('Maintenance V1'); Console::success(APP_NAME . ' maintenance process v1 has started'); @@ -41,7 +42,7 @@ class Maintenance extends Action $cacheRetention = (int) System::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days $schedulesDeletionRetention = (int) System::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day - Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForPlatform, $queueForDeletes, $queueForCertificates) { + Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForPlatform, $console, $queueForDeletes, $queueForCertificates) { $time = DateTime::now(); Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); @@ -52,9 +53,14 @@ class Maintenance extends Action ->setProject($project) ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) ->trigger(); - }); + $queueForDeletes + ->setType(DELETE_TYPE_MAINTENANCE) + ->setProject($console) + ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) + ->trigger(); + $this->notifyDeleteConnections($queueForDeletes); $this->renewCertificates($dbForPlatform, $queueForCertificates); $this->notifyDeleteCache($cacheRetention, $queueForDeletes); From 61377ceb6e518d664f050337eafbde64e507e749 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 10 Mar 2025 10:48:44 +0000 Subject: [PATCH 134/275] chore: shift initialization of console project to config --- app/cli.php | 43 +--------------------------------- app/config/console.php | 52 ++++++++++++++++++++++++++++++++++++++++++ app/init.php | 41 ++------------------------------- 3 files changed, 55 insertions(+), 81 deletions(-) create mode 100644 app/config/console.php diff --git a/app/cli.php b/app/cli.php index dbbdfdee4d..ce978c6d9d 100644 --- a/app/cli.php +++ b/app/cli.php @@ -2,13 +2,11 @@ require_once __DIR__ . '/init.php'; -use Appwrite\Auth\Auth; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; use Appwrite\Event\Func; use Appwrite\Event\StatsResources; use Appwrite\Event\StatsUsage; -use Appwrite\Network\Validator\Origin; use Appwrite\Platform\Appwrite; use Appwrite\Runtimes\Runtimes; use Utopia\Cache\Adapter\Sharding; @@ -18,7 +16,6 @@ use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Helpers\ID; use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; use Utopia\Logger\Log; @@ -103,45 +100,7 @@ CLI::setResource('dbForPlatform', function ($pools, $cache) { }, ['pools', 'cache']); CLI::setResource('console', function () { - return new Document([ - '$id' => ID::custom('console'), - '$internalId' => ID::custom('console'), - 'name' => 'Appwrite', - '$collection' => ID::custom('projects'), - 'description' => 'Appwrite core engine', - 'logo' => '', - 'teamId' => null, - 'webhooks' => [], - 'keys' => [], - 'platforms' => [ - [ - '$collection' => ID::custom('platforms'), - 'name' => 'Localhost', - 'type' => Origin::CLIENT_TYPE_WEB, - 'hostname' => 'localhost', - ], // Current host is added on app init - ], - 'legalName' => '', - 'legalCountry' => '', - 'legalState' => '', - 'legalCity' => '', - 'legalAddress' => '', - 'legalTaxId' => '', - 'auths' => [ - 'mockNumbers' => [], - 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', - 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user - 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds - 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' - ], - 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], - 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], - 'oAuthProviders' => [ - 'githubEnabled' => true, - 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), - 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') - ], - ]); + return new Document(Config::getParam('console')); }, []); CLI::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { diff --git a/app/config/console.php b/app/config/console.php new file mode 100644 index 0000000000..5c15a7930f --- /dev/null +++ b/app/config/console.php @@ -0,0 +1,52 @@ + ID::custom('console'), + '$internalId' => ID::custom('console'), + 'name' => 'Appwrite', + '$collection' => ID::custom('projects'), + 'description' => 'Appwrite core engine', + 'logo' => '', + 'teamId' => null, + 'webhooks' => [], + 'keys' => [], + 'platforms' => [ + [ + '$collection' => ID::custom('platforms'), + 'name' => 'Localhost', + 'type' => Origin::CLIENT_TYPE_WEB, + 'hostname' => 'localhost', + ], // Current host is added on app init + ], + 'legalName' => '', + 'legalCountry' => '', + 'legalState' => '', + 'legalCity' => '', + 'legalAddress' => '', + 'legalTaxId' => '', + 'auths' => [ + 'mockNumbers' => [], + 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', + 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user + 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds + 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' + ], + 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], + 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], + 'oAuthProviders' => [ + 'githubEnabled' => true, + 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), + 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') + ], +]; + +return $console; diff --git a/app/init.php b/app/init.php index 1d7f734723..17c1a884f9 100644 --- a/app/init.php +++ b/app/init.php @@ -347,6 +347,7 @@ Config::load('apis', __DIR__ . '/config/apis.php'); // List of APIs Config::load('errors', __DIR__ . '/config/errors.php'); Config::load('oAuthProviders', __DIR__ . '/config/oAuthProviders.php'); Config::load('platforms', __DIR__ . '/config/platforms.php'); +Config::load('console', __DIR__ . '/config/console.php'); Config::load('collections', __DIR__ . '/config/collections.php'); Config::load('runtimes', __DIR__ . '/config/runtimes.php'); Config::load('runtimes-v2', __DIR__ . '/config/runtimes-v2.php'); @@ -1408,45 +1409,7 @@ App::setResource('session', function (Document $user) { }, ['user']); App::setResource('console', function () { - return new Document([ - '$id' => ID::custom('console'), - '$internalId' => ID::custom('console'), - 'name' => 'Appwrite', - '$collection' => ID::custom('projects'), - 'description' => 'Appwrite core engine', - 'logo' => '', - 'teamId' => null, - 'webhooks' => [], - 'keys' => [], - 'platforms' => [ - [ - '$collection' => ID::custom('platforms'), - 'name' => 'Localhost', - 'type' => Origin::CLIENT_TYPE_WEB, - 'hostname' => 'localhost', - ], // Current host is added on app init - ], - 'legalName' => '', - 'legalCountry' => '', - 'legalState' => '', - 'legalCity' => '', - 'legalAddress' => '', - 'legalTaxId' => '', - 'auths' => [ - 'mockNumbers' => [], - 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', - 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user - 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds - 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' - ], - 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], - 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], - 'oAuthProviders' => [ - 'githubEnabled' => true, - 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), - 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') - ], - ]); + return new Document(Config::getParam('console')); }, []); App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) { From e3f59c696d7588140134d1cdfcedfe471a5c24cc Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 11 Mar 2025 05:43:30 +0000 Subject: [PATCH 135/275] bump: python version --- app/config/platforms.php | 2 +- composer.lock | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/app/config/platforms.php b/app/config/platforms.php index 477d2a47b2..86f0a756c9 100644 --- a/app/config/platforms.php +++ b/app/config/platforms.php @@ -299,7 +299,7 @@ return [ [ 'key' => 'python', 'name' => 'Python', - 'version' => '6.2.0', + 'version' => '9.0.2', 'url' => 'https://github.com/appwrite/sdk-for-python', 'package' => 'https://pypi.org/project/appwrite/', 'enabled' => true, diff --git a/composer.lock b/composer.lock index e89d6b53b5..f033ea4ec0 100644 --- a/composer.lock +++ b/composer.lock @@ -5043,16 +5043,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.2", + "version": "0.40.6", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "56f09482d9e2f223911277ab887f197402708049" + "reference": "d8816209a07e7d64ef62dbcaf8ad4aa1262f58b9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/56f09482d9e2f223911277ab887f197402708049", - "reference": "56f09482d9e2f223911277ab887f197402708049", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/d8816209a07e7d64ef62dbcaf8ad4aa1262f58b9", + "reference": "d8816209a07e7d64ef62dbcaf8ad4aa1262f58b9", "shasum": "" }, "require": { @@ -5088,9 +5088,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.2" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.6" }, - "time": "2025-03-06T16:31:03+00:00" + "time": "2025-03-10T19:04:24+00:00" }, { "name": "doctrine/annotations", @@ -5317,16 +5317,16 @@ }, { "name": "laravel/pint", - "version": "v1.21.0", + "version": "v1.21.1", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "531fa0871fbde719c51b12afa3a443b8f4e4b425" + "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/531fa0871fbde719c51b12afa3a443b8f4e4b425", - "reference": "531fa0871fbde719c51b12afa3a443b8f4e4b425", + "url": "https://api.github.com/repos/laravel/pint/zipball/c44bffbb2334e90fba560933c45948fa4a3f3e86", + "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86", "shasum": "" }, "require": { @@ -5337,9 +5337,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.68.5", - "illuminate/view": "^11.42.0", - "larastan/larastan": "^3.0.4", + "friendsofphp/php-cs-fixer": "^3.70.2", + "illuminate/view": "^11.44.1", + "larastan/larastan": "^3.1.0", "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3", @@ -5379,7 +5379,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-02-18T03:18:57+00:00" + "time": "2025-03-11T03:22:21+00:00" }, { "name": "matthiasmullie/minify", From b387823d63a04b97bd69d91d7152a37ac60e3827 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 11 Mar 2025 09:17:28 +0200 Subject: [PATCH 136/275] Update audit timestamp origin --- src/Appwrite/Platform/Workers/Audits.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 269b29c4c8..4c052cbcaa 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -102,7 +102,7 @@ class Audits extends Action 'mode' => $mode, 'data' => $auditPayload, ], - 'timestamp' => DateTime::formatTz(DateTime::now()) + 'timestamp' => date("Y-m-d H:i:s", $message->getTimestamp()), ]; if (isset($this->logs[$project->getInternalId()])) { From 141f0fc376abd23fda870d15c3f1a28913fb3cda Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 11 Mar 2025 10:27:23 +0200 Subject: [PATCH 137/275] linter --- src/Appwrite/Platform/Workers/Audits.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 4c052cbcaa..ed5ff8010a 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -7,7 +7,6 @@ use Exception; use Throwable; use Utopia\Audit\Audit; use Utopia\CLI\Console; -use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; From 2052fb975bf0cb6c3be90aa98af258af0642f38e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Mar 2025 16:53:08 +1300 Subject: [PATCH 138/275] Use bulk deletes for audit cleanup --- src/Appwrite/Platform/Workers/Deletes.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 643fc4d010..06838e0232 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -732,10 +732,12 @@ class Deletes extends Action { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); - $audit = new Audit($dbForProject); try { - $audit->cleanup($auditRetention); + $this->deleteByGroup(Audit::COLLECTION, [ + Query::lessThan('time', $auditRetention), + Query::orderDesc('time'), + ], $dbForProject); } catch (DatabaseException $e) { Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); } @@ -943,7 +945,7 @@ class Deletes extends Action * @param Database $database * @param ?callable $callback * @return void - * @throws Exception + * @throws DatabaseException */ protected function deleteByGroup( string $collection, From d77940d22f1d0a537591d6a5ad37d8b60cdd7367 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 12 Mar 2025 10:00:07 +0000 Subject: [PATCH 139/275] Feat: calculate and log time taken for each project --- src/Appwrite/Platform/Workers/StatsResources.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 969d43e895..420aeb5f77 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -77,7 +77,21 @@ class StatsResources extends Action // Reset documents for each job $this->documents = []; + $startTime = microtime(true); $this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project); + $endTime = microtime(true); + $executionTime = $endTime - $startTime; + Console::info('Project: ' . $project->getId() . '(' . $project->getInternalId() . ') aggregated in ' . $this->formatTime($executionTime) .''); + } + + public function formatTime($microseconds) + { + $seconds = $microseconds / 1000000; // Convert microseconds to seconds + $hours = floor($seconds / 3600); + $minutes = floor(($seconds % 3600) / 60); + $remainingSeconds = $seconds % 60; + + return sprintf('%02d:%02d:%06.3f', $hours, $minutes, $remainingSeconds); } From 5c25b583519b868ac994f2866dfff9a885a3113a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 12 Mar 2025 10:19:55 +0000 Subject: [PATCH 140/275] Fix date format --- src/Appwrite/Platform/Workers/StatsResources.php | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 420aeb5f77..ecfe80bf2d 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -81,20 +81,9 @@ class StatsResources extends Action $this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project); $endTime = microtime(true); $executionTime = $endTime - $startTime; - Console::info('Project: ' . $project->getId() . '(' . $project->getInternalId() . ') aggregated in ' . $this->formatTime($executionTime) .''); + Console::info('Project: ' . $project->getId() . '(' . $project->getInternalId() . ') aggregated in ' . date('H:i:s.u', intval($executionTime)) .''); } - public function formatTime($microseconds) - { - $seconds = $microseconds / 1000000; // Convert microseconds to seconds - $hours = floor($seconds / 3600); - $minutes = floor(($seconds % 3600) / 60); - $remainingSeconds = $seconds % 60; - - return sprintf('%02d:%02d:%06.3f', $hours, $minutes, $remainingSeconds); - } - - protected function countForProject(Database $dbForPlatform, callable $getLogsDB, callable $getProjectDB, Document $project): void { Console::info('Begining count for: ' . $project->getId()); From 74f4ebf90ef32ac84242b77ec434221b1ad35d2a Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 12 Mar 2025 10:26:01 +0000 Subject: [PATCH 141/275] chore: update sdk generator --- composer.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/composer.lock b/composer.lock index f033ea4ec0..83c8ea479a 100644 --- a/composer.lock +++ b/composer.lock @@ -5043,16 +5043,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.6", + "version": "0.40.7", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "d8816209a07e7d64ef62dbcaf8ad4aa1262f58b9" + "reference": "9e89b0bc4d8e6c81817d27096629f34a149fa873" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/d8816209a07e7d64ef62dbcaf8ad4aa1262f58b9", - "reference": "d8816209a07e7d64ef62dbcaf8ad4aa1262f58b9", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/9e89b0bc4d8e6c81817d27096629f34a149fa873", + "reference": "9e89b0bc4d8e6c81817d27096629f34a149fa873", "shasum": "" }, "require": { @@ -5088,9 +5088,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.6" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.7" }, - "time": "2025-03-10T19:04:24+00:00" + "time": "2025-03-12T08:43:55+00:00" }, { "name": "doctrine/annotations", @@ -5794,16 +5794,16 @@ }, { "name": "phpbench/phpbench", - "version": "1.4.0", + "version": "1.4.1", "source": { "type": "git", "url": "https://github.com/phpbench/phpbench.git", - "reference": "4248817222514421cba466bfa7adc7d8932345d4" + "reference": "78cd98a9aa34e0f8f80ca01972a8b88d2c30194b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpbench/phpbench/zipball/4248817222514421cba466bfa7adc7d8932345d4", - "reference": "4248817222514421cba466bfa7adc7d8932345d4", + "url": "https://api.github.com/repos/phpbench/phpbench/zipball/78cd98a9aa34e0f8f80ca01972a8b88d2c30194b", + "reference": "78cd98a9aa34e0f8f80ca01972a8b88d2c30194b", "shasum": "" }, "require": { @@ -5880,7 +5880,7 @@ ], "support": { "issues": "https://github.com/phpbench/phpbench/issues", - "source": "https://github.com/phpbench/phpbench/tree/1.4.0" + "source": "https://github.com/phpbench/phpbench/tree/1.4.1" }, "funding": [ { @@ -5888,7 +5888,7 @@ "type": "github" } ], - "time": "2025-01-26T19:54:45+00:00" + "time": "2025-03-12T08:01:40+00:00" }, { "name": "phpunit/php-code-coverage", From 81159e82d588a93c4b26827636a7c7c7eac40399 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 12 Mar 2025 10:29:07 +0000 Subject: [PATCH 142/275] show time in seconds --- src/Appwrite/Platform/Workers/StatsResources.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index ecfe80bf2d..1140698342 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -81,7 +81,7 @@ class StatsResources extends Action $this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project); $endTime = microtime(true); $executionTime = $endTime - $startTime; - Console::info('Project: ' . $project->getId() . '(' . $project->getInternalId() . ') aggregated in ' . date('H:i:s.u', intval($executionTime)) .''); + Console::info('Project: ' . $project->getId() . '(' . $project->getInternalId() . ') aggregated in ' . $executionTime .' seconds'); } protected function countForProject(Database $dbForPlatform, callable $getLogsDB, callable $getProjectDB, Document $project): void From ae00a69592d665fc63d5b246708ea3b87c14aac0 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 12 Mar 2025 11:58:06 +0000 Subject: [PATCH 143/275] chore: update initializing dbForLogs --- src/Appwrite/Platform/Workers/Deletes.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 643fc4d010..d93ee3ebcf 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -414,8 +414,11 @@ class Deletes extends Action */ private function deleteUsageStats(Document $project, callable $getProjectDB, callable $getLogsDB, string $hourlyUsageRetentionDatetime): void { + /** @var \Utopia\Database\Database $dbForProject*/ $dbForProject = $getProjectDB($project); - $dbForLogs = $getLogsDB($project); + + /** @var \Utopia\Database\Database $dbForLogs*/ + $dbForLogs = call_user_func($getLogsDB, $project); // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ From 46693faa67673bb0c24d23c7b8533e4825f3f3d1 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 12 Mar 2025 12:35:56 +0000 Subject: [PATCH 144/275] chore: prevent console project to queue for delete stats --- src/Appwrite/Platform/Workers/Deletes.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index d93ee3ebcf..433b99c7a9 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -417,20 +417,22 @@ class Deletes extends Action /** @var \Utopia\Database\Database $dbForProject*/ $dbForProject = $getProjectDB($project); - /** @var \Utopia\Database\Database $dbForLogs*/ - $dbForLogs = call_user_func($getLogsDB, $project); - // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::equal('period', ['1h']), ], $dbForProject); - // Delete Usage stats from logsDB - $this->deleteByGroup('stats', [ - Query::lessThan('time', $hourlyUsageRetentionDatetime), - Query::equal('period', ['1h']), - ], $dbForLogs); + if ($project->getId() !== 'console') { + /** @var \Utopia\Database\Database $dbForLogs*/ + $dbForLogs = call_user_func($getLogsDB, $project); + + // Delete Usage stats from logsDB + $this->deleteByGroup('stats', [ + Query::lessThan('time', $hourlyUsageRetentionDatetime), + Query::equal('period', ['1h']), + ], $dbForLogs); + } } /** From 7402145c9d2deb7e5c120d08973469d5553cb596 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 12 Mar 2025 15:22:43 -0700 Subject: [PATCH 145/275] Bump console to version 5.2.53 --- app/views/install/compose.phtml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 62fcd03624..b16806895f 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -168,7 +168,7 @@ $image = $this->getParam('image', ''); appwrite-console: <<: *x-logging container_name: appwrite-console - image: /console:5.2.27 + image: /console:5.2.53 restart: unless-stopped networks: - appwrite diff --git a/docker-compose.yml b/docker-compose.yml index facf0e6db9..b88f46e674 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -201,7 +201,7 @@ services: appwrite-console: <<: *x-logging container_name: appwrite-console - image: appwrite/console:5.2.27 + image: appwrite/console:5.2.53 restart: unless-stopped networks: - appwrite From 5bce1e17cb4b96d83d529787416021e017ac5a46 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 12 Mar 2025 15:34:38 -0700 Subject: [PATCH 146/275] chore: regenerate latest and 1.6.x specs --- app/config/specs/open-api3-1.6.x-client.json | 10 +- app/config/specs/open-api3-1.6.x-console.json | 222 ++++++++++------- app/config/specs/open-api3-1.6.x-server.json | 182 +++++++------- .../specs/open-api3-latest-console.json | 22 +- app/config/specs/open-api3-latest-server.json | 4 - app/config/specs/swagger2-1.6.x-client.json | 10 +- app/config/specs/swagger2-1.6.x-console.json | 226 +++++++++++------- app/config/specs/swagger2-1.6.x-server.json | 184 +++++++------- app/config/specs/swagger2-latest-console.json | 23 +- app/config/specs/swagger2-latest-server.json | 4 - 10 files changed, 495 insertions(+), 392 deletions(-) diff --git a/app/config/specs/open-api3-1.6.x-client.json b/app/config/specs/open-api3-1.6.x-client.json index 820b1f55e0..316fe13116 100644 --- a/app/config/specs/open-api3-1.6.x-client.json +++ b/app/config/specs/open-api3-1.6.x-client.json @@ -3383,7 +3383,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "schema": { "type": "string", @@ -3404,7 +3404,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3423,7 +3424,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ] }, "in": "path" @@ -4365,7 +4367,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", diff --git a/app/config/specs/open-api3-1.6.x-console.json b/app/config/specs/open-api3-1.6.x-console.json index 7f57dfc437..54161c4262 100644 --- a/app/config/specs/open-api3-1.6.x-console.json +++ b/app/config/specs/open-api3-1.6.x-console.json @@ -3387,7 +3387,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "schema": { "type": "string", @@ -3408,7 +3408,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3427,7 +3428,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ] }, "in": "path" @@ -6407,8 +6409,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -6644,8 +6644,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -7823,7 +7821,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", @@ -11585,7 +11583,7 @@ }, "x-appwrite": { "method": "getCertificate", - "weight": 134, + "weight": 133, "cookies": false, "type": "", "deprecated": false, @@ -11692,7 +11690,7 @@ }, "x-appwrite": { "method": "getPubSub", - "weight": 130, + "weight": 129, "cookies": false, "type": "", "deprecated": false, @@ -11718,54 +11716,6 @@ ] } }, - "\/health\/queue": { - "get": { - "summary": "Get queue", - "operationId": "healthGetQueue", - "tags": [ - "health" - ], - "description": "Check the Appwrite queue messaging servers are up and connection is successful.", - "responses": { - "200": { - "description": "Health Status", - "content": { - "application\/json": { - "schema": { - "$ref": "#\/components\/schemas\/healthStatus" - } - } - } - } - }, - "x-appwrite": { - "method": "getQueue", - "weight": 129, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "health\/get-queue.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "health.read", - "platforms": [ - "server" - ], - "packaging": false, - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [], - "Key": [] - } - ] - } - }, "\/health\/queue\/builds": { "get": { "summary": "Get builds queue", @@ -11788,7 +11738,7 @@ }, "x-appwrite": { "method": "getQueueBuilds", - "weight": 136, + "weight": 135, "cookies": false, "type": "", "deprecated": false, @@ -11849,7 +11799,7 @@ }, "x-appwrite": { "method": "getQueueCertificates", - "weight": 135, + "weight": 134, "cookies": false, "type": "", "deprecated": false, @@ -11910,7 +11860,7 @@ }, "x-appwrite": { "method": "getQueueDatabases", - "weight": 137, + "weight": 136, "cookies": false, "type": "", "deprecated": false, @@ -11982,7 +11932,7 @@ }, "x-appwrite": { "method": "getQueueDeletes", - "weight": 138, + "weight": 137, "cookies": false, "type": "", "deprecated": false, @@ -12081,8 +12031,9 @@ "v1-audits", "v1-mails", "v1-functions", - "v1-usage", - "v1-usage-dump", + "v1-stats-resources", + "v1-stats-usage", + "v1-stats-usage-dump", "v1-webhooks", "v1-certificates", "v1-builds", @@ -12130,7 +12081,7 @@ }, "x-appwrite": { "method": "getQueueFunctions", - "weight": 142, + "weight": 141, "cookies": false, "type": "", "deprecated": false, @@ -12191,7 +12142,7 @@ }, "x-appwrite": { "method": "getQueueLogs", - "weight": 133, + "weight": 132, "cookies": false, "type": "", "deprecated": false, @@ -12252,7 +12203,7 @@ }, "x-appwrite": { "method": "getQueueMails", - "weight": 139, + "weight": 138, "cookies": false, "type": "", "deprecated": false, @@ -12313,7 +12264,7 @@ }, "x-appwrite": { "method": "getQueueMessaging", - "weight": 140, + "weight": 139, "cookies": false, "type": "", "deprecated": false, @@ -12374,7 +12325,7 @@ }, "x-appwrite": { "method": "getQueueMigrations", - "weight": 141, + "weight": 140, "cookies": false, "type": "", "deprecated": false, @@ -12413,14 +12364,14 @@ ] } }, - "\/health\/queue\/usage": { + "\/health\/queue\/stats-resources": { "get": { - "summary": "Get usage queue", - "operationId": "healthGetQueueUsage", + "summary": "Get stats resources queue", + "operationId": "healthGetQueueStatsResources", "tags": [ "health" ], - "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "description": "Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.", "responses": { "200": { "description": "Health Queue", @@ -12434,13 +12385,13 @@ } }, "x-appwrite": { - "method": "getQueueUsage", - "weight": 143, + "method": "getQueueStatsResources", + "weight": 142, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage.md", + "demo": "health\/get-queue-stats-resources.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-resources.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -12474,10 +12425,71 @@ ] } }, - "\/health\/queue\/usage-dump": { + "\/health\/queue\/stats-usage": { + "get": { + "summary": "Get stats usage queue", + "operationId": "healthGetQueueUsage", + "tags": [ + "health" + ], + "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "responses": { + "200": { + "description": "Health Queue", + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/healthQueue" + } + } + } + } + }, + "x-appwrite": { + "method": "getQueueUsage", + "weight": 143, + "cookies": false, + "type": "", + "deprecated": false, + "demo": "health\/get-queue-usage.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage.md", + "rate-limit": 0, + "rate-time": 3600, + "rate-key": "url:{url},ip:{ip}", + "scope": "health.read", + "platforms": [ + "server" + ], + "packaging": false, + "auth": { + "Project": [] + } + }, + "security": [ + { + "Project": [], + "Key": [] + } + ], + "parameters": [ + { + "name": "threshold", + "description": "Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.", + "required": false, + "schema": { + "type": "integer", + "format": "int32", + "default": 5000 + }, + "in": "query" + } + ] + } + }, + "\/health\/queue\/stats-usage-dump": { "get": { "summary": "Get usage dump queue", - "operationId": "healthGetQueueUsageDump", + "operationId": "healthGetQueueStatsUsageDump", "tags": [ "health" ], @@ -12495,13 +12507,13 @@ } }, "x-appwrite": { - "method": "getQueueUsageDump", + "method": "getQueueStatsUsageDump", "weight": 144, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage-dump.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage-dump.md", + "demo": "health\/get-queue-stats-usage-dump.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage-dump.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -12557,7 +12569,7 @@ }, "x-appwrite": { "method": "getQueueWebhooks", - "weight": 132, + "weight": 131, "cookies": false, "type": "", "deprecated": false, @@ -12714,7 +12726,7 @@ }, "x-appwrite": { "method": "getTime", - "weight": 131, + "weight": 130, "cookies": false, "type": "", "deprecated": false, @@ -17609,17 +17621,17 @@ }, "endpoint": { "type": "string", - "description": "Source's Appwrite Endpoint", + "description": "Source Appwrite endpoint", "x-example": "https:\/\/example.com" }, "projectId": { "type": "string", - "description": "Source's Project ID", + "description": "Source Project ID", "x-example": "" }, "apiKey": { "type": "string", - "description": "Source's API Key", + "description": "Source API Key", "x-example": "" } }, @@ -36052,6 +36064,20 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "imageTransformations": { + "type": "array", + "description": "Aggregated number of files transformations per period.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] + }, + "imageTransformationsTotal": { + "type": "integer", + "description": "Total aggregated number of files transformations.", + "x-example": 0, + "format": "int32" } }, "required": [ @@ -36059,7 +36085,9 @@ "filesTotal", "filesStorageTotal", "files", - "storage" + "storage", + "imageTransformations", + "imageTransformationsTotal" ] }, "usageFunctions": { @@ -36597,6 +36625,20 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "imageTransformations": { + "type": "array", + "description": "An array of aggregated number of image transformations.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] + }, + "imageTransformationsTotal": { + "type": "integer", + "description": "Total aggregated number of image transformations.", + "x-example": 0, + "format": "int32" } }, "required": [ @@ -36628,7 +36670,9 @@ "authPhoneEstimate", "authPhoneCountryBreakdown", "databasesReads", - "databasesWrites" + "databasesWrites", + "imageTransformations", + "imageTransformationsTotal" ] }, "headers": { diff --git a/app/config/specs/open-api3-1.6.x-server.json b/app/config/specs/open-api3-1.6.x-server.json index 68d408762a..3d32d3e978 100644 --- a/app/config/specs/open-api3-1.6.x-server.json +++ b/app/config/specs/open-api3-1.6.x-server.json @@ -3077,7 +3077,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "schema": { "type": "string", @@ -3098,7 +3098,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3117,7 +3118,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ] }, "in": "path" @@ -5951,8 +5953,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -6190,8 +6190,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -7381,7 +7379,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", @@ -10461,7 +10459,7 @@ }, "x-appwrite": { "method": "getCertificate", - "weight": 134, + "weight": 133, "cookies": false, "type": "", "deprecated": false, @@ -10570,7 +10568,7 @@ }, "x-appwrite": { "method": "getPubSub", - "weight": 130, + "weight": 129, "cookies": false, "type": "", "deprecated": false, @@ -10597,55 +10595,6 @@ ] } }, - "\/health\/queue": { - "get": { - "summary": "Get queue", - "operationId": "healthGetQueue", - "tags": [ - "health" - ], - "description": "Check the Appwrite queue messaging servers are up and connection is successful.", - "responses": { - "200": { - "description": "Health Status", - "content": { - "application\/json": { - "schema": { - "$ref": "#\/components\/schemas\/healthStatus" - } - } - } - } - }, - "x-appwrite": { - "method": "getQueue", - "weight": 129, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "health\/get-queue.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "health.read", - "platforms": [ - "server" - ], - "packaging": false, - "auth": { - "Project": [], - "Key": [] - } - }, - "security": [ - { - "Project": [], - "Key": [] - } - ] - } - }, "\/health\/queue\/builds": { "get": { "summary": "Get builds queue", @@ -10668,7 +10617,7 @@ }, "x-appwrite": { "method": "getQueueBuilds", - "weight": 136, + "weight": 135, "cookies": false, "type": "", "deprecated": false, @@ -10730,7 +10679,7 @@ }, "x-appwrite": { "method": "getQueueCertificates", - "weight": 135, + "weight": 134, "cookies": false, "type": "", "deprecated": false, @@ -10792,7 +10741,7 @@ }, "x-appwrite": { "method": "getQueueDatabases", - "weight": 137, + "weight": 136, "cookies": false, "type": "", "deprecated": false, @@ -10865,7 +10814,7 @@ }, "x-appwrite": { "method": "getQueueDeletes", - "weight": 138, + "weight": 137, "cookies": false, "type": "", "deprecated": false, @@ -10966,8 +10915,9 @@ "v1-audits", "v1-mails", "v1-functions", - "v1-usage", - "v1-usage-dump", + "v1-stats-resources", + "v1-stats-usage", + "v1-stats-usage-dump", "v1-webhooks", "v1-certificates", "v1-builds", @@ -11015,7 +10965,7 @@ }, "x-appwrite": { "method": "getQueueFunctions", - "weight": 142, + "weight": 141, "cookies": false, "type": "", "deprecated": false, @@ -11077,7 +11027,7 @@ }, "x-appwrite": { "method": "getQueueLogs", - "weight": 133, + "weight": 132, "cookies": false, "type": "", "deprecated": false, @@ -11139,7 +11089,7 @@ }, "x-appwrite": { "method": "getQueueMails", - "weight": 139, + "weight": 138, "cookies": false, "type": "", "deprecated": false, @@ -11201,7 +11151,7 @@ }, "x-appwrite": { "method": "getQueueMessaging", - "weight": 140, + "weight": 139, "cookies": false, "type": "", "deprecated": false, @@ -11263,7 +11213,7 @@ }, "x-appwrite": { "method": "getQueueMigrations", - "weight": 141, + "weight": 140, "cookies": false, "type": "", "deprecated": false, @@ -11303,14 +11253,14 @@ ] } }, - "\/health\/queue\/usage": { + "\/health\/queue\/stats-resources": { "get": { - "summary": "Get usage queue", - "operationId": "healthGetQueueUsage", + "summary": "Get stats resources queue", + "operationId": "healthGetQueueStatsResources", "tags": [ "health" ], - "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "description": "Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.", "responses": { "200": { "description": "Health Queue", @@ -11324,13 +11274,13 @@ } }, "x-appwrite": { - "method": "getQueueUsage", - "weight": 143, + "method": "getQueueStatsResources", + "weight": 142, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage.md", + "demo": "health\/get-queue-stats-resources.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-resources.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -11365,10 +11315,72 @@ ] } }, - "\/health\/queue\/usage-dump": { + "\/health\/queue\/stats-usage": { + "get": { + "summary": "Get stats usage queue", + "operationId": "healthGetQueueUsage", + "tags": [ + "health" + ], + "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "responses": { + "200": { + "description": "Health Queue", + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/healthQueue" + } + } + } + } + }, + "x-appwrite": { + "method": "getQueueUsage", + "weight": 143, + "cookies": false, + "type": "", + "deprecated": false, + "demo": "health\/get-queue-usage.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage.md", + "rate-limit": 0, + "rate-time": 3600, + "rate-key": "url:{url},ip:{ip}", + "scope": "health.read", + "platforms": [ + "server" + ], + "packaging": false, + "auth": { + "Project": [], + "Key": [] + } + }, + "security": [ + { + "Project": [], + "Key": [] + } + ], + "parameters": [ + { + "name": "threshold", + "description": "Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.", + "required": false, + "schema": { + "type": "integer", + "format": "int32", + "default": 5000 + }, + "in": "query" + } + ] + } + }, + "\/health\/queue\/stats-usage-dump": { "get": { "summary": "Get usage dump queue", - "operationId": "healthGetQueueUsageDump", + "operationId": "healthGetQueueStatsUsageDump", "tags": [ "health" ], @@ -11386,13 +11398,13 @@ } }, "x-appwrite": { - "method": "getQueueUsageDump", + "method": "getQueueStatsUsageDump", "weight": 144, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage-dump.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage-dump.md", + "demo": "health\/get-queue-stats-usage-dump.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage-dump.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -11449,7 +11461,7 @@ }, "x-appwrite": { "method": "getQueueWebhooks", - "weight": 132, + "weight": 131, "cookies": false, "type": "", "deprecated": false, @@ -11609,7 +11621,7 @@ }, "x-appwrite": { "method": "getTime", - "weight": 131, + "weight": 130, "cookies": false, "type": "", "deprecated": false, diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index bc216226fb..54161c4262 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -6409,8 +6409,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -6646,8 +6644,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -36630,13 +36626,15 @@ }, "x-example": [] }, - "imageTransformationsTotal": { - "type": "integer", - "description": "An array of aggregated number of image transformations.", - "x-example": 0, - "format": "int32" - }, "imageTransformations": { + "type": "array", + "description": "An array of aggregated number of image transformations.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] + }, + "imageTransformationsTotal": { "type": "integer", "description": "Total aggregated number of image transformations.", "x-example": 0, @@ -36673,8 +36671,8 @@ "authPhoneCountryBreakdown", "databasesReads", "databasesWrites", - "imageTransformationsTotal", - "imageTransformations" + "imageTransformations", + "imageTransformationsTotal" ] }, "headers": { diff --git a/app/config/specs/open-api3-latest-server.json b/app/config/specs/open-api3-latest-server.json index 280c080514..3d32d3e978 100644 --- a/app/config/specs/open-api3-latest-server.json +++ b/app/config/specs/open-api3-latest-server.json @@ -5953,8 +5953,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -6192,8 +6190,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } diff --git a/app/config/specs/swagger2-1.6.x-client.json b/app/config/specs/swagger2-1.6.x-client.json index c0980c44ce..8960bfaa5c 100644 --- a/app/config/specs/swagger2-1.6.x-client.json +++ b/app/config/specs/swagger2-1.6.x-client.json @@ -3557,7 +3557,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "type": "string", "x-example": "amex", @@ -3577,7 +3577,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3596,7 +3597,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ], "in": "path" }, @@ -4547,7 +4549,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", diff --git a/app/config/specs/swagger2-1.6.x-console.json b/app/config/specs/swagger2-1.6.x-console.json index 94b0d55199..8fc7e7daf3 100644 --- a/app/config/specs/swagger2-1.6.x-console.json +++ b/app/config/specs/swagger2-1.6.x-console.json @@ -3577,7 +3577,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "type": "string", "x-example": "amex", @@ -3597,7 +3597,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3616,7 +3617,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ], "in": "path" }, @@ -6607,8 +6609,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -6845,8 +6845,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -8012,7 +8010,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", @@ -11810,7 +11808,7 @@ }, "x-appwrite": { "method": "getCertificate", - "weight": 134, + "weight": 133, "cookies": false, "type": "", "deprecated": false, @@ -11919,7 +11917,7 @@ }, "x-appwrite": { "method": "getPubSub", - "weight": 130, + "weight": 129, "cookies": false, "type": "", "deprecated": false, @@ -11945,56 +11943,6 @@ ] } }, - "\/health\/queue": { - "get": { - "summary": "Get queue", - "operationId": "healthGetQueue", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "health" - ], - "description": "Check the Appwrite queue messaging servers are up and connection is successful.", - "responses": { - "200": { - "description": "Health Status", - "schema": { - "$ref": "#\/definitions\/healthStatus" - } - } - }, - "x-appwrite": { - "method": "getQueue", - "weight": 129, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "health\/get-queue.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "health.read", - "platforms": [ - "server" - ], - "packaging": false, - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [], - "Key": [] - } - ] - } - }, "\/health\/queue\/builds": { "get": { "summary": "Get builds queue", @@ -12019,7 +11967,7 @@ }, "x-appwrite": { "method": "getQueueBuilds", - "weight": 136, + "weight": 135, "cookies": false, "type": "", "deprecated": false, @@ -12080,7 +12028,7 @@ }, "x-appwrite": { "method": "getQueueCertificates", - "weight": 135, + "weight": 134, "cookies": false, "type": "", "deprecated": false, @@ -12141,7 +12089,7 @@ }, "x-appwrite": { "method": "getQueueDatabases", - "weight": 137, + "weight": 136, "cookies": false, "type": "", "deprecated": false, @@ -12211,7 +12159,7 @@ }, "x-appwrite": { "method": "getQueueDeletes", - "weight": 138, + "weight": 137, "cookies": false, "type": "", "deprecated": false, @@ -12309,8 +12257,9 @@ "v1-audits", "v1-mails", "v1-functions", - "v1-usage", - "v1-usage-dump", + "v1-stats-resources", + "v1-stats-usage", + "v1-stats-usage-dump", "v1-webhooks", "v1-certificates", "v1-builds", @@ -12357,7 +12306,7 @@ }, "x-appwrite": { "method": "getQueueFunctions", - "weight": 142, + "weight": 141, "cookies": false, "type": "", "deprecated": false, @@ -12418,7 +12367,7 @@ }, "x-appwrite": { "method": "getQueueLogs", - "weight": 133, + "weight": 132, "cookies": false, "type": "", "deprecated": false, @@ -12479,7 +12428,7 @@ }, "x-appwrite": { "method": "getQueueMails", - "weight": 139, + "weight": 138, "cookies": false, "type": "", "deprecated": false, @@ -12540,7 +12489,7 @@ }, "x-appwrite": { "method": "getQueueMessaging", - "weight": 140, + "weight": 139, "cookies": false, "type": "", "deprecated": false, @@ -12601,7 +12550,7 @@ }, "x-appwrite": { "method": "getQueueMigrations", - "weight": 141, + "weight": 140, "cookies": false, "type": "", "deprecated": false, @@ -12638,10 +12587,10 @@ ] } }, - "\/health\/queue\/usage": { + "\/health\/queue\/stats-resources": { "get": { - "summary": "Get usage queue", - "operationId": "healthGetQueueUsage", + "summary": "Get stats resources queue", + "operationId": "healthGetQueueStatsResources", "consumes": [ "application\/json" ], @@ -12651,7 +12600,7 @@ "tags": [ "health" ], - "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "description": "Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.", "responses": { "200": { "description": "Health Queue", @@ -12661,13 +12610,13 @@ } }, "x-appwrite": { - "method": "getQueueUsage", - "weight": 143, + "method": "getQueueStatsResources", + "weight": 142, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage.md", + "demo": "health\/get-queue-stats-resources.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-resources.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -12699,10 +12648,71 @@ ] } }, - "\/health\/queue\/usage-dump": { + "\/health\/queue\/stats-usage": { + "get": { + "summary": "Get stats usage queue", + "operationId": "healthGetQueueUsage", + "consumes": [ + "application\/json" + ], + "produces": [ + "application\/json" + ], + "tags": [ + "health" + ], + "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "responses": { + "200": { + "description": "Health Queue", + "schema": { + "$ref": "#\/definitions\/healthQueue" + } + } + }, + "x-appwrite": { + "method": "getQueueUsage", + "weight": 143, + "cookies": false, + "type": "", + "deprecated": false, + "demo": "health\/get-queue-usage.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage.md", + "rate-limit": 0, + "rate-time": 3600, + "rate-key": "url:{url},ip:{ip}", + "scope": "health.read", + "platforms": [ + "server" + ], + "packaging": false, + "auth": { + "Project": [] + } + }, + "security": [ + { + "Project": [], + "Key": [] + } + ], + "parameters": [ + { + "name": "threshold", + "description": "Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.", + "required": false, + "type": "integer", + "format": "int32", + "default": 5000, + "in": "query" + } + ] + } + }, + "\/health\/queue\/stats-usage-dump": { "get": { "summary": "Get usage dump queue", - "operationId": "healthGetQueueUsageDump", + "operationId": "healthGetQueueStatsUsageDump", "consumes": [ "application\/json" ], @@ -12722,13 +12732,13 @@ } }, "x-appwrite": { - "method": "getQueueUsageDump", + "method": "getQueueStatsUsageDump", "weight": 144, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage-dump.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage-dump.md", + "demo": "health\/get-queue-stats-usage-dump.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage-dump.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -12784,7 +12794,7 @@ }, "x-appwrite": { "method": "getQueueWebhooks", - "weight": 132, + "weight": 131, "cookies": false, "type": "", "deprecated": false, @@ -12945,7 +12955,7 @@ }, "x-appwrite": { "method": "getTime", - "weight": 131, + "weight": 130, "cookies": false, "type": "", "deprecated": false, @@ -18070,19 +18080,19 @@ }, "endpoint": { "type": "string", - "description": "Source's Appwrite Endpoint", + "description": "Source Appwrite endpoint", "default": null, "x-example": "https:\/\/example.com" }, "projectId": { "type": "string", - "description": "Source's Project ID", + "description": "Source Project ID", "default": null, "x-example": "" }, "apiKey": { "type": "string", - "description": "Source's API Key", + "description": "Source API Key", "default": null, "x-example": "" } @@ -36608,6 +36618,21 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "imageTransformations": { + "type": "array", + "description": "Aggregated number of files transformations per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] + }, + "imageTransformationsTotal": { + "type": "integer", + "description": "Total aggregated number of files transformations.", + "x-example": 0, + "format": "int32" } }, "required": [ @@ -36615,7 +36640,9 @@ "filesTotal", "filesStorageTotal", "files", - "storage" + "storage", + "imageTransformations", + "imageTransformationsTotal" ] }, "usageFunctions": { @@ -37185,6 +37212,21 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "imageTransformations": { + "type": "array", + "description": "An array of aggregated number of image transformations.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] + }, + "imageTransformationsTotal": { + "type": "integer", + "description": "Total aggregated number of image transformations.", + "x-example": 0, + "format": "int32" } }, "required": [ @@ -37216,7 +37258,9 @@ "authPhoneEstimate", "authPhoneCountryBreakdown", "databasesReads", - "databasesWrites" + "databasesWrites", + "imageTransformations", + "imageTransformationsTotal" ] }, "headers": { diff --git a/app/config/specs/swagger2-1.6.x-server.json b/app/config/specs/swagger2-1.6.x-server.json index e38495629c..83757c94f4 100644 --- a/app/config/specs/swagger2-1.6.x-server.json +++ b/app/config/specs/swagger2-1.6.x-server.json @@ -3261,7 +3261,7 @@ "parameters": [ { "name": "code", - "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro.", + "description": "Credit Card Code. Possible values: amex, argencard, cabal, cencosud, diners, discover, elo, hipercard, jcb, mastercard, naranja, targeta-shopping, union-china-pay, visa, mir, maestro, rupay.", "required": true, "type": "string", "x-example": "amex", @@ -3281,7 +3281,8 @@ "union-china-pay", "visa", "mir", - "maestro" + "maestro", + "rupay" ], "x-enum-name": "CreditCard", "x-enum-keys": [ @@ -3300,7 +3301,8 @@ "Union China Pay", "Visa", "MIR", - "Maestro" + "Maestro", + "Rupay" ], "in": "path" }, @@ -6133,8 +6135,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -6373,8 +6373,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -7552,7 +7550,7 @@ "tags": [ "databases" ], - "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.", + "description": "Create a new Document. Before using this route, you should create a new collection resource using either a [server integration](https:\/\/appwrite.io\/docs\/server\/databases#databasesCreateCollection) API or directly from your database console.\n", "responses": { "201": { "description": "Document", @@ -10689,7 +10687,7 @@ }, "x-appwrite": { "method": "getCertificate", - "weight": 134, + "weight": 133, "cookies": false, "type": "", "deprecated": false, @@ -10800,7 +10798,7 @@ }, "x-appwrite": { "method": "getPubSub", - "weight": 130, + "weight": 129, "cookies": false, "type": "", "deprecated": false, @@ -10827,57 +10825,6 @@ ] } }, - "\/health\/queue": { - "get": { - "summary": "Get queue", - "operationId": "healthGetQueue", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "health" - ], - "description": "Check the Appwrite queue messaging servers are up and connection is successful.", - "responses": { - "200": { - "description": "Health Status", - "schema": { - "$ref": "#\/definitions\/healthStatus" - } - } - }, - "x-appwrite": { - "method": "getQueue", - "weight": 129, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "health\/get-queue.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "health.read", - "platforms": [ - "server" - ], - "packaging": false, - "auth": { - "Project": [], - "Key": [] - } - }, - "security": [ - { - "Project": [], - "Key": [] - } - ] - } - }, "\/health\/queue\/builds": { "get": { "summary": "Get builds queue", @@ -10902,7 +10849,7 @@ }, "x-appwrite": { "method": "getQueueBuilds", - "weight": 136, + "weight": 135, "cookies": false, "type": "", "deprecated": false, @@ -10964,7 +10911,7 @@ }, "x-appwrite": { "method": "getQueueCertificates", - "weight": 135, + "weight": 134, "cookies": false, "type": "", "deprecated": false, @@ -11026,7 +10973,7 @@ }, "x-appwrite": { "method": "getQueueDatabases", - "weight": 137, + "weight": 136, "cookies": false, "type": "", "deprecated": false, @@ -11097,7 +11044,7 @@ }, "x-appwrite": { "method": "getQueueDeletes", - "weight": 138, + "weight": 137, "cookies": false, "type": "", "deprecated": false, @@ -11197,8 +11144,9 @@ "v1-audits", "v1-mails", "v1-functions", - "v1-usage", - "v1-usage-dump", + "v1-stats-resources", + "v1-stats-usage", + "v1-stats-usage-dump", "v1-webhooks", "v1-certificates", "v1-builds", @@ -11245,7 +11193,7 @@ }, "x-appwrite": { "method": "getQueueFunctions", - "weight": 142, + "weight": 141, "cookies": false, "type": "", "deprecated": false, @@ -11307,7 +11255,7 @@ }, "x-appwrite": { "method": "getQueueLogs", - "weight": 133, + "weight": 132, "cookies": false, "type": "", "deprecated": false, @@ -11369,7 +11317,7 @@ }, "x-appwrite": { "method": "getQueueMails", - "weight": 139, + "weight": 138, "cookies": false, "type": "", "deprecated": false, @@ -11431,7 +11379,7 @@ }, "x-appwrite": { "method": "getQueueMessaging", - "weight": 140, + "weight": 139, "cookies": false, "type": "", "deprecated": false, @@ -11493,7 +11441,7 @@ }, "x-appwrite": { "method": "getQueueMigrations", - "weight": 141, + "weight": 140, "cookies": false, "type": "", "deprecated": false, @@ -11531,10 +11479,10 @@ ] } }, - "\/health\/queue\/usage": { + "\/health\/queue\/stats-resources": { "get": { - "summary": "Get usage queue", - "operationId": "healthGetQueueUsage", + "summary": "Get stats resources queue", + "operationId": "healthGetQueueStatsResources", "consumes": [ "application\/json" ], @@ -11544,7 +11492,7 @@ "tags": [ "health" ], - "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "description": "Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.", "responses": { "200": { "description": "Health Queue", @@ -11554,13 +11502,13 @@ } }, "x-appwrite": { - "method": "getQueueUsage", - "weight": 143, + "method": "getQueueStatsResources", + "weight": 142, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage.md", + "demo": "health\/get-queue-stats-resources.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-resources.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -11593,10 +11541,72 @@ ] } }, - "\/health\/queue\/usage-dump": { + "\/health\/queue\/stats-usage": { + "get": { + "summary": "Get stats usage queue", + "operationId": "healthGetQueueUsage", + "consumes": [ + "application\/json" + ], + "produces": [ + "application\/json" + ], + "tags": [ + "health" + ], + "description": "Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.", + "responses": { + "200": { + "description": "Health Queue", + "schema": { + "$ref": "#\/definitions\/healthQueue" + } + } + }, + "x-appwrite": { + "method": "getQueueUsage", + "weight": 143, + "cookies": false, + "type": "", + "deprecated": false, + "demo": "health\/get-queue-usage.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage.md", + "rate-limit": 0, + "rate-time": 3600, + "rate-key": "url:{url},ip:{ip}", + "scope": "health.read", + "platforms": [ + "server" + ], + "packaging": false, + "auth": { + "Project": [], + "Key": [] + } + }, + "security": [ + { + "Project": [], + "Key": [] + } + ], + "parameters": [ + { + "name": "threshold", + "description": "Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.", + "required": false, + "type": "integer", + "format": "int32", + "default": 5000, + "in": "query" + } + ] + } + }, + "\/health\/queue\/stats-usage-dump": { "get": { "summary": "Get usage dump queue", - "operationId": "healthGetQueueUsageDump", + "operationId": "healthGetQueueStatsUsageDump", "consumes": [ "application\/json" ], @@ -11616,13 +11626,13 @@ } }, "x-appwrite": { - "method": "getQueueUsageDump", + "method": "getQueueStatsUsageDump", "weight": 144, "cookies": false, "type": "", "deprecated": false, - "demo": "health\/get-queue-usage-dump.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-usage-dump.md", + "demo": "health\/get-queue-stats-usage-dump.md", + "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/health\/get-queue-stats-usage-dump.md", "rate-limit": 0, "rate-time": 3600, "rate-key": "url:{url},ip:{ip}", @@ -11679,7 +11689,7 @@ }, "x-appwrite": { "method": "getQueueWebhooks", - "weight": 132, + "weight": 131, "cookies": false, "type": "", "deprecated": false, @@ -11843,7 +11853,7 @@ }, "x-appwrite": { "method": "getTime", - "weight": 131, + "weight": 130, "cookies": false, "type": "", "deprecated": false, diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index 2dddc72802..8fc7e7daf3 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -6609,8 +6609,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -6847,8 +6845,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -37217,13 +37213,16 @@ }, "x-example": [] }, - "imageTransformationsTotal": { - "type": "integer", - "description": "An array of aggregated number of image transformations.", - "x-example": 0, - "format": "int32" - }, "imageTransformations": { + "type": "array", + "description": "An array of aggregated number of image transformations.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] + }, + "imageTransformationsTotal": { "type": "integer", "description": "Total aggregated number of image transformations.", "x-example": 0, @@ -37260,8 +37259,8 @@ "authPhoneCountryBreakdown", "databasesReads", "databasesWrites", - "imageTransformationsTotal", - "imageTransformations" + "imageTransformations", + "imageTransformationsTotal" ] }, "headers": { diff --git a/app/config/specs/swagger2-latest-server.json b/app/config/specs/swagger2-latest-server.json index 749f87553b..83757c94f4 100644 --- a/app/config/specs/swagger2-latest-server.json +++ b/app/config/specs/swagger2-latest-server.json @@ -6135,8 +6135,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } @@ -6375,8 +6373,6 @@ }, "required": [ "required", - "min", - "max", "default" ] } From 301471354ee985b698f201f613be975e1d55378c Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 12 Mar 2025 15:45:40 -0700 Subject: [PATCH 147/275] chore: regenerated 1.6.x examples To ensure we didn't have anything extra, I deleted the 1.6.x folder and then regenerated the examples via the sdk command. --- .../java/functions/get-deployment-download.md | 23 ----------- .../java/functions/get-template.md | 22 ---------- .../java/functions/list-templates.md | 25 ----------- .../functions/get-deployment-download.md | 14 ------- .../kotlin/functions/get-template.md | 13 ------ .../kotlin/functions/list-templates.md | 16 -------- .../examples/account/update-mfa-challenge.md | 2 +- .../functions/get-deployment-download.md | 13 ------ .../examples/functions/get-template.md | 12 ------ .../examples/functions/list-templates.md | 15 ------- .../examples/account/update-mfa-challenge.md | 2 +- .../functions/get-deployment-download.md | 29 ------------- .../examples/functions/get-template.md | 11 ----- .../examples/functions/list-templates.md | 14 ------- .../examples/account/update-mfa-challenge.md | 30 +++++++++++++- .../functions/get-deployment-download.md | 8 ---- .../examples/functions/get-template.md | 35 ---------------- .../examples/functions/list-templates.md | 41 ------------------- .../functions/get-deployment-download.md | 14 ------- .../examples/functions/get-template.md | 13 ------ .../examples/functions/list-templates.md | 16 -------- .../functions/get-deployment-download.md | 8 ---- .../examples/functions/get-template.md | 6 --- .../examples/functions/list-templates.md | 6 --- .../functions/get-deployment-download.md | 14 ------- .../examples/functions/get-template.md | 13 ------ .../examples/functions/list-templates.md | 16 -------- .../examples/functions/download-deployment.md | 3 -- .../examples/functions/get-specifications.md | 1 - .../create-firebase-o-auth-migration.md | 3 -- .../migrations/delete-firebase-auth.md | 1 - .../migrations/get-firebase-report-o-auth.md | 3 -- .../migrations/list-firebase-projects.md | 1 - .../examples/functions/download-deployment.md | 14 ------- .../examples/functions/get-specifications.md | 11 ----- .../create-firebase-o-auth-migration.md | 14 ------- .../migrations/delete-firebase-auth.md | 11 ----- .../migrations/get-firebase-report-o-auth.md | 14 ------- .../migrations/list-firebase-projects.md | 11 ----- .../examples/account/update-mfa-challenge.md | 2 +- .../examples/functions/download-deployment.md | 13 ------ .../examples/functions/get-template.md | 11 ----- .../examples/functions/list-templates.md | 14 ------- .../examples/functions/download-deployment.md | 13 ------ .../examples/functions/get-template.md | 11 ----- .../examples/functions/list-templates.md | 14 ------- .../examples/account/update-mfa-challenge.md | 2 +- .../examples/functions/download-deployment.md | 15 ------- .../examples/functions/get-template.md | 13 ------ .../examples/functions/list-templates.md | 16 -------- .../examples/functions/download-deployment.md | 27 ------------ .../examples/functions/get-template.md | 25 ----------- .../examples/functions/list-templates.md | 28 ------------- .../examples/account/update-mfa-challenge.md | 30 +++++++++++++- .../examples/functions/download-deployment.md | 8 ---- .../examples/functions/get-template.md | 35 ---------------- .../examples/functions/list-templates.md | 41 ------------------- .../users/delete-mfa-authenticator.md | 30 -------------- .../java/functions/download-deployment.md | 24 ----------- .../java/functions/get-template.md | 22 ---------- .../java/functions/list-templates.md | 25 ----------- .../kotlin/functions/download-deployment.md | 15 ------- .../kotlin/functions/get-template.md | 13 ------ .../kotlin/functions/list-templates.md | 16 -------- .../examples/functions/download-deployment.md | 13 ------ .../examples/functions/get-template.md | 11 ----- .../examples/functions/list-templates.md | 14 ------- .../examples/functions/download-deployment.md | 16 -------- .../examples/functions/get-template.md | 14 ------- .../examples/functions/list-templates.md | 17 -------- .../examples/functions/download-deployment.md | 13 ------ .../examples/functions/get-template.md | 11 ----- .../examples/functions/list-templates.md | 14 ------- .../examples/functions/download-deployment.md | 7 ---- .../examples/functions/get-template.md | 6 --- .../examples/functions/list-templates.md | 6 --- .../examples/functions/download-deployment.md | 15 ------- .../examples/functions/get-template.md | 13 ------ .../examples/functions/list-templates.md | 16 -------- .../examples/account/update-mfa-challenge.md | 2 +- .../examples/functions/download-deployment.md | 14 ------- .../examples/functions/get-template.md | 12 ------ .../examples/functions/list-templates.md | 15 ------- .../users/delete-mfa-authenticator.md | 2 +- 84 files changed, 64 insertions(+), 1148 deletions(-) delete mode 100644 docs/examples/1.6.x/client-android/java/functions/get-deployment-download.md delete mode 100644 docs/examples/1.6.x/client-android/java/functions/get-template.md delete mode 100644 docs/examples/1.6.x/client-android/java/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/client-android/kotlin/functions/get-deployment-download.md delete mode 100644 docs/examples/1.6.x/client-android/kotlin/functions/get-template.md delete mode 100644 docs/examples/1.6.x/client-android/kotlin/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/client-apple/examples/functions/get-deployment-download.md delete mode 100644 docs/examples/1.6.x/client-apple/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/client-apple/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/client-flutter/examples/functions/get-deployment-download.md delete mode 100644 docs/examples/1.6.x/client-flutter/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/client-flutter/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/client-graphql/examples/functions/get-deployment-download.md delete mode 100644 docs/examples/1.6.x/client-graphql/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/client-graphql/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/client-react-native/examples/functions/get-deployment-download.md delete mode 100644 docs/examples/1.6.x/client-react-native/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/client-react-native/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/client-rest/examples/functions/get-deployment-download.md delete mode 100644 docs/examples/1.6.x/client-rest/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/client-rest/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/client-web/examples/functions/get-deployment-download.md delete mode 100644 docs/examples/1.6.x/client-web/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/client-web/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/console-cli/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/console-cli/examples/functions/get-specifications.md delete mode 100644 docs/examples/1.6.x/console-cli/examples/migrations/create-firebase-o-auth-migration.md delete mode 100644 docs/examples/1.6.x/console-cli/examples/migrations/delete-firebase-auth.md delete mode 100644 docs/examples/1.6.x/console-cli/examples/migrations/get-firebase-report-o-auth.md delete mode 100644 docs/examples/1.6.x/console-cli/examples/migrations/list-firebase-projects.md delete mode 100644 docs/examples/1.6.x/console-web/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/console-web/examples/functions/get-specifications.md delete mode 100644 docs/examples/1.6.x/console-web/examples/migrations/create-firebase-o-auth-migration.md delete mode 100644 docs/examples/1.6.x/console-web/examples/migrations/delete-firebase-auth.md delete mode 100644 docs/examples/1.6.x/console-web/examples/migrations/get-firebase-report-o-auth.md delete mode 100644 docs/examples/1.6.x/console-web/examples/migrations/list-firebase-projects.md delete mode 100644 docs/examples/1.6.x/server-dart/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-dart/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-dart/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/server-deno/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-deno/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-deno/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/server-dotnet/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-dotnet/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-dotnet/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/server-go/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-go/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-go/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/server-graphql/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-graphql/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-graphql/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/server-kotlin/java/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-kotlin/java/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-kotlin/java/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/server-kotlin/kotlin/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-kotlin/kotlin/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-kotlin/kotlin/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/server-nodejs/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-nodejs/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-nodejs/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/server-php/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-php/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-php/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/server-python/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-python/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-python/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/server-rest/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-rest/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-rest/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/server-ruby/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-ruby/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-ruby/examples/functions/list-templates.md delete mode 100644 docs/examples/1.6.x/server-swift/examples/functions/download-deployment.md delete mode 100644 docs/examples/1.6.x/server-swift/examples/functions/get-template.md delete mode 100644 docs/examples/1.6.x/server-swift/examples/functions/list-templates.md diff --git a/docs/examples/1.6.x/client-android/java/functions/get-deployment-download.md b/docs/examples/1.6.x/client-android/java/functions/get-deployment-download.md deleted file mode 100644 index 200cdf2fc5..0000000000 --- a/docs/examples/1.6.x/client-android/java/functions/get-deployment-download.md +++ /dev/null @@ -1,23 +0,0 @@ -import io.appwrite.Client; -import io.appwrite.coroutines.CoroutineCallback; -import io.appwrite.services.Functions; - -Client client = new Client(context) - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject(""); // Your project ID - -Functions functions = new Functions(client); - -functions.getDeploymentDownload( - "", // functionId - "", // deploymentId - new CoroutineCallback<>((result, error) -> { - if (error != null) { - error.printStackTrace(); - return; - } - - Log.d("Appwrite", result.toString()); - }) -); - diff --git a/docs/examples/1.6.x/client-android/java/functions/get-template.md b/docs/examples/1.6.x/client-android/java/functions/get-template.md deleted file mode 100644 index af9efc7b92..0000000000 --- a/docs/examples/1.6.x/client-android/java/functions/get-template.md +++ /dev/null @@ -1,22 +0,0 @@ -import io.appwrite.Client; -import io.appwrite.coroutines.CoroutineCallback; -import io.appwrite.services.Functions; - -Client client = new Client(context) - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject(""); // Your project ID - -Functions functions = new Functions(client); - -functions.getTemplate( - "", // templateId - new CoroutineCallback<>((result, error) -> { - if (error != null) { - error.printStackTrace(); - return; - } - - Log.d("Appwrite", result.toString()); - }) -); - diff --git a/docs/examples/1.6.x/client-android/java/functions/list-templates.md b/docs/examples/1.6.x/client-android/java/functions/list-templates.md deleted file mode 100644 index 6c051f3b04..0000000000 --- a/docs/examples/1.6.x/client-android/java/functions/list-templates.md +++ /dev/null @@ -1,25 +0,0 @@ -import io.appwrite.Client; -import io.appwrite.coroutines.CoroutineCallback; -import io.appwrite.services.Functions; - -Client client = new Client(context) - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject(""); // Your project ID - -Functions functions = new Functions(client); - -functions.listTemplates( - listOf(), // runtimes (optional) - listOf(), // useCases (optional) - 1, // limit (optional) - 0, // offset (optional) - new CoroutineCallback<>((result, error) -> { - if (error != null) { - error.printStackTrace(); - return; - } - - Log.d("Appwrite", result.toString()); - }) -); - diff --git a/docs/examples/1.6.x/client-android/kotlin/functions/get-deployment-download.md b/docs/examples/1.6.x/client-android/kotlin/functions/get-deployment-download.md deleted file mode 100644 index 21e99cb731..0000000000 --- a/docs/examples/1.6.x/client-android/kotlin/functions/get-deployment-download.md +++ /dev/null @@ -1,14 +0,0 @@ -import io.appwrite.Client -import io.appwrite.coroutines.CoroutineCallback -import io.appwrite.services.Functions - -val client = Client(context) - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - -val functions = Functions(client) - -val result = functions.getDeploymentDownload( - functionId = "", - deploymentId = "", -) \ No newline at end of file diff --git a/docs/examples/1.6.x/client-android/kotlin/functions/get-template.md b/docs/examples/1.6.x/client-android/kotlin/functions/get-template.md deleted file mode 100644 index 4af83bda3e..0000000000 --- a/docs/examples/1.6.x/client-android/kotlin/functions/get-template.md +++ /dev/null @@ -1,13 +0,0 @@ -import io.appwrite.Client -import io.appwrite.coroutines.CoroutineCallback -import io.appwrite.services.Functions - -val client = Client(context) - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - -val functions = Functions(client) - -val result = functions.getTemplate( - templateId = "", -) \ No newline at end of file diff --git a/docs/examples/1.6.x/client-android/kotlin/functions/list-templates.md b/docs/examples/1.6.x/client-android/kotlin/functions/list-templates.md deleted file mode 100644 index a1a59d9438..0000000000 --- a/docs/examples/1.6.x/client-android/kotlin/functions/list-templates.md +++ /dev/null @@ -1,16 +0,0 @@ -import io.appwrite.Client -import io.appwrite.coroutines.CoroutineCallback -import io.appwrite.services.Functions - -val client = Client(context) - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - -val functions = Functions(client) - -val result = functions.listTemplates( - runtimes = listOf(), // (optional) - useCases = listOf(), // (optional) - limit = 1, // (optional) - offset = 0, // (optional) -) \ No newline at end of file diff --git a/docs/examples/1.6.x/client-apple/examples/account/update-mfa-challenge.md b/docs/examples/1.6.x/client-apple/examples/account/update-mfa-challenge.md index a237537ae3..1c5874f784 100644 --- a/docs/examples/1.6.x/client-apple/examples/account/update-mfa-challenge.md +++ b/docs/examples/1.6.x/client-apple/examples/account/update-mfa-challenge.md @@ -6,7 +6,7 @@ let client = Client() let account = Account(client) -let result = try await account.updateMfaChallenge( +let session = try await account.updateMfaChallenge( challengeId: "", otp: "" ) diff --git a/docs/examples/1.6.x/client-apple/examples/functions/get-deployment-download.md b/docs/examples/1.6.x/client-apple/examples/functions/get-deployment-download.md deleted file mode 100644 index 0e6659969c..0000000000 --- a/docs/examples/1.6.x/client-apple/examples/functions/get-deployment-download.md +++ /dev/null @@ -1,13 +0,0 @@ -import Appwrite - -let client = Client() - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - -let functions = Functions(client) - -let bytes = try await functions.getDeploymentDownload( - functionId: "", - deploymentId: "" -) - diff --git a/docs/examples/1.6.x/client-apple/examples/functions/get-template.md b/docs/examples/1.6.x/client-apple/examples/functions/get-template.md deleted file mode 100644 index bc7a9a3aef..0000000000 --- a/docs/examples/1.6.x/client-apple/examples/functions/get-template.md +++ /dev/null @@ -1,12 +0,0 @@ -import Appwrite - -let client = Client() - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - -let functions = Functions(client) - -let templateFunction = try await functions.getTemplate( - templateId: "" -) - diff --git a/docs/examples/1.6.x/client-apple/examples/functions/list-templates.md b/docs/examples/1.6.x/client-apple/examples/functions/list-templates.md deleted file mode 100644 index d0090ab803..0000000000 --- a/docs/examples/1.6.x/client-apple/examples/functions/list-templates.md +++ /dev/null @@ -1,15 +0,0 @@ -import Appwrite - -let client = Client() - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - -let functions = Functions(client) - -let templateFunctionList = try await functions.listTemplates( - runtimes: [], // optional - useCases: [], // optional - limit: 1, // optional - offset: 0 // optional -) - diff --git a/docs/examples/1.6.x/client-flutter/examples/account/update-mfa-challenge.md b/docs/examples/1.6.x/client-flutter/examples/account/update-mfa-challenge.md index c8e1af7e90..bbe7c03470 100644 --- a/docs/examples/1.6.x/client-flutter/examples/account/update-mfa-challenge.md +++ b/docs/examples/1.6.x/client-flutter/examples/account/update-mfa-challenge.md @@ -6,7 +6,7 @@ Client client = Client() Account account = Account(client); - result = await account.updateMfaChallenge( +Session result = await account.updateMfaChallenge( challengeId: '', otp: '', ); diff --git a/docs/examples/1.6.x/client-flutter/examples/functions/get-deployment-download.md b/docs/examples/1.6.x/client-flutter/examples/functions/get-deployment-download.md deleted file mode 100644 index bb0ee903df..0000000000 --- a/docs/examples/1.6.x/client-flutter/examples/functions/get-deployment-download.md +++ /dev/null @@ -1,29 +0,0 @@ -import 'package:appwrite/appwrite.dart'; - -Client client = Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -Functions functions = Functions(client); - -// Downloading file -UInt8List bytes = await functions.getDeploymentDownload( - functionId: '', - deploymentId: '', -) - -final file = File('path_to_file/filename.ext'); -file.writeAsBytesSync(bytes); - -// Displaying image preview -FutureBuilder( - future: functions.getDeploymentDownload( - functionId:'' , - deploymentId:'' , -), // Works for both public file and private file, for private files you need to be logged in - builder: (context, snapshot) { - return snapshot.hasData && snapshot.data != null - ? Image.memory(snapshot.data) - : CircularProgressIndicator(); - } -); diff --git a/docs/examples/1.6.x/client-flutter/examples/functions/get-template.md b/docs/examples/1.6.x/client-flutter/examples/functions/get-template.md deleted file mode 100644 index 560a4d94ed..0000000000 --- a/docs/examples/1.6.x/client-flutter/examples/functions/get-template.md +++ /dev/null @@ -1,11 +0,0 @@ -import 'package:appwrite/appwrite.dart'; - -Client client = Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -Functions functions = Functions(client); - -TemplateFunction result = await functions.getTemplate( - templateId: '', -); diff --git a/docs/examples/1.6.x/client-flutter/examples/functions/list-templates.md b/docs/examples/1.6.x/client-flutter/examples/functions/list-templates.md deleted file mode 100644 index d3d8c7aa4c..0000000000 --- a/docs/examples/1.6.x/client-flutter/examples/functions/list-templates.md +++ /dev/null @@ -1,14 +0,0 @@ -import 'package:appwrite/appwrite.dart'; - -Client client = Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -Functions functions = Functions(client); - -TemplateFunctionList result = await functions.listTemplates( - runtimes: [], // optional - useCases: [], // optional - limit: 1, // optional - offset: 0, // optional -); diff --git a/docs/examples/1.6.x/client-graphql/examples/account/update-mfa-challenge.md b/docs/examples/1.6.x/client-graphql/examples/account/update-mfa-challenge.md index 8237431bcf..0bcec2157f 100644 --- a/docs/examples/1.6.x/client-graphql/examples/account/update-mfa-challenge.md +++ b/docs/examples/1.6.x/client-graphql/examples/account/update-mfa-challenge.md @@ -3,6 +3,34 @@ mutation { challengeId: "", otp: "" ) { - status + _id + _createdAt + _updatedAt + userId + expire + provider + providerUid + providerAccessToken + providerAccessTokenExpiry + providerRefreshToken + ip + osCode + osName + osVersion + clientType + clientCode + clientName + clientVersion + clientEngine + clientEngineVersion + deviceName + deviceBrand + deviceModel + countryCode + countryName + current + factors + secret + mfaUpdatedAt } } diff --git a/docs/examples/1.6.x/client-graphql/examples/functions/get-deployment-download.md b/docs/examples/1.6.x/client-graphql/examples/functions/get-deployment-download.md deleted file mode 100644 index 396047bb94..0000000000 --- a/docs/examples/1.6.x/client-graphql/examples/functions/get-deployment-download.md +++ /dev/null @@ -1,8 +0,0 @@ -query { - functionsGetDeploymentDownload( - functionId: "", - deploymentId: "" - ) { - status - } -} diff --git a/docs/examples/1.6.x/client-graphql/examples/functions/get-template.md b/docs/examples/1.6.x/client-graphql/examples/functions/get-template.md deleted file mode 100644 index 44a466de27..0000000000 --- a/docs/examples/1.6.x/client-graphql/examples/functions/get-template.md +++ /dev/null @@ -1,35 +0,0 @@ -query { - functionsGetTemplate( - templateId: "" - ) { - icon - id - name - tagline - permissions - events - cron - timeout - useCases - runtimes { - name - commands - entrypoint - providerRootDirectory - } - instructions - vcsProvider - providerRepositoryId - providerOwner - providerVersion - variables { - name - description - value - placeholder - required - type - } - scopes - } -} diff --git a/docs/examples/1.6.x/client-graphql/examples/functions/list-templates.md b/docs/examples/1.6.x/client-graphql/examples/functions/list-templates.md deleted file mode 100644 index cb289ddabb..0000000000 --- a/docs/examples/1.6.x/client-graphql/examples/functions/list-templates.md +++ /dev/null @@ -1,41 +0,0 @@ -query { - functionsListTemplates( - runtimes: [], - useCases: [], - limit: 1, - offset: 0 - ) { - total - templates { - icon - id - name - tagline - permissions - events - cron - timeout - useCases - runtimes { - name - commands - entrypoint - providerRootDirectory - } - instructions - vcsProvider - providerRepositoryId - providerOwner - providerVersion - variables { - name - description - value - placeholder - required - type - } - scopes - } - } -} diff --git a/docs/examples/1.6.x/client-react-native/examples/functions/get-deployment-download.md b/docs/examples/1.6.x/client-react-native/examples/functions/get-deployment-download.md deleted file mode 100644 index 7eac2b753f..0000000000 --- a/docs/examples/1.6.x/client-react-native/examples/functions/get-deployment-download.md +++ /dev/null @@ -1,14 +0,0 @@ -import { Client, Functions } from "react-native-appwrite"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const functions = new Functions(client); - -const result = functions.getDeploymentDownload( - '', // functionId - '' // deploymentId -); - -console.log(result); diff --git a/docs/examples/1.6.x/client-react-native/examples/functions/get-template.md b/docs/examples/1.6.x/client-react-native/examples/functions/get-template.md deleted file mode 100644 index ea756323dc..0000000000 --- a/docs/examples/1.6.x/client-react-native/examples/functions/get-template.md +++ /dev/null @@ -1,13 +0,0 @@ -import { Client, Functions } from "react-native-appwrite"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const functions = new Functions(client); - -const result = await functions.getTemplate( - '' // templateId -); - -console.log(result); diff --git a/docs/examples/1.6.x/client-react-native/examples/functions/list-templates.md b/docs/examples/1.6.x/client-react-native/examples/functions/list-templates.md deleted file mode 100644 index 3811dd8e40..0000000000 --- a/docs/examples/1.6.x/client-react-native/examples/functions/list-templates.md +++ /dev/null @@ -1,16 +0,0 @@ -import { Client, Functions } from "react-native-appwrite"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const functions = new Functions(client); - -const result = await functions.listTemplates( - [], // runtimes (optional) - [], // useCases (optional) - 1, // limit (optional) - 0 // offset (optional) -); - -console.log(result); diff --git a/docs/examples/1.6.x/client-rest/examples/functions/get-deployment-download.md b/docs/examples/1.6.x/client-rest/examples/functions/get-deployment-download.md deleted file mode 100644 index 0f6166b77f..0000000000 --- a/docs/examples/1.6.x/client-rest/examples/functions/get-deployment-download.md +++ /dev/null @@ -1,8 +0,0 @@ -GET /v1/functions/{functionId}/deployments/{deploymentId}/download HTTP/1.1 -Host: cloud.appwrite.io -Content-Type: application/json -X-Appwrite-Response-Format: 1.6.0 -X-Appwrite-Project: -X-Appwrite-Session: -X-Appwrite-JWT: - diff --git a/docs/examples/1.6.x/client-rest/examples/functions/get-template.md b/docs/examples/1.6.x/client-rest/examples/functions/get-template.md deleted file mode 100644 index d4e1f95cf5..0000000000 --- a/docs/examples/1.6.x/client-rest/examples/functions/get-template.md +++ /dev/null @@ -1,6 +0,0 @@ -GET /v1/functions/templates/{templateId} HTTP/1.1 -Host: cloud.appwrite.io -Content-Type: application/json -X-Appwrite-Response-Format: 1.6.0 -X-Appwrite-Project: - diff --git a/docs/examples/1.6.x/client-rest/examples/functions/list-templates.md b/docs/examples/1.6.x/client-rest/examples/functions/list-templates.md deleted file mode 100644 index b671bedebf..0000000000 --- a/docs/examples/1.6.x/client-rest/examples/functions/list-templates.md +++ /dev/null @@ -1,6 +0,0 @@ -GET /v1/functions/templates HTTP/1.1 -Host: cloud.appwrite.io -Content-Type: application/json -X-Appwrite-Response-Format: 1.6.0 -X-Appwrite-Project: - diff --git a/docs/examples/1.6.x/client-web/examples/functions/get-deployment-download.md b/docs/examples/1.6.x/client-web/examples/functions/get-deployment-download.md deleted file mode 100644 index 62e3ee6551..0000000000 --- a/docs/examples/1.6.x/client-web/examples/functions/get-deployment-download.md +++ /dev/null @@ -1,14 +0,0 @@ -import { Client, Functions } from "appwrite"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const functions = new Functions(client); - -const result = functions.getDeploymentDownload( - '', // functionId - '' // deploymentId -); - -console.log(result); diff --git a/docs/examples/1.6.x/client-web/examples/functions/get-template.md b/docs/examples/1.6.x/client-web/examples/functions/get-template.md deleted file mode 100644 index dd2d20284e..0000000000 --- a/docs/examples/1.6.x/client-web/examples/functions/get-template.md +++ /dev/null @@ -1,13 +0,0 @@ -import { Client, Functions } from "appwrite"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const functions = new Functions(client); - -const result = await functions.getTemplate( - '' // templateId -); - -console.log(result); diff --git a/docs/examples/1.6.x/client-web/examples/functions/list-templates.md b/docs/examples/1.6.x/client-web/examples/functions/list-templates.md deleted file mode 100644 index 141c56322b..0000000000 --- a/docs/examples/1.6.x/client-web/examples/functions/list-templates.md +++ /dev/null @@ -1,16 +0,0 @@ -import { Client, Functions } from "appwrite"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const functions = new Functions(client); - -const result = await functions.listTemplates( - [], // runtimes (optional) - [], // useCases (optional) - 1, // limit (optional) - 0 // offset (optional) -); - -console.log(result); diff --git a/docs/examples/1.6.x/console-cli/examples/functions/download-deployment.md b/docs/examples/1.6.x/console-cli/examples/functions/download-deployment.md deleted file mode 100644 index aa440b5145..0000000000 --- a/docs/examples/1.6.x/console-cli/examples/functions/download-deployment.md +++ /dev/null @@ -1,3 +0,0 @@ -appwrite functions downloadDeployment \ - --functionId \ - --deploymentId diff --git a/docs/examples/1.6.x/console-cli/examples/functions/get-specifications.md b/docs/examples/1.6.x/console-cli/examples/functions/get-specifications.md deleted file mode 100644 index 4e612280e7..0000000000 --- a/docs/examples/1.6.x/console-cli/examples/functions/get-specifications.md +++ /dev/null @@ -1 +0,0 @@ -appwrite functions getSpecifications diff --git a/docs/examples/1.6.x/console-cli/examples/migrations/create-firebase-o-auth-migration.md b/docs/examples/1.6.x/console-cli/examples/migrations/create-firebase-o-auth-migration.md deleted file mode 100644 index 207becba03..0000000000 --- a/docs/examples/1.6.x/console-cli/examples/migrations/create-firebase-o-auth-migration.md +++ /dev/null @@ -1,3 +0,0 @@ -appwrite migrations createFirebaseOAuthMigration \ - --resources one two three \ - --projectId diff --git a/docs/examples/1.6.x/console-cli/examples/migrations/delete-firebase-auth.md b/docs/examples/1.6.x/console-cli/examples/migrations/delete-firebase-auth.md deleted file mode 100644 index d58abd9878..0000000000 --- a/docs/examples/1.6.x/console-cli/examples/migrations/delete-firebase-auth.md +++ /dev/null @@ -1 +0,0 @@ -appwrite migrations deleteFirebaseAuth diff --git a/docs/examples/1.6.x/console-cli/examples/migrations/get-firebase-report-o-auth.md b/docs/examples/1.6.x/console-cli/examples/migrations/get-firebase-report-o-auth.md deleted file mode 100644 index 0bdbf3e435..0000000000 --- a/docs/examples/1.6.x/console-cli/examples/migrations/get-firebase-report-o-auth.md +++ /dev/null @@ -1,3 +0,0 @@ -appwrite migrations getFirebaseReportOAuth \ - --resources one two three \ - --projectId diff --git a/docs/examples/1.6.x/console-cli/examples/migrations/list-firebase-projects.md b/docs/examples/1.6.x/console-cli/examples/migrations/list-firebase-projects.md deleted file mode 100644 index a0e59713eb..0000000000 --- a/docs/examples/1.6.x/console-cli/examples/migrations/list-firebase-projects.md +++ /dev/null @@ -1 +0,0 @@ -appwrite migrations listFirebaseProjects diff --git a/docs/examples/1.6.x/console-web/examples/functions/download-deployment.md b/docs/examples/1.6.x/console-web/examples/functions/download-deployment.md deleted file mode 100644 index 8115c7e130..0000000000 --- a/docs/examples/1.6.x/console-web/examples/functions/download-deployment.md +++ /dev/null @@ -1,14 +0,0 @@ -import { Client, Functions } from "@appwrite.io/console"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const functions = new Functions(client); - -const result = functions.downloadDeployment( - '', // functionId - '' // deploymentId -); - -console.log(result); diff --git a/docs/examples/1.6.x/console-web/examples/functions/get-specifications.md b/docs/examples/1.6.x/console-web/examples/functions/get-specifications.md deleted file mode 100644 index c5cb3fc043..0000000000 --- a/docs/examples/1.6.x/console-web/examples/functions/get-specifications.md +++ /dev/null @@ -1,11 +0,0 @@ -import { Client, Functions } from "@appwrite.io/console"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const functions = new Functions(client); - -const result = await functions.getSpecifications(); - -console.log(result); diff --git a/docs/examples/1.6.x/console-web/examples/migrations/create-firebase-o-auth-migration.md b/docs/examples/1.6.x/console-web/examples/migrations/create-firebase-o-auth-migration.md deleted file mode 100644 index bf57d427e9..0000000000 --- a/docs/examples/1.6.x/console-web/examples/migrations/create-firebase-o-auth-migration.md +++ /dev/null @@ -1,14 +0,0 @@ -import { Client, Migrations } from "@appwrite.io/console"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const migrations = new Migrations(client); - -const result = await migrations.createFirebaseOAuthMigration( - [], // resources - '' // projectId -); - -console.log(result); diff --git a/docs/examples/1.6.x/console-web/examples/migrations/delete-firebase-auth.md b/docs/examples/1.6.x/console-web/examples/migrations/delete-firebase-auth.md deleted file mode 100644 index 8c7683b46a..0000000000 --- a/docs/examples/1.6.x/console-web/examples/migrations/delete-firebase-auth.md +++ /dev/null @@ -1,11 +0,0 @@ -import { Client, Migrations } from "@appwrite.io/console"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const migrations = new Migrations(client); - -const result = await migrations.deleteFirebaseAuth(); - -console.log(result); diff --git a/docs/examples/1.6.x/console-web/examples/migrations/get-firebase-report-o-auth.md b/docs/examples/1.6.x/console-web/examples/migrations/get-firebase-report-o-auth.md deleted file mode 100644 index 055ddfba7c..0000000000 --- a/docs/examples/1.6.x/console-web/examples/migrations/get-firebase-report-o-auth.md +++ /dev/null @@ -1,14 +0,0 @@ -import { Client, Migrations } from "@appwrite.io/console"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const migrations = new Migrations(client); - -const result = await migrations.getFirebaseReportOAuth( - [], // resources - '' // projectId -); - -console.log(result); diff --git a/docs/examples/1.6.x/console-web/examples/migrations/list-firebase-projects.md b/docs/examples/1.6.x/console-web/examples/migrations/list-firebase-projects.md deleted file mode 100644 index 833c05fe24..0000000000 --- a/docs/examples/1.6.x/console-web/examples/migrations/list-firebase-projects.md +++ /dev/null @@ -1,11 +0,0 @@ -import { Client, Migrations } from "@appwrite.io/console"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const migrations = new Migrations(client); - -const result = await migrations.listFirebaseProjects(); - -console.log(result); diff --git a/docs/examples/1.6.x/server-dart/examples/account/update-mfa-challenge.md b/docs/examples/1.6.x/server-dart/examples/account/update-mfa-challenge.md index fd64c61cf9..2843d2f1b4 100644 --- a/docs/examples/1.6.x/server-dart/examples/account/update-mfa-challenge.md +++ b/docs/examples/1.6.x/server-dart/examples/account/update-mfa-challenge.md @@ -7,7 +7,7 @@ Client client = Client() Account account = Account(client); - result = await account.updateMfaChallenge( +Session result = await account.updateMfaChallenge( challengeId: '', otp: '', ); diff --git a/docs/examples/1.6.x/server-dart/examples/functions/download-deployment.md b/docs/examples/1.6.x/server-dart/examples/functions/download-deployment.md deleted file mode 100644 index 981de76756..0000000000 --- a/docs/examples/1.6.x/server-dart/examples/functions/download-deployment.md +++ /dev/null @@ -1,13 +0,0 @@ -import 'package:dart_appwrite/dart_appwrite.dart'; - -Client client = Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject('') // Your project ID - .setKey(''); // Your secret API key - -Functions functions = Functions(client); - -UInt8List result = await functions.downloadDeployment( - functionId: '', - deploymentId: '', -); diff --git a/docs/examples/1.6.x/server-dart/examples/functions/get-template.md b/docs/examples/1.6.x/server-dart/examples/functions/get-template.md deleted file mode 100644 index 881dac6782..0000000000 --- a/docs/examples/1.6.x/server-dart/examples/functions/get-template.md +++ /dev/null @@ -1,11 +0,0 @@ -import 'package:dart_appwrite/dart_appwrite.dart'; - -Client client = Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -Functions functions = Functions(client); - -TemplateFunction result = await functions.getTemplate( - templateId: '', -); diff --git a/docs/examples/1.6.x/server-dart/examples/functions/list-templates.md b/docs/examples/1.6.x/server-dart/examples/functions/list-templates.md deleted file mode 100644 index 7bef5106ed..0000000000 --- a/docs/examples/1.6.x/server-dart/examples/functions/list-templates.md +++ /dev/null @@ -1,14 +0,0 @@ -import 'package:dart_appwrite/dart_appwrite.dart'; - -Client client = Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -Functions functions = Functions(client); - -TemplateFunctionList result = await functions.listTemplates( - runtimes: [], // (optional) - useCases: [], // (optional) - limit: 1, // (optional) - offset: 0, // (optional) -); diff --git a/docs/examples/1.6.x/server-deno/examples/functions/download-deployment.md b/docs/examples/1.6.x/server-deno/examples/functions/download-deployment.md deleted file mode 100644 index bb0a4e71b4..0000000000 --- a/docs/examples/1.6.x/server-deno/examples/functions/download-deployment.md +++ /dev/null @@ -1,13 +0,0 @@ -import { Client, Functions } from "https://deno.land/x/appwrite/mod.ts"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject('') // Your project ID - .setKey(''); // Your secret API key - -const functions = new Functions(client); - -const result = functions.downloadDeployment( - '', // functionId - '' // deploymentId -); diff --git a/docs/examples/1.6.x/server-deno/examples/functions/get-template.md b/docs/examples/1.6.x/server-deno/examples/functions/get-template.md deleted file mode 100644 index 7a58c95a34..0000000000 --- a/docs/examples/1.6.x/server-deno/examples/functions/get-template.md +++ /dev/null @@ -1,11 +0,0 @@ -import { Client, Functions } from "https://deno.land/x/appwrite/mod.ts"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const functions = new Functions(client); - -const response = await functions.getTemplate( - '' // templateId -); diff --git a/docs/examples/1.6.x/server-deno/examples/functions/list-templates.md b/docs/examples/1.6.x/server-deno/examples/functions/list-templates.md deleted file mode 100644 index 06203e404b..0000000000 --- a/docs/examples/1.6.x/server-deno/examples/functions/list-templates.md +++ /dev/null @@ -1,14 +0,0 @@ -import { Client, Functions } from "https://deno.land/x/appwrite/mod.ts"; - -const client = new Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const functions = new Functions(client); - -const response = await functions.listTemplates( - [], // runtimes (optional) - [], // useCases (optional) - 1, // limit (optional) - 0 // offset (optional) -); diff --git a/docs/examples/1.6.x/server-dotnet/examples/account/update-mfa-challenge.md b/docs/examples/1.6.x/server-dotnet/examples/account/update-mfa-challenge.md index edf863a36e..ec32f8c900 100644 --- a/docs/examples/1.6.x/server-dotnet/examples/account/update-mfa-challenge.md +++ b/docs/examples/1.6.x/server-dotnet/examples/account/update-mfa-challenge.md @@ -9,7 +9,7 @@ Client client = new Client() Account account = new Account(client); - result = await account.UpdateMfaChallenge( +Session result = await account.UpdateMfaChallenge( challengeId: "", otp: "" ); \ No newline at end of file diff --git a/docs/examples/1.6.x/server-dotnet/examples/functions/download-deployment.md b/docs/examples/1.6.x/server-dotnet/examples/functions/download-deployment.md deleted file mode 100644 index 83f56a8e14..0000000000 --- a/docs/examples/1.6.x/server-dotnet/examples/functions/download-deployment.md +++ /dev/null @@ -1,15 +0,0 @@ -using Appwrite; -using Appwrite.Models; -using Appwrite.Services; - -Client client = new Client() - .SetEndPoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .SetProject("") // Your project ID - .SetKey(""); // Your secret API key - -Functions functions = new Functions(client); - -byte[] result = await functions.DownloadDeployment( - functionId: "", - deploymentId: "" -); \ No newline at end of file diff --git a/docs/examples/1.6.x/server-dotnet/examples/functions/get-template.md b/docs/examples/1.6.x/server-dotnet/examples/functions/get-template.md deleted file mode 100644 index f57d61a024..0000000000 --- a/docs/examples/1.6.x/server-dotnet/examples/functions/get-template.md +++ /dev/null @@ -1,13 +0,0 @@ -using Appwrite; -using Appwrite.Models; -using Appwrite.Services; - -Client client = new Client() - .SetEndPoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .SetProject(""); // Your project ID - -Functions functions = new Functions(client); - -TemplateFunction result = await functions.GetTemplate( - templateId: "" -); \ No newline at end of file diff --git a/docs/examples/1.6.x/server-dotnet/examples/functions/list-templates.md b/docs/examples/1.6.x/server-dotnet/examples/functions/list-templates.md deleted file mode 100644 index f76bd5fa9c..0000000000 --- a/docs/examples/1.6.x/server-dotnet/examples/functions/list-templates.md +++ /dev/null @@ -1,16 +0,0 @@ -using Appwrite; -using Appwrite.Models; -using Appwrite.Services; - -Client client = new Client() - .SetEndPoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .SetProject(""); // Your project ID - -Functions functions = new Functions(client); - -TemplateFunctionList result = await functions.ListTemplates( - runtimes: new List(), // optional - useCases: new List(), // optional - limit: 1, // optional - offset: 0 // optional -); \ No newline at end of file diff --git a/docs/examples/1.6.x/server-go/examples/functions/download-deployment.md b/docs/examples/1.6.x/server-go/examples/functions/download-deployment.md deleted file mode 100644 index 37d2149541..0000000000 --- a/docs/examples/1.6.x/server-go/examples/functions/download-deployment.md +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "fmt" - "github.com/appwrite/sdk-for-go/client" - "github.com/appwrite/sdk-for-go/functions" -) - -func main() { - client := client.NewClient() - - client.SetEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - client.SetProject("") // Your project ID - client.SetKey("") // Your secret API key - - service := functions.NewFunctions(client) - response, error := service.DownloadDeployment( - "", - "", - ) - - if error != nil { - panic(error) - } - - fmt.Println(response) -} diff --git a/docs/examples/1.6.x/server-go/examples/functions/get-template.md b/docs/examples/1.6.x/server-go/examples/functions/get-template.md deleted file mode 100644 index fe4e1debb9..0000000000 --- a/docs/examples/1.6.x/server-go/examples/functions/get-template.md +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "fmt" - "github.com/appwrite/sdk-for-go/client" - "github.com/appwrite/sdk-for-go/functions" -) - -func main() { - client := client.NewClient() - - client.SetEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - client.SetProject("") // Your project ID - - service := functions.NewFunctions(client) - response, error := service.GetTemplate( - "", - ) - - if error != nil { - panic(error) - } - - fmt.Println(response) -} diff --git a/docs/examples/1.6.x/server-go/examples/functions/list-templates.md b/docs/examples/1.6.x/server-go/examples/functions/list-templates.md deleted file mode 100644 index 19edf6d360..0000000000 --- a/docs/examples/1.6.x/server-go/examples/functions/list-templates.md +++ /dev/null @@ -1,28 +0,0 @@ -package main - -import ( - "fmt" - "github.com/appwrite/sdk-for-go/client" - "github.com/appwrite/sdk-for-go/functions" -) - -func main() { - client := client.NewClient() - - client.SetEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - client.SetProject("") // Your project ID - - service := functions.NewFunctions(client) - response, error := service.ListTemplates( - functions.WithListTemplatesRuntimes([]interface{}{}), - functions.WithListTemplatesUseCases([]interface{}{}), - functions.WithListTemplatesLimit(1), - functions.WithListTemplatesOffset(0), - ) - - if error != nil { - panic(error) - } - - fmt.Println(response) -} diff --git a/docs/examples/1.6.x/server-graphql/examples/account/update-mfa-challenge.md b/docs/examples/1.6.x/server-graphql/examples/account/update-mfa-challenge.md index 8237431bcf..0bcec2157f 100644 --- a/docs/examples/1.6.x/server-graphql/examples/account/update-mfa-challenge.md +++ b/docs/examples/1.6.x/server-graphql/examples/account/update-mfa-challenge.md @@ -3,6 +3,34 @@ mutation { challengeId: "", otp: "" ) { - status + _id + _createdAt + _updatedAt + userId + expire + provider + providerUid + providerAccessToken + providerAccessTokenExpiry + providerRefreshToken + ip + osCode + osName + osVersion + clientType + clientCode + clientName + clientVersion + clientEngine + clientEngineVersion + deviceName + deviceBrand + deviceModel + countryCode + countryName + current + factors + secret + mfaUpdatedAt } } diff --git a/docs/examples/1.6.x/server-graphql/examples/functions/download-deployment.md b/docs/examples/1.6.x/server-graphql/examples/functions/download-deployment.md deleted file mode 100644 index f791338765..0000000000 --- a/docs/examples/1.6.x/server-graphql/examples/functions/download-deployment.md +++ /dev/null @@ -1,8 +0,0 @@ -query { - functionsDownloadDeployment( - functionId: "", - deploymentId: "" - ) { - status - } -} diff --git a/docs/examples/1.6.x/server-graphql/examples/functions/get-template.md b/docs/examples/1.6.x/server-graphql/examples/functions/get-template.md deleted file mode 100644 index 44a466de27..0000000000 --- a/docs/examples/1.6.x/server-graphql/examples/functions/get-template.md +++ /dev/null @@ -1,35 +0,0 @@ -query { - functionsGetTemplate( - templateId: "" - ) { - icon - id - name - tagline - permissions - events - cron - timeout - useCases - runtimes { - name - commands - entrypoint - providerRootDirectory - } - instructions - vcsProvider - providerRepositoryId - providerOwner - providerVersion - variables { - name - description - value - placeholder - required - type - } - scopes - } -} diff --git a/docs/examples/1.6.x/server-graphql/examples/functions/list-templates.md b/docs/examples/1.6.x/server-graphql/examples/functions/list-templates.md deleted file mode 100644 index cb289ddabb..0000000000 --- a/docs/examples/1.6.x/server-graphql/examples/functions/list-templates.md +++ /dev/null @@ -1,41 +0,0 @@ -query { - functionsListTemplates( - runtimes: [], - useCases: [], - limit: 1, - offset: 0 - ) { - total - templates { - icon - id - name - tagline - permissions - events - cron - timeout - useCases - runtimes { - name - commands - entrypoint - providerRootDirectory - } - instructions - vcsProvider - providerRepositoryId - providerOwner - providerVersion - variables { - name - description - value - placeholder - required - type - } - scopes - } - } -} diff --git a/docs/examples/1.6.x/server-graphql/examples/users/delete-mfa-authenticator.md b/docs/examples/1.6.x/server-graphql/examples/users/delete-mfa-authenticator.md index 26c9594a53..43f73404f0 100644 --- a/docs/examples/1.6.x/server-graphql/examples/users/delete-mfa-authenticator.md +++ b/docs/examples/1.6.x/server-graphql/examples/users/delete-mfa-authenticator.md @@ -3,36 +3,6 @@ mutation { userId: "", type: "totp" ) { - _id - _createdAt - _updatedAt - name - password - hash - hashOptions - registration status - labels - passwordUpdate - email - phone - emailVerification - phoneVerification - mfa - prefs { - data - } - targets { - _id - _createdAt - _updatedAt - name - userId - providerId - providerType - identifier - expired - } - accessedAt } } diff --git a/docs/examples/1.6.x/server-kotlin/java/functions/download-deployment.md b/docs/examples/1.6.x/server-kotlin/java/functions/download-deployment.md deleted file mode 100644 index 77e4809379..0000000000 --- a/docs/examples/1.6.x/server-kotlin/java/functions/download-deployment.md +++ /dev/null @@ -1,24 +0,0 @@ -import io.appwrite.Client; -import io.appwrite.coroutines.CoroutineCallback; -import io.appwrite.services.Functions; - -Client client = new Client() - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - .setKey(""); // Your secret API key - -Functions functions = new Functions(client); - -functions.downloadDeployment( - "", // functionId - "", // deploymentId - new CoroutineCallback<>((result, error) -> { - if (error != null) { - error.printStackTrace(); - return; - } - - System.out.println(result); - }) -); - diff --git a/docs/examples/1.6.x/server-kotlin/java/functions/get-template.md b/docs/examples/1.6.x/server-kotlin/java/functions/get-template.md deleted file mode 100644 index 1521fa47c1..0000000000 --- a/docs/examples/1.6.x/server-kotlin/java/functions/get-template.md +++ /dev/null @@ -1,22 +0,0 @@ -import io.appwrite.Client; -import io.appwrite.coroutines.CoroutineCallback; -import io.appwrite.services.Functions; - -Client client = new Client() - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject(""); // Your project ID - -Functions functions = new Functions(client); - -functions.getTemplate( - "", // templateId - new CoroutineCallback<>((result, error) -> { - if (error != null) { - error.printStackTrace(); - return; - } - - System.out.println(result); - }) -); - diff --git a/docs/examples/1.6.x/server-kotlin/java/functions/list-templates.md b/docs/examples/1.6.x/server-kotlin/java/functions/list-templates.md deleted file mode 100644 index e88e19124f..0000000000 --- a/docs/examples/1.6.x/server-kotlin/java/functions/list-templates.md +++ /dev/null @@ -1,25 +0,0 @@ -import io.appwrite.Client; -import io.appwrite.coroutines.CoroutineCallback; -import io.appwrite.services.Functions; - -Client client = new Client() - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject(""); // Your project ID - -Functions functions = new Functions(client); - -functions.listTemplates( - listOf(), // runtimes (optional) - listOf(), // useCases (optional) - 1, // limit (optional) - 0, // offset (optional) - new CoroutineCallback<>((result, error) -> { - if (error != null) { - error.printStackTrace(); - return; - } - - System.out.println(result); - }) -); - diff --git a/docs/examples/1.6.x/server-kotlin/kotlin/functions/download-deployment.md b/docs/examples/1.6.x/server-kotlin/kotlin/functions/download-deployment.md deleted file mode 100644 index 0ae36b314f..0000000000 --- a/docs/examples/1.6.x/server-kotlin/kotlin/functions/download-deployment.md +++ /dev/null @@ -1,15 +0,0 @@ -import io.appwrite.Client -import io.appwrite.coroutines.CoroutineCallback -import io.appwrite.services.Functions - -val client = Client() - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - .setKey("") // Your secret API key - -val functions = Functions(client) - -val result = functions.downloadDeployment( - functionId = "", - deploymentId = "" -) diff --git a/docs/examples/1.6.x/server-kotlin/kotlin/functions/get-template.md b/docs/examples/1.6.x/server-kotlin/kotlin/functions/get-template.md deleted file mode 100644 index 53d838f7e6..0000000000 --- a/docs/examples/1.6.x/server-kotlin/kotlin/functions/get-template.md +++ /dev/null @@ -1,13 +0,0 @@ -import io.appwrite.Client -import io.appwrite.coroutines.CoroutineCallback -import io.appwrite.services.Functions - -val client = Client() - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - -val functions = Functions(client) - -val response = functions.getTemplate( - templateId = "" -) diff --git a/docs/examples/1.6.x/server-kotlin/kotlin/functions/list-templates.md b/docs/examples/1.6.x/server-kotlin/kotlin/functions/list-templates.md deleted file mode 100644 index 37dc89ce89..0000000000 --- a/docs/examples/1.6.x/server-kotlin/kotlin/functions/list-templates.md +++ /dev/null @@ -1,16 +0,0 @@ -import io.appwrite.Client -import io.appwrite.coroutines.CoroutineCallback -import io.appwrite.services.Functions - -val client = Client() - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - -val functions = Functions(client) - -val response = functions.listTemplates( - runtimes = listOf(), // optional - useCases = listOf(), // optional - limit = 1, // optional - offset = 0 // optional -) diff --git a/docs/examples/1.6.x/server-nodejs/examples/functions/download-deployment.md b/docs/examples/1.6.x/server-nodejs/examples/functions/download-deployment.md deleted file mode 100644 index e0962db0ed..0000000000 --- a/docs/examples/1.6.x/server-nodejs/examples/functions/download-deployment.md +++ /dev/null @@ -1,13 +0,0 @@ -const sdk = require('node-appwrite'); - -const client = new sdk.Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject('') // Your project ID - .setKey(''); // Your secret API key - -const functions = new sdk.Functions(client); - -const result = await functions.downloadDeployment( - '', // functionId - '' // deploymentId -); diff --git a/docs/examples/1.6.x/server-nodejs/examples/functions/get-template.md b/docs/examples/1.6.x/server-nodejs/examples/functions/get-template.md deleted file mode 100644 index 36195db7f3..0000000000 --- a/docs/examples/1.6.x/server-nodejs/examples/functions/get-template.md +++ /dev/null @@ -1,11 +0,0 @@ -const sdk = require('node-appwrite'); - -const client = new sdk.Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const functions = new sdk.Functions(client); - -const result = await functions.getTemplate( - '' // templateId -); diff --git a/docs/examples/1.6.x/server-nodejs/examples/functions/list-templates.md b/docs/examples/1.6.x/server-nodejs/examples/functions/list-templates.md deleted file mode 100644 index 6f896cac97..0000000000 --- a/docs/examples/1.6.x/server-nodejs/examples/functions/list-templates.md +++ /dev/null @@ -1,14 +0,0 @@ -const sdk = require('node-appwrite'); - -const client = new sdk.Client() - .setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - .setProject(''); // Your project ID - -const functions = new sdk.Functions(client); - -const result = await functions.listTemplates( - [], // runtimes (optional) - [], // useCases (optional) - 1, // limit (optional) - 0 // offset (optional) -); diff --git a/docs/examples/1.6.x/server-php/examples/functions/download-deployment.md b/docs/examples/1.6.x/server-php/examples/functions/download-deployment.md deleted file mode 100644 index 83c8bb756f..0000000000 --- a/docs/examples/1.6.x/server-php/examples/functions/download-deployment.md +++ /dev/null @@ -1,16 +0,0 @@ -setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - ->setProject('') // Your project ID - ->setKey(''); // Your secret API key - -$functions = new Functions($client); - -$result = $functions->downloadDeployment( - functionId: '', - deploymentId: '' -); \ No newline at end of file diff --git a/docs/examples/1.6.x/server-php/examples/functions/get-template.md b/docs/examples/1.6.x/server-php/examples/functions/get-template.md deleted file mode 100644 index 421557dcd2..0000000000 --- a/docs/examples/1.6.x/server-php/examples/functions/get-template.md +++ /dev/null @@ -1,14 +0,0 @@ -setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - ->setProject(''); // Your project ID - -$functions = new Functions($client); - -$result = $functions->getTemplate( - templateId: '' -); \ No newline at end of file diff --git a/docs/examples/1.6.x/server-php/examples/functions/list-templates.md b/docs/examples/1.6.x/server-php/examples/functions/list-templates.md deleted file mode 100644 index a661903306..0000000000 --- a/docs/examples/1.6.x/server-php/examples/functions/list-templates.md +++ /dev/null @@ -1,17 +0,0 @@ -setEndpoint('https://cloud.appwrite.io/v1') // Your API Endpoint - ->setProject(''); // Your project ID - -$functions = new Functions($client); - -$result = $functions->listTemplates( - runtimes: [], // optional - useCases: [], // optional - limit: 1, // optional - offset: 0 // optional -); \ No newline at end of file diff --git a/docs/examples/1.6.x/server-python/examples/functions/download-deployment.md b/docs/examples/1.6.x/server-python/examples/functions/download-deployment.md deleted file mode 100644 index f9d395451e..0000000000 --- a/docs/examples/1.6.x/server-python/examples/functions/download-deployment.md +++ /dev/null @@ -1,13 +0,0 @@ -from appwrite.client import Client - -client = Client() -client.set_endpoint('https://cloud.appwrite.io/v1') # Your API Endpoint -client.set_project('') # Your project ID -client.set_key('') # Your secret API key - -functions = Functions(client) - -result = functions.download_deployment( - function_id = '', - deployment_id = '' -) diff --git a/docs/examples/1.6.x/server-python/examples/functions/get-template.md b/docs/examples/1.6.x/server-python/examples/functions/get-template.md deleted file mode 100644 index bea5b6ab03..0000000000 --- a/docs/examples/1.6.x/server-python/examples/functions/get-template.md +++ /dev/null @@ -1,11 +0,0 @@ -from appwrite.client import Client - -client = Client() -client.set_endpoint('https://cloud.appwrite.io/v1') # Your API Endpoint -client.set_project('') # Your project ID - -functions = Functions(client) - -result = functions.get_template( - template_id = '' -) diff --git a/docs/examples/1.6.x/server-python/examples/functions/list-templates.md b/docs/examples/1.6.x/server-python/examples/functions/list-templates.md deleted file mode 100644 index b3ee38aef7..0000000000 --- a/docs/examples/1.6.x/server-python/examples/functions/list-templates.md +++ /dev/null @@ -1,14 +0,0 @@ -from appwrite.client import Client - -client = Client() -client.set_endpoint('https://cloud.appwrite.io/v1') # Your API Endpoint -client.set_project('') # Your project ID - -functions = Functions(client) - -result = functions.list_templates( - runtimes = [], # optional - use_cases = [], # optional - limit = 1, # optional - offset = 0 # optional -) diff --git a/docs/examples/1.6.x/server-rest/examples/functions/download-deployment.md b/docs/examples/1.6.x/server-rest/examples/functions/download-deployment.md deleted file mode 100644 index ccd37283c9..0000000000 --- a/docs/examples/1.6.x/server-rest/examples/functions/download-deployment.md +++ /dev/null @@ -1,7 +0,0 @@ -GET /v1/functions/{functionId}/deployments/{deploymentId}/download HTTP/1.1 -Host: cloud.appwrite.io -Content-Type: application/json -X-Appwrite-Response-Format: 1.5.0 -X-Appwrite-Project: -X-Appwrite-Key: - diff --git a/docs/examples/1.6.x/server-rest/examples/functions/get-template.md b/docs/examples/1.6.x/server-rest/examples/functions/get-template.md deleted file mode 100644 index d4e1f95cf5..0000000000 --- a/docs/examples/1.6.x/server-rest/examples/functions/get-template.md +++ /dev/null @@ -1,6 +0,0 @@ -GET /v1/functions/templates/{templateId} HTTP/1.1 -Host: cloud.appwrite.io -Content-Type: application/json -X-Appwrite-Response-Format: 1.6.0 -X-Appwrite-Project: - diff --git a/docs/examples/1.6.x/server-rest/examples/functions/list-templates.md b/docs/examples/1.6.x/server-rest/examples/functions/list-templates.md deleted file mode 100644 index b671bedebf..0000000000 --- a/docs/examples/1.6.x/server-rest/examples/functions/list-templates.md +++ /dev/null @@ -1,6 +0,0 @@ -GET /v1/functions/templates HTTP/1.1 -Host: cloud.appwrite.io -Content-Type: application/json -X-Appwrite-Response-Format: 1.6.0 -X-Appwrite-Project: - diff --git a/docs/examples/1.6.x/server-ruby/examples/functions/download-deployment.md b/docs/examples/1.6.x/server-ruby/examples/functions/download-deployment.md deleted file mode 100644 index 748d0c9d01..0000000000 --- a/docs/examples/1.6.x/server-ruby/examples/functions/download-deployment.md +++ /dev/null @@ -1,15 +0,0 @@ -require 'appwrite' - -include Appwrite - -client = Client.new - .set_endpoint('https://cloud.appwrite.io/v1') # Your API Endpoint - .set_project('') # Your project ID - .set_key('') # Your secret API key - -functions = Functions.new(client) - -result = functions.download_deployment( - function_id: '', - deployment_id: '' -) diff --git a/docs/examples/1.6.x/server-ruby/examples/functions/get-template.md b/docs/examples/1.6.x/server-ruby/examples/functions/get-template.md deleted file mode 100644 index 5274880f67..0000000000 --- a/docs/examples/1.6.x/server-ruby/examples/functions/get-template.md +++ /dev/null @@ -1,13 +0,0 @@ -require 'appwrite' - -include Appwrite - -client = Client.new - .set_endpoint('https://cloud.appwrite.io/v1') # Your API Endpoint - .set_project('') # Your project ID - -functions = Functions.new(client) - -result = functions.get_template( - template_id: '' -) diff --git a/docs/examples/1.6.x/server-ruby/examples/functions/list-templates.md b/docs/examples/1.6.x/server-ruby/examples/functions/list-templates.md deleted file mode 100644 index 8bee6b0187..0000000000 --- a/docs/examples/1.6.x/server-ruby/examples/functions/list-templates.md +++ /dev/null @@ -1,16 +0,0 @@ -require 'appwrite' - -include Appwrite - -client = Client.new - .set_endpoint('https://cloud.appwrite.io/v1') # Your API Endpoint - .set_project('') # Your project ID - -functions = Functions.new(client) - -result = functions.list_templates( - runtimes: [], # optional - use_cases: [], # optional - limit: 1, # optional - offset: 0 # optional -) diff --git a/docs/examples/1.6.x/server-swift/examples/account/update-mfa-challenge.md b/docs/examples/1.6.x/server-swift/examples/account/update-mfa-challenge.md index eed3bfade7..fee76bf0dd 100644 --- a/docs/examples/1.6.x/server-swift/examples/account/update-mfa-challenge.md +++ b/docs/examples/1.6.x/server-swift/examples/account/update-mfa-challenge.md @@ -7,7 +7,7 @@ let client = Client() let account = Account(client) -let result = try await account.updateMfaChallenge( +let session = try await account.updateMfaChallenge( challengeId: "", otp: "" ) diff --git a/docs/examples/1.6.x/server-swift/examples/functions/download-deployment.md b/docs/examples/1.6.x/server-swift/examples/functions/download-deployment.md deleted file mode 100644 index 753d7058e6..0000000000 --- a/docs/examples/1.6.x/server-swift/examples/functions/download-deployment.md +++ /dev/null @@ -1,14 +0,0 @@ -import Appwrite - -let client = Client() - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - .setKey("") // Your secret API key - -let functions = Functions(client) - -let bytes = try await functions.downloadDeployment( - functionId: "", - deploymentId: "" -) - diff --git a/docs/examples/1.6.x/server-swift/examples/functions/get-template.md b/docs/examples/1.6.x/server-swift/examples/functions/get-template.md deleted file mode 100644 index bc7a9a3aef..0000000000 --- a/docs/examples/1.6.x/server-swift/examples/functions/get-template.md +++ /dev/null @@ -1,12 +0,0 @@ -import Appwrite - -let client = Client() - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - -let functions = Functions(client) - -let templateFunction = try await functions.getTemplate( - templateId: "" -) - diff --git a/docs/examples/1.6.x/server-swift/examples/functions/list-templates.md b/docs/examples/1.6.x/server-swift/examples/functions/list-templates.md deleted file mode 100644 index d0090ab803..0000000000 --- a/docs/examples/1.6.x/server-swift/examples/functions/list-templates.md +++ /dev/null @@ -1,15 +0,0 @@ -import Appwrite - -let client = Client() - .setEndpoint("https://cloud.appwrite.io/v1") // Your API Endpoint - .setProject("") // Your project ID - -let functions = Functions(client) - -let templateFunctionList = try await functions.listTemplates( - runtimes: [], // optional - useCases: [], // optional - limit: 1, // optional - offset: 0 // optional -) - diff --git a/docs/examples/1.6.x/server-swift/examples/users/delete-mfa-authenticator.md b/docs/examples/1.6.x/server-swift/examples/users/delete-mfa-authenticator.md index 902d0c904c..5f1d6a0eeb 100644 --- a/docs/examples/1.6.x/server-swift/examples/users/delete-mfa-authenticator.md +++ b/docs/examples/1.6.x/server-swift/examples/users/delete-mfa-authenticator.md @@ -8,7 +8,7 @@ let client = Client() let users = Users(client) -let user = try await users.deleteMfaAuthenticator( +let result = try await users.deleteMfaAuthenticator( userId: "", type: .totp ) From 556f7338c104b3bb01b484b7591b9ee9572d4020 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 12 Mar 2025 15:52:29 -0700 Subject: [PATCH 148/275] Bump appwrite version to 1.6.2 --- README-CN.md | 6 +++--- README.md | 6 +++--- app/init.php | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README-CN.md b/README-CN.md index c024491907..a81d99c3c3 100644 --- a/README-CN.md +++ b/README-CN.md @@ -72,7 +72,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.6.1 + appwrite/appwrite:1.6.2 ``` ### Windows @@ -84,7 +84,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.6.1 + appwrite/appwrite:1.6.2 ``` #### PowerShell @@ -94,7 +94,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.6.1 + appwrite/appwrite:1.6.2 ``` 运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index 5bccf4739f..c2b1151e36 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.6.1 + appwrite/appwrite:1.6.2 ``` ### Windows @@ -91,7 +91,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.6.1 + appwrite/appwrite:1.6.2 ``` #### PowerShell @@ -101,7 +101,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.6.1 + appwrite/appwrite:1.6.2 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/init.php b/app/init.php index 17c1a884f9..9a499412db 100644 --- a/app/init.php +++ b/app/init.php @@ -125,7 +125,7 @@ const APP_PROJECT_ACCESS = 24 * 60 * 60; // 24 hours const APP_FILE_ACCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 4318; -const APP_VERSION_STABLE = '1.6.1'; +const APP_VERSION_STABLE = '1.6.2'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; From e119bf6777effd3f18a627b4f141e460d60999bb Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 13 Mar 2025 05:30:09 +0000 Subject: [PATCH 149/275] chore: update changelogs --- docs/sdks/dart/CHANGELOG.md | 12 ++++++++++++ docs/sdks/flutter/CHANGELOG.md | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/sdks/dart/CHANGELOG.md b/docs/sdks/dart/CHANGELOG.md index a46f1dcf0f..c41413b9ec 100644 --- a/docs/sdks/dart/CHANGELOG.md +++ b/docs/sdks/dart/CHANGELOG.md @@ -1,3 +1,15 @@ +## 14.0.0 + +* Breaking changes: + * Changed the typing of `AppwriteException`'s response parameter from a `dynamic` object to an optional string (`?String`). + +## 13.0.0 + +* Fixed realtime pong response. +* Fixed issues with `chunkedUpload` method. +* Fixed type mismatch bug where `List` was incorrectly causing runtime type errors. +* Updated return type of `updateMfaChallenge()` from raw data to properly typed `models.Session` object. + ## 12.0.0 * Support for Appwrite 1.6 diff --git a/docs/sdks/flutter/CHANGELOG.md b/docs/sdks/flutter/CHANGELOG.md index abb8967f84..110b6d08c3 100644 --- a/docs/sdks/flutter/CHANGELOG.md +++ b/docs/sdks/flutter/CHANGELOG.md @@ -1,3 +1,13 @@ +## 15.0.0 + +* Breaking changes: + * Changed the typing of `AppwriteException`'s response parameter from a `dynamic` object to an optional string (`?String`). + +## 14.0.0 + +* Fixed realtime pong response. +* Fixed issues with `chunkedUpload` method. + ## 13.0.0 * Fixed realtime reconnection issues From d422f693f75e4ac882f3f53e1841c1649e20caeb Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 13 Mar 2025 09:48:39 +0000 Subject: [PATCH 150/275] feat: add query by role to memberships --- app/config/collections/common.php | 7 +++++++ app/controllers/api/users.php | 20 +++++++++++++++++-- src/Appwrite/Migration/Version/V22.php | 8 ++++++++ .../Validator/Queries/Memberships.php | 4 ++-- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/app/config/collections/common.php b/app/config/collections/common.php index f68400e226..866d7b6d5b 100644 --- a/app/config/collections/common.php +++ b/app/config/collections/common.php @@ -1417,6 +1417,13 @@ return [ 'lengths' => [], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => ID::custom('_key_roles'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['roles'], + 'lengths' => [128], + 'orders' => [], + ], ], ], diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 4a551b7478..17096731fa 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -21,6 +21,7 @@ use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Database\Validator\Queries\Identities; +use Appwrite\Utopia\Database\Validator\Queries\Memberships; use Appwrite\Utopia\Database\Validator\Queries\Targets; use Appwrite\Utopia\Database\Validator\Queries\Users; use Appwrite\Utopia\Request; @@ -799,9 +800,11 @@ App::get('/v1/users/:userId/memberships') ] )) ->param('userId', '', new UID(), 'User ID.') + ->param('queries', [], new Memberships(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Memberships::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) ->inject('response') ->inject('dbForProject') - ->action(function (string $userId, Response $response, Database $dbForProject) { + ->action(function (string $userId, array $queries, string $search, Response $response, Database $dbForProject) { $user = $dbForProject->getDocument('users', $userId); @@ -809,6 +812,19 @@ App::get('/v1/users/:userId/memberships') throw new Exception(Exception::USER_NOT_FOUND); } + try { + $queries = Query::parseQueries($queries); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + if (!empty($search)) { + $queries[] = Query::search('search', $search); + } + + // Set internal queries + $queries[] = Query::equal('userInternalId', [$user->getInternalId()]); + $memberships = array_map(function ($membership) use ($dbForProject, $user) { $team = $dbForProject->getDocument('teams', $membership->getAttribute('teamId')); @@ -818,7 +834,7 @@ App::get('/v1/users/:userId/memberships') ->setAttribute('userEmail', $user->getAttribute('email')); return $membership; - }, $user->getAttribute('memberships', [])); + }, $dbForProject->find('memberships', $queries)); $response->dynamic(new Document([ 'memberships' => $memberships, diff --git a/src/Appwrite/Migration/Version/V22.php b/src/Appwrite/Migration/Version/V22.php index 4d15662112..c5fdfc9ed6 100644 --- a/src/Appwrite/Migration/Version/V22.php +++ b/src/Appwrite/Migration/Version/V22.php @@ -75,6 +75,14 @@ class V22 extends Migration Console::warning("'personalRefreshToken' from {$id}: {$th->getMessage()}"); } break; + case 'memberships': + // Create roles index + try { + $this->createIndexFromCollection($this->projectDB, $id, '_key_roles'); + } catch (Throwable $th) { + Console::warning("'_key_roles' from {$id}: {$th->getMessage()}"); + } + break; } usleep(50000); diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Memberships.php b/src/Appwrite/Utopia/Database/Validator/Queries/Memberships.php index 5ff0098662..cef562ba2c 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Memberships.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Memberships.php @@ -9,12 +9,12 @@ class Memberships extends Base 'teamId', 'invited', 'joined', - 'confirm' + 'confirm', + 'roles', ]; /** * Expression constructor - * */ public function __construct() { From 33ab1513d0b1b7673a30aafb29f4f3cfd6a87d35 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 13 Mar 2025 10:46:55 +0000 Subject: [PATCH 151/275] chore: update specs --- app/config/specs/open-api3-latest-client.json | 4 +-- .../specs/open-api3-latest-console.json | 28 +++++++++++++++++-- app/config/specs/open-api3-latest-server.json | 28 +++++++++++++++++-- app/config/specs/swagger2-latest-client.json | 4 +-- app/config/specs/swagger2-latest-console.json | 25 +++++++++++++++-- app/config/specs/swagger2-latest-server.json | 25 +++++++++++++++-- 6 files changed, 102 insertions(+), 12 deletions(-) diff --git a/app/config/specs/open-api3-latest-client.json b/app/config/specs/open-api3-latest-client.json index 316fe13116..e0b4e18e99 100644 --- a/app/config/specs/open-api3-latest-client.json +++ b/app/config/specs/open-api3-latest-client.json @@ -1,7 +1,7 @@ { "openapi": "3.0.0", "info": { - "version": "1.6.1", + "version": "1.6.2", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -6859,7 +6859,7 @@ }, { "name": "queries", - "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm", + "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm, roles", "required": false, "schema": { "type": "array", diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 54161c4262..53a8962172 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -1,7 +1,7 @@ { "openapi": "3.0.0", "info": { - "version": "1.6.1", + "version": "1.6.2", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -25889,7 +25889,7 @@ }, { "name": "queries", - "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm", + "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm, roles", "required": false, "schema": { "type": "array", @@ -27982,6 +27982,30 @@ "x-example": "" }, "in": "path" + }, + { + "name": "queries", + "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm, roles", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "in": "query" + }, + { + "name": "search", + "description": "Search term to filter your list results. Max length: 256 chars.", + "required": false, + "schema": { + "type": "string", + "x-example": "", + "default": "" + }, + "in": "query" } ] } diff --git a/app/config/specs/open-api3-latest-server.json b/app/config/specs/open-api3-latest-server.json index 3d32d3e978..7fd174b6e0 100644 --- a/app/config/specs/open-api3-latest-server.json +++ b/app/config/specs/open-api3-latest-server.json @@ -1,7 +1,7 @@ { "openapi": "3.0.0", "info": { - "version": "1.6.1", + "version": "1.6.2", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -18099,7 +18099,7 @@ }, { "name": "queries", - "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm", + "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm, roles", "required": false, "schema": { "type": "array", @@ -20153,6 +20153,30 @@ "x-example": "" }, "in": "path" + }, + { + "name": "queries", + "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm, roles", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "in": "query" + }, + { + "name": "search", + "description": "Search term to filter your list results. Max length: 256 chars.", + "required": false, + "schema": { + "type": "string", + "x-example": "", + "default": "" + }, + "in": "query" } ] } diff --git a/app/config/specs/swagger2-latest-client.json b/app/config/specs/swagger2-latest-client.json index 8960bfaa5c..2ea4847792 100644 --- a/app/config/specs/swagger2-latest-client.json +++ b/app/config/specs/swagger2-latest-client.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "version": "1.6.1", + "version": "1.6.2", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -7069,7 +7069,7 @@ }, { "name": "queries", - "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm", + "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm, roles", "required": false, "type": "array", "collectionFormat": "multi", diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index 8fc7e7daf3..6e6b1317ee 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "version": "1.6.1", + "version": "1.6.2", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -26371,7 +26371,7 @@ }, { "name": "queries", - "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm", + "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm, roles", "required": false, "type": "array", "collectionFormat": "multi", @@ -28514,6 +28514,27 @@ "type": "string", "x-example": "", "in": "path" + }, + { + "name": "queries", + "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm, roles", + "required": false, + "type": "array", + "collectionFormat": "multi", + "items": { + "type": "string" + }, + "default": [], + "in": "query" + }, + { + "name": "search", + "description": "Search term to filter your list results. Max length: 256 chars.", + "required": false, + "type": "string", + "x-example": "", + "default": "", + "in": "query" } ] } diff --git a/app/config/specs/swagger2-latest-server.json b/app/config/specs/swagger2-latest-server.json index 83757c94f4..7c65c89764 100644 --- a/app/config/specs/swagger2-latest-server.json +++ b/app/config/specs/swagger2-latest-server.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "version": "1.6.1", + "version": "1.6.2", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -18565,7 +18565,7 @@ }, { "name": "queries", - "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm", + "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm, roles", "required": false, "type": "array", "collectionFormat": "multi", @@ -20669,6 +20669,27 @@ "type": "string", "x-example": "", "in": "path" + }, + { + "name": "queries", + "description": "Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https:\/\/appwrite.io\/docs\/queries). Maximum of 100 queries are allowed, each 4096 characters long. You may filter on the following attributes: userId, teamId, invited, joined, confirm, roles", + "required": false, + "type": "array", + "collectionFormat": "multi", + "items": { + "type": "string" + }, + "default": [], + "in": "query" + }, + { + "name": "search", + "description": "Search term to filter your list results. Max length: 256 chars.", + "required": false, + "type": "string", + "x-example": "", + "default": "", + "in": "query" } ] } From fcf15689e8ec27dee3c2da11e1b2a281e48b10a0 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Thu, 13 Mar 2025 14:02:32 +0100 Subject: [PATCH 152/275] update queue to 0.9.* --- composer.json | 4 ++-- composer.lock | 30 +++++++++++++++--------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/composer.json b/composer.json index c9b057b061..bf64e4dc7f 100644 --- a/composer.json +++ b/composer.json @@ -62,10 +62,10 @@ "utopia-php/messaging": "0.16.*", "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", - "utopia-php/platform": "0.7.3", + "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.5.*", "utopia-php/preloader": "0.2.*", - "utopia-php/queue": "0.8.*", + "utopia-php/queue": "0.9.*", "utopia-php/registry": "0.5.*", "utopia-php/storage": "0.18.*", "utopia-php/swoole": "0.8.*", diff --git a/composer.lock b/composer.lock index 83c8ea479a..561220ba60 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "58ad2e1375ec47d944b96864d4aae63f", + "content-hash": "418a223d14b5ec5a5a8751623d5172b6", "packages": [ { "name": "adhocore/jwt", @@ -4325,16 +4325,16 @@ }, { "name": "utopia-php/platform", - "version": "0.7.3", + "version": "0.7.4", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "463c2d817c893d7dbb678c2eac7a8291f2710e25" + "reference": "a5b93d8177702ec458c3af9137663133c012b71b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/463c2d817c893d7dbb678c2eac7a8291f2710e25", - "reference": "463c2d817c893d7dbb678c2eac7a8291f2710e25", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/a5b93d8177702ec458c3af9137663133c012b71b", + "reference": "a5b93d8177702ec458c3af9137663133c012b71b", "shasum": "" }, "require": { @@ -4343,7 +4343,7 @@ "php": ">=8.0", "utopia-php/cli": "0.15.*", "utopia-php/framework": "0.33.*", - "utopia-php/queue": "0.8.*" + "utopia-php/queue": "0.9.*" }, "require-dev": { "laravel/pint": "1.2.*", @@ -4369,9 +4369,9 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/0.7.3" + "source": "https://github.com/utopia-php/platform/tree/0.7.4" }, - "time": "2025-02-04T15:09:00+00:00" + "time": "2025-03-13T13:00:12+00:00" }, { "name": "utopia-php/pools", @@ -4479,16 +4479,16 @@ }, { "name": "utopia-php/queue", - "version": "0.8.6", + "version": "0.9.0", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "b713b997285c29d120bbcbe3d6e93762d850f87c" + "reference": "077075f1d57afa430f76c35ed3bf4616e0eee8e7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/b713b997285c29d120bbcbe3d6e93762d850f87c", - "reference": "b713b997285c29d120bbcbe3d6e93762d850f87c", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/077075f1d57afa430f76c35ed3bf4616e0eee8e7", + "reference": "077075f1d57afa430f76c35ed3bf4616e0eee8e7", "shasum": "" }, "require": { @@ -4538,9 +4538,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.8.6" + "source": "https://github.com/utopia-php/queue/tree/0.9.0" }, - "time": "2025-02-10T03:35:00+00:00" + "time": "2025-03-13T12:22:41+00:00" }, { "name": "utopia-php/registry", @@ -8402,7 +8402,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 49d89a42065e7ae0c47eb35a32f459d7bd44eebb Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 14 Mar 2025 15:29:23 +1300 Subject: [PATCH 153/275] Force clear timeout to ensure it's not set --- app/cli.php | 1 - app/worker.php | 5 ++- composer.json | 6 +-- composer.lock | 120 ++++++++++++++++++++++++------------------------- 4 files changed, 67 insertions(+), 65 deletions(-) diff --git a/app/cli.php b/app/cli.php index ce978c6d9d..6de98b28c5 100644 --- a/app/cli.php +++ b/app/cli.php @@ -187,7 +187,6 @@ CLI::setResource('getLogsDB', function (Group $pools, Cache $cache) { $database ->setSharedTables(true) ->setNamespace('logsV1') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); // set tenant diff --git a/app/worker.php b/app/worker.php index f3e0541c9a..b217af074c 100644 --- a/app/worker.php +++ b/app/worker.php @@ -112,6 +112,8 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, ->setNamespace('_' . $project->getInternalId()); } + $database->clearTimeout(); + return $database; }, ['cache', 'register', 'message', 'project', 'dbForPlatform']); @@ -173,6 +175,8 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf ->setNamespace('_' . $project->getInternalId()); } + $database->clearTimeout(); + return $database; }; }, ['pools', 'dbForPlatform', 'cache']); @@ -198,7 +202,6 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache) { $database ->setSharedTables(true) ->setNamespace('logsV1') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); // set tenant diff --git a/composer.json b/composer.json index c9b057b061..62d565a438 100644 --- a/composer.json +++ b/composer.json @@ -45,13 +45,13 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "0.51.*", + "utopia-php/abuse": "0.52.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.54.0", + "utopia-php/audit": "0.55.*", "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.60.*", + "utopia-php/database": "0.61.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index e89d6b53b5..ea13de51d3 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "58ad2e1375ec47d944b96864d4aae63f", + "content-hash": "71622d147e04860529ddba9e230035b8", "packages": [ { "name": "adhocore/jwt", @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.30.0", + "version": "v4.30.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6" + "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/e1d66682f6836aa87820400f0aa07d9eb566feb6", - "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/f29ba8a30dfd940efb3a8a75dc44446539101f24", + "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.0" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.1" }, - "time": "2025-03-04T22:54:49+00:00" + "time": "2025-03-13T21:08:17+00:00" }, { "name": "jean85/pretty-package-versions", @@ -3364,16 +3364,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.51.0", + "version": "0.52.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "661687b03277f1d202a0e8cf9da6e58c97da2b5e" + "reference": "a0d6421e7e5baa3ac02755496dca9fdeaa814b93" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/661687b03277f1d202a0e8cf9da6e58c97da2b5e", - "reference": "661687b03277f1d202a0e8cf9da6e58c97da2b5e", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/a0d6421e7e5baa3ac02755496dca9fdeaa814b93", + "reference": "a0d6421e7e5baa3ac02755496dca9fdeaa814b93", "shasum": "" }, "require": { @@ -3381,7 +3381,7 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/database": "0.60.*" + "utopia-php/database": "0.*.*" }, "require-dev": { "laravel/pint": "1.*", @@ -3409,9 +3409,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.51.0" + "source": "https://github.com/utopia-php/abuse/tree/0.52.0" }, - "time": "2025-02-17T11:10:18+00:00" + "time": "2025-03-06T03:48:29+00:00" }, { "name": "utopia-php/analytics", @@ -3461,21 +3461,21 @@ }, { "name": "utopia-php/audit", - "version": "0.54.0", + "version": "0.55.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975" + "reference": "9f8cfe5fa5d5011b8dbf93b710236dfa91dc5518" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975", - "reference": "1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/9f8cfe5fa5d5011b8dbf93b710236dfa91dc5518", + "reference": "9f8cfe5fa5d5011b8dbf93b710236dfa91dc5518", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/database": "0.60.*" + "utopia-php/database": "0.*.*" }, "require-dev": { "laravel/pint": "1.*", @@ -3502,9 +3502,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.54.0" + "source": "https://github.com/utopia-php/audit/tree/0.55.0" }, - "time": "2025-02-25T07:21:07+00:00" + "time": "2025-03-06T03:47:47+00:00" }, { "name": "utopia-php/cache", @@ -3705,16 +3705,16 @@ }, { "name": "utopia-php/database", - "version": "0.60.6", + "version": "0.61.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "f3c9aa964b39c6205069f038a26e709a15541406" + "reference": "2e0165bd14a570ec151f400ed381108e81d15b94" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/f3c9aa964b39c6205069f038a26e709a15541406", - "reference": "f3c9aa964b39c6205069f038a26e709a15541406", + "url": "https://api.github.com/repos/utopia-php/database/zipball/2e0165bd14a570ec151f400ed381108e81d15b94", + "reference": "2e0165bd14a570ec151f400ed381108e81d15b94", "shasum": "" }, "require": { @@ -3755,9 +3755,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.60.6" + "source": "https://github.com/utopia-php/database/tree/0.61.1" }, - "time": "2025-03-05T01:23:14+00:00" + "time": "2025-03-14T01:19:38+00:00" }, { "name": "utopia-php/domains", @@ -4159,16 +4159,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.20", + "version": "0.6.22", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba" + "reference": "a0269746bd318ff0993f5aa008675b971689d5b5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba", - "reference": "8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/a0269746bd318ff0993f5aa008675b971689d5b5", + "reference": "a0269746bd318ff0993f5aa008675b971689d5b5", "shasum": "" }, "require": { @@ -4176,7 +4176,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.60.*", + "utopia-php/database": "0.61.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4209,9 +4209,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.20" + "source": "https://github.com/utopia-php/migration/tree/0.6.22" }, - "time": "2025-02-17T11:02:15+00:00" + "time": "2025-03-13T07:35:55+00:00" }, { "name": "utopia-php/mongo", @@ -4810,16 +4810,16 @@ }, { "name": "utopia-php/vcs", - "version": "0.9.3", + "version": "0.9.4", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "865a00c67e81a20938b883f9aa802303790dd3b5" + "reference": "1a8d280b176acc99ea8d9e7364b8767cbb206b4a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/865a00c67e81a20938b883f9aa802303790dd3b5", - "reference": "865a00c67e81a20938b883f9aa802303790dd3b5", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/1a8d280b176acc99ea8d9e7364b8767cbb206b4a", + "reference": "1a8d280b176acc99ea8d9e7364b8767cbb206b4a", "shasum": "" }, "require": { @@ -4854,9 +4854,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/0.9.3" + "source": "https://github.com/utopia-php/vcs/tree/0.9.4" }, - "time": "2025-02-26T16:33:35+00:00" + "time": "2025-03-13T10:09:45+00:00" }, { "name": "utopia-php/websocket", @@ -5043,16 +5043,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.2", + "version": "0.40.7", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "56f09482d9e2f223911277ab887f197402708049" + "reference": "9e89b0bc4d8e6c81817d27096629f34a149fa873" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/56f09482d9e2f223911277ab887f197402708049", - "reference": "56f09482d9e2f223911277ab887f197402708049", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/9e89b0bc4d8e6c81817d27096629f34a149fa873", + "reference": "9e89b0bc4d8e6c81817d27096629f34a149fa873", "shasum": "" }, "require": { @@ -5088,9 +5088,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.2" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.7" }, - "time": "2025-03-06T16:31:03+00:00" + "time": "2025-03-12T08:43:55+00:00" }, { "name": "doctrine/annotations", @@ -5317,16 +5317,16 @@ }, { "name": "laravel/pint", - "version": "v1.21.0", + "version": "v1.21.1", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "531fa0871fbde719c51b12afa3a443b8f4e4b425" + "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/531fa0871fbde719c51b12afa3a443b8f4e4b425", - "reference": "531fa0871fbde719c51b12afa3a443b8f4e4b425", + "url": "https://api.github.com/repos/laravel/pint/zipball/c44bffbb2334e90fba560933c45948fa4a3f3e86", + "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86", "shasum": "" }, "require": { @@ -5337,9 +5337,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.68.5", - "illuminate/view": "^11.42.0", - "larastan/larastan": "^3.0.4", + "friendsofphp/php-cs-fixer": "^3.70.2", + "illuminate/view": "^11.44.1", + "larastan/larastan": "^3.1.0", "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3", @@ -5379,7 +5379,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-02-18T03:18:57+00:00" + "time": "2025-03-11T03:22:21+00:00" }, { "name": "matthiasmullie/minify", @@ -5794,16 +5794,16 @@ }, { "name": "phpbench/phpbench", - "version": "1.4.0", + "version": "1.4.1", "source": { "type": "git", "url": "https://github.com/phpbench/phpbench.git", - "reference": "4248817222514421cba466bfa7adc7d8932345d4" + "reference": "78cd98a9aa34e0f8f80ca01972a8b88d2c30194b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpbench/phpbench/zipball/4248817222514421cba466bfa7adc7d8932345d4", - "reference": "4248817222514421cba466bfa7adc7d8932345d4", + "url": "https://api.github.com/repos/phpbench/phpbench/zipball/78cd98a9aa34e0f8f80ca01972a8b88d2c30194b", + "reference": "78cd98a9aa34e0f8f80ca01972a8b88d2c30194b", "shasum": "" }, "require": { @@ -5880,7 +5880,7 @@ ], "support": { "issues": "https://github.com/phpbench/phpbench/issues", - "source": "https://github.com/phpbench/phpbench/tree/1.4.0" + "source": "https://github.com/phpbench/phpbench/tree/1.4.1" }, "funding": [ { @@ -5888,7 +5888,7 @@ "type": "github" } ], - "time": "2025-01-26T19:54:45+00:00" + "time": "2025-03-12T08:01:40+00:00" }, { "name": "phpunit/php-code-coverage", From 246c8ce93945f9a059bbe3decdee55cd48c26d22 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 14 Mar 2025 15:29:52 +1300 Subject: [PATCH 154/275] Add order on inequality queries --- src/Appwrite/Platform/Workers/Deletes.php | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 52dc3d73c9..dd219765c2 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -387,6 +387,7 @@ class Deletes extends Action $query = [ Query::lessThan('accessedAt', $datetime), + Query::orderDesc('accessedAt') ]; $this->deleteByGroup( @@ -420,6 +421,7 @@ class Deletes extends Action // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), + Query::orderDesc('time'), Query::equal('period', ['1h']), ], $dbForProject); @@ -430,6 +432,7 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), + Query::orderDesc('time'), Query::equal('period', ['1h']), ], $dbForLogs); } @@ -688,9 +691,11 @@ class Deletes extends Action private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void { $dbForProject = $getProjectDB($project); + // Delete Executions $this->deleteByGroup('executions', [ - Query::lessThan('$createdAt', $datetime) + Query::lessThan('$createdAt', $datetime), + Query::orderDesc('$createdAt'), ], $dbForProject); } @@ -708,7 +713,8 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ - Query::lessThan('$createdAt', $expired) + Query::lessThan('$createdAt', $expired), + Query::orderDesc('$createdAt'), ], $dbForProject); } @@ -722,7 +728,8 @@ class Deletes extends Action { // Delete Dead Realtime Logs $this->deleteByGroup('realtime', [ - Query::lessThan('timestamp', $datetime) + Query::lessThan('timestamp', $datetime), + Query::orderDesc('timestamp'), ], $dbForPlatform); } From d96910138f2da6065e84e565f844685908d9798f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 14 Mar 2025 15:40:05 +1300 Subject: [PATCH 155/275] Update lock --- composer.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.lock b/composer.lock index 2ecba25e1f..23937694f8 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "71622d147e04860529ddba9e230035b8", + "content-hash": "44c6436ced36b0b026139edba252052e", "packages": [ { "name": "adhocore/jwt", @@ -8402,7 +8402,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 817a3808e52bb5a2e66ed02b8148808664c6655a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 14 Mar 2025 16:40:57 +1300 Subject: [PATCH 156/275] Add 5 min timeout for workers and tasks DBs --- app/cli.php | 6 +++--- app/init.php | 12 +++++++----- app/worker.php | 5 +++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/app/cli.php b/app/cli.php index 6de98b28c5..f080217365 100644 --- a/app/cli.php +++ b/app/cli.php @@ -25,7 +25,7 @@ use Utopia\Queue\Publisher; use Utopia\Registry\Registry; use Utopia\System\System; -// overwriting runtimes to be architectur agnostic for CLI +// Overwriting runtimes to be architecture agnostic for CLI Config::setParam('runtimes', (new Runtimes('v4'))->getAll(supported: false)); // require controllers after overwriting runtimes @@ -43,8 +43,7 @@ CLI::setResource('cache', function ($pools) { $adapters[] = $pools ->get($value) ->pop() - ->getResource() - ; + ->getResource(); } return new Cache(new Sharding($adapters)); @@ -187,6 +186,7 @@ CLI::setResource('getLogsDB', function (Group $pools, Cache $cache) { $database ->setSharedTables(true) ->setNamespace('logsV1') + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_TASK) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); // set tenant diff --git a/app/init.php b/app/init.php index 9a499412db..6eb2baafd8 100644 --- a/app/init.php +++ b/app/init.php @@ -134,7 +134,9 @@ const APP_DATABASE_ATTRIBUTE_URL = 'url'; const APP_DATABASE_ATTRIBUTE_INT_RANGE = 'intRange'; const APP_DATABASE_ATTRIBUTE_FLOAT_RANGE = 'floatRange'; const APP_DATABASE_ATTRIBUTE_STRING_MAX_LENGTH = 1_073_741_824; // 2^32 bits / 4 bits per char -const APP_DATABASE_TIMEOUT_MILLISECONDS = 15_000; +const APP_DATABASE_TIMEOUT_MILLISECONDS_API = 15 * 1000; // 15 seconds +const APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER = 300 * 1000; // 5 minutes +const APP_DATABASE_TIMEOUT_MILLISECONDS_TASK = 300 * 1000; // 5 minutes const APP_DATABASE_QUERY_MAX_VALUES = 500; const APP_STORAGE_UPLOADS = '/storage/uploads'; const APP_STORAGE_FUNCTIONS = '/storage/functions'; @@ -1434,7 +1436,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform $database ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); @@ -1466,7 +1468,7 @@ App::setResource('dbForPlatform', function (Group $pools, Cache $cache) { ->setNamespace('_console') ->setMetadata('host', \gethostname()) ->setMetadata('project', 'console') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); return $database; @@ -1491,7 +1493,7 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform $database ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); @@ -1549,7 +1551,7 @@ App::setResource('getLogsDB', function (Group $pools, Cache $cache) { $database ->setSharedTables(true) ->setNamespace('logsV1') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); // set tenant diff --git a/app/worker.php b/app/worker.php index b217af074c..eeefe80000 100644 --- a/app/worker.php +++ b/app/worker.php @@ -112,7 +112,7 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, ->setNamespace('_' . $project->getInternalId()); } - $database->clearTimeout(); + $database->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER); return $database; }, ['cache', 'register', 'message', 'project', 'dbForPlatform']); @@ -175,7 +175,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf ->setNamespace('_' . $project->getInternalId()); } - $database->clearTimeout(); + $database->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER); return $database; }; @@ -202,6 +202,7 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache) { $database ->setSharedTables(true) ->setNamespace('logsV1') + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); // set tenant From f6fc425c981916d5b1fccfbe8c26ae8bbb3b2e18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 14 Mar 2025 09:37:44 +0100 Subject: [PATCH 157/275] Add rule attributes --- app/config/collections/platform.php | 38 ++++++++++++++++++++++++++++- app/controllers/api/functions.php | 2 ++ app/controllers/api/proxy.php | 2 ++ app/controllers/general.php | 10 +++++++- 4 files changed, 50 insertions(+), 2 deletions(-) diff --git a/app/config/collections/platform.php b/app/config/collections/platform.php index 8a46bfd3ec..8725f4b4ce 100644 --- a/app/config/collections/platform.php +++ b/app/config/collections/platform.php @@ -1066,7 +1066,29 @@ return [ 'default' => null, 'array' => false, 'filters' => [], - ] + ], + [ + '$id' => ID::custom('owner'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16, + 'signed' => true, + 'required' => false, + 'default' => '', // "Appwrite" or empty string + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('region'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ @@ -1111,6 +1133,20 @@ return [ 'lengths' => [], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => ID::custom('_key_owner'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['owner'], + 'lengths' => [16], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => ID::custom('_key_region'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['region'], + 'lengths' => [16], + 'orders' => [Database::ORDER_ASC], + ], ], ], diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 583468f6c1..88e1e2ae7b 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -387,6 +387,8 @@ App::post('/v1/functions') 'resourceInternalId' => $function->getInternalId(), 'status' => 'verified', 'certificateId' => '', + 'owner' => 'Appwrite', + 'region' => $project->getAttribute('region') ])) ); diff --git a/app/controllers/api/proxy.php b/app/controllers/api/proxy.php index 2567c23be6..393caad93d 100644 --- a/app/controllers/api/proxy.php +++ b/app/controllers/api/proxy.php @@ -130,6 +130,8 @@ App::post('/v1/proxy/rules') 'resourceId' => $resourceId, 'resourceInternalId' => $resourceInternalId, 'certificateId' => '', + 'owner' => '', + 'region' => $project->getAttribute('region') ]); $status = 'created'; diff --git a/app/controllers/general.php b/app/controllers/general.php index d93766a5e7..e3c28f4276 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -582,6 +582,12 @@ App::init() ]); } + $owner = ''; + $functionsDomain = System::getEnv('_APP_DOMAIN_FUNCTIONS', ''); + if (!empty($functionsDomain) && \str_ends_with($domain->get(), $functionsDomain)) { + $owner = 'Appwrite'; + } + if ($domainDocument->isEmpty()) { $domainDocument = new Document([ // TODO: @christyjacob remove once we migrate the rules in 1.7.x @@ -590,7 +596,9 @@ App::init() 'resourceType' => 'api', 'status' => 'verifying', 'projectId' => 'console', - 'projectInternalId' => 'console' + 'projectInternalId' => 'console', + 'owner' => $owner, + 'region' => 'fra' ]); $domainDocument = $dbForPlatform->createDocument('rules', $domainDocument); From 5f2512dc4f31ef8c457b746cd73f87a0788217e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 14 Mar 2025 12:07:38 +0100 Subject: [PATCH 158/275] code quality improvement --- app/config/console.php | 1 + app/controllers/general.php | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/config/console.php b/app/config/console.php index 5c15a7930f..e37c9b7836 100644 --- a/app/config/console.php +++ b/app/config/console.php @@ -27,6 +27,7 @@ $console = [ 'hostname' => 'localhost', ], // Current host is added on app init ], + 'region' => 'fra', 'legalName' => '', 'legalCountry' => '', 'legalState' => '', diff --git a/app/controllers/general.php b/app/controllers/general.php index e3c28f4276..a374aa3c39 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -595,10 +595,10 @@ App::init() 'domain' => $domain->get(), 'resourceType' => 'api', 'status' => 'verifying', - 'projectId' => 'console', - 'projectInternalId' => 'console', + 'projectId' => $console->getId(), + 'projectInternalId' => $console->getInternalId(), 'owner' => $owner, - 'region' => 'fra' + 'region' => $console->getAttribute('region') ]); $domainDocument = $dbForPlatform->createDocument('rules', $domainDocument); From 1933e48d9c6be0b8fac2f0fb0fd960f5d07592a7 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Mar 2025 12:00:36 +0100 Subject: [PATCH 159/275] Cleanups --- app/init.php | 1738 +-------------------------------- app/init/config.php | 36 + app/init/constants.php | 189 ++++ app/init/database/filters.php | 397 ++++++++ app/init/database/formats.php | 43 + app/init/locale.php | 26 + app/init/registers.php | 333 +++++++ app/init/resources.php | 732 ++++++++++++++ composer.json | 4 + 9 files changed, 1767 insertions(+), 1731 deletions(-) create mode 100644 app/init/config.php create mode 100644 app/init/constants.php create mode 100644 app/init/database/filters.php create mode 100644 app/init/database/formats.php create mode 100644 app/init/locale.php create mode 100644 app/init/registers.php create mode 100644 app/init/resources.php diff --git a/app/init.php b/app/init.php index b4ab772e0e..3e63062bac 100644 --- a/app/init.php +++ b/app/init.php @@ -18,1051 +18,15 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { \ini_set('default_socket_timeout', -1); \error_reporting(E_ALL); -use Ahc\Jwt\JWT; -use Ahc\Jwt\JWTException; -use Appwrite\Auth\Auth; -use Appwrite\Event\Audit; -use Appwrite\Event\Build; -use Appwrite\Event\Certificate; -use Appwrite\Event\Database as EventDatabase; -use Appwrite\Event\Delete; -use Appwrite\Event\Event; -use Appwrite\Event\Func; -use Appwrite\Event\Mail; -use Appwrite\Event\Messaging; -use Appwrite\Event\Migration; -use Appwrite\Event\Usage; -use Appwrite\Extend\Exception; -use Appwrite\Functions\Specification; -use Appwrite\GraphQL\Promises\Adapter\Swoole; -use Appwrite\GraphQL\Schema; -use Appwrite\Hooks\Hooks; -use Appwrite\Network\Validator\Email; -use Appwrite\Network\Validator\Origin; -use Appwrite\OpenSSL\OpenSSL; -use Appwrite\URL\URL as AppwriteURL; -use MaxMind\Db\Reader; -use PHPMailer\PHPMailer\PHPMailer; -use Swoole\Database\PDOProxy; -use Utopia\App; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Adapter\Sharding; -use Utopia\Cache\Cache; -use Utopia\CLI\Console; -use Utopia\Config\Config; -use Utopia\Database\Adapter\MariaDB; -use Utopia\Database\Adapter\MySQL; -use Utopia\Database\Adapter\SQL; -use Utopia\Database\Database; -use Utopia\Database\Document; -use Utopia\Database\Helpers\ID; -use Utopia\Database\Query; -use Utopia\Database\Validator\Authorization; -use Utopia\Database\Validator\Datetime as DatetimeValidator; -use Utopia\Database\Validator\Structure; -use Utopia\Domains\Validator\PublicDomain; -use Utopia\DSN\DSN; -use Utopia\Locale\Locale; -use Utopia\Logger\Adapter\AppSignal; -use Utopia\Logger\Adapter\LogOwl; -use Utopia\Logger\Adapter\Raygun; -use Utopia\Logger\Adapter\Sentry; -use Utopia\Logger\Log; -use Utopia\Logger\Logger; -use Utopia\Pools\Group; -use Utopia\Pools\Pool; -use Utopia\Queue; -use Utopia\Queue\Connection; -use Utopia\Registry\Registry; -use Utopia\Storage\Device; -use Utopia\Storage\Device\Backblaze; -use Utopia\Storage\Device\DOSpaces; -use Utopia\Storage\Device\Linode; -use Utopia\Storage\Device\Local; -use Utopia\Storage\Device\S3; -use Utopia\Storage\Device\Wasabi; -use Utopia\Storage\Storage; use Utopia\System\System; -use Utopia\Validator\Hostname; -use Utopia\Validator\IP; -use Utopia\Validator\Range; -use Utopia\Validator\URL; -use Utopia\Validator\WhiteList; -use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub; -const APP_NAME = 'Appwrite'; -const APP_DOMAIN = 'appwrite.io'; -const APP_EMAIL_TEAM = 'team@localhost.test'; // Default email address -const APP_EMAIL_SECURITY = ''; // Default security email address -const APP_USERAGENT = APP_NAME . '-Server v%s. Please report abuse at %s'; -const APP_MODE_DEFAULT = 'default'; -const APP_MODE_ADMIN = 'admin'; -const APP_PAGING_LIMIT = 12; -const APP_LIMIT_COUNT = 5000; -const APP_LIMIT_USERS = 10_000; -const APP_LIMIT_USER_PASSWORD_HISTORY = 20; -const APP_LIMIT_USER_SESSIONS_MAX = 100; -const APP_LIMIT_USER_SESSIONS_DEFAULT = 10; -const APP_LIMIT_ANTIVIRUS = 20_000_000; //20MB -const APP_LIMIT_ENCRYPTION = 20_000_000; //20MB -const APP_LIMIT_COMPRESSION = 20_000_000; //20MB -const APP_LIMIT_ARRAY_PARAMS_SIZE = 100; // Default maximum of how many elements can there be in API parameter that expects array value -const APP_LIMIT_ARRAY_LABELS_SIZE = 1000; // Default maximum of how many labels elements can there be in API parameter that expects array value -const APP_LIMIT_ARRAY_ELEMENT_SIZE = 4096; // Default maximum length of element in array parameter represented by maximum URL length. -const APP_LIMIT_SUBQUERY = 1000; -const APP_LIMIT_SUBSCRIBERS_SUBQUERY = 1_000_000; -const APP_LIMIT_WRITE_RATE_DEFAULT = 60; // Default maximum write rate per rate period -const APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT = 60; // Default maximum write rate period in seconds -const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return in list API calls -const APP_KEY_ACCESS = 24 * 60 * 60; // 24 hours -const APP_USER_ACCESS = 24 * 60 * 60; // 24 hours -const APP_PROJECT_ACCESS = 24 * 60 * 60; // 24 hours -const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 4318; -const APP_VERSION_STABLE = '1.6.0'; -const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; -const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; -const APP_DATABASE_ATTRIBUTE_IP = 'ip'; -const APP_DATABASE_ATTRIBUTE_DATETIME = 'datetime'; -const APP_DATABASE_ATTRIBUTE_URL = 'url'; -const APP_DATABASE_ATTRIBUTE_INT_RANGE = 'intRange'; -const APP_DATABASE_ATTRIBUTE_FLOAT_RANGE = 'floatRange'; -const APP_DATABASE_ATTRIBUTE_STRING_MAX_LENGTH = 1_073_741_824; // 2^32 bits / 4 bits per char -const APP_DATABASE_TIMEOUT_MILLISECONDS = 15_000; -const APP_STORAGE_UPLOADS = '/storage/uploads'; -const APP_STORAGE_FUNCTIONS = '/storage/functions'; -const APP_STORAGE_BUILDS = '/storage/builds'; -const APP_STORAGE_CACHE = '/storage/cache'; -const APP_STORAGE_CERTIFICATES = '/storage/certificates'; -const APP_STORAGE_CONFIG = '/storage/config'; -const APP_STORAGE_READ_BUFFER = 20 * (1000 * 1000); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT` -const APP_SOCIAL_TWITTER = 'https://twitter.com/appwrite'; -const APP_SOCIAL_TWITTER_HANDLE = 'appwrite'; -const APP_SOCIAL_FACEBOOK = 'https://www.facebook.com/appwrite.io'; -const APP_SOCIAL_LINKEDIN = 'https://www.linkedin.com/company/appwrite'; -const APP_SOCIAL_INSTAGRAM = 'https://www.instagram.com/appwrite.io'; -const APP_SOCIAL_GITHUB = 'https://github.com/appwrite'; -const APP_SOCIAL_DISCORD = 'https://appwrite.io/discord'; -const APP_SOCIAL_DISCORD_CHANNEL = '564160730845151244'; -const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; -const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; -const APP_SOCIAL_YOUTUBE = 'https://www.youtube.com/c/appwrite?sub_confirmation=1'; -const APP_HOSTNAME_INTERNAL = 'appwrite'; -const APP_FUNCTION_SPECIFICATION_DEFAULT = Specification::S_05VCPU_512MB; -const APP_FUNCTION_CPUS_DEFAULT = 0.5; -const APP_FUNCTION_MEMORY_DEFAULT = 512; - -// Database Reconnect -const DATABASE_RECONNECT_SLEEP = 2; -const DATABASE_RECONNECT_MAX_ATTEMPTS = 10; - -// Database Worker Types -const DATABASE_TYPE_CREATE_ATTRIBUTE = 'createAttribute'; -const DATABASE_TYPE_CREATE_INDEX = 'createIndex'; -const DATABASE_TYPE_DELETE_ATTRIBUTE = 'deleteAttribute'; -const DATABASE_TYPE_DELETE_INDEX = 'deleteIndex'; -const DATABASE_TYPE_DELETE_COLLECTION = 'deleteCollection'; -const DATABASE_TYPE_DELETE_DATABASE = 'deleteDatabase'; - -// Build Worker Types -const BUILD_TYPE_DEPLOYMENT = 'deployment'; -const BUILD_TYPE_RETRY = 'retry'; - -// Deletion Types -const DELETE_TYPE_DATABASES = 'databases'; -const DELETE_TYPE_DOCUMENT = 'document'; -const DELETE_TYPE_COLLECTIONS = 'collections'; -const DELETE_TYPE_PROJECTS = 'projects'; -const DELETE_TYPE_FUNCTIONS = 'functions'; -const DELETE_TYPE_DEPLOYMENTS = 'deployments'; -const DELETE_TYPE_USERS = 'users'; -const DELETE_TYPE_TEAM_PROJECTS = 'teams_projects'; -const DELETE_TYPE_EXECUTIONS = 'executions'; -const DELETE_TYPE_AUDIT = 'audit'; -const DELETE_TYPE_ABUSE = 'abuse'; -const DELETE_TYPE_USAGE = 'usage'; -const DELETE_TYPE_REALTIME = 'realtime'; -const DELETE_TYPE_BUCKETS = 'buckets'; -const DELETE_TYPE_INSTALLATIONS = 'installations'; -const DELETE_TYPE_RULES = 'rules'; -const DELETE_TYPE_SESSIONS = 'sessions'; -const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp'; -const DELETE_TYPE_CACHE_BY_RESOURCE = 'cacheByResource'; -const DELETE_TYPE_SCHEDULES = 'schedules'; -const DELETE_TYPE_TOPIC = 'topic'; -const DELETE_TYPE_TARGET = 'target'; -const DELETE_TYPE_EXPIRED_TARGETS = 'invalid_targets'; -const DELETE_TYPE_SESSION_TARGETS = 'session_targets'; - -// Message types -const MESSAGE_SEND_TYPE_INTERNAL = 'internal'; -const MESSAGE_SEND_TYPE_EXTERNAL = 'external'; -// Mail Types -const MAIL_TYPE_VERIFICATION = 'verification'; -const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; -const MAIL_TYPE_RECOVERY = 'recovery'; -const MAIL_TYPE_INVITATION = 'invitation'; -const MAIL_TYPE_CERTIFICATE = 'certificate'; -// Auth Types -const APP_AUTH_TYPE_SESSION = 'Session'; -const APP_AUTH_TYPE_JWT = 'JWT'; -const APP_AUTH_TYPE_KEY = 'Key'; -const APP_AUTH_TYPE_ADMIN = 'Admin'; -// Response related -const MAX_OUTPUT_CHUNK_SIZE = 10 * 1024 * 1024; // 10MB -// Function headers -const FUNCTION_ALLOWLIST_HEADERS_REQUEST = ['content-type', 'agent', 'content-length', 'host']; -const FUNCTION_ALLOWLIST_HEADERS_RESPONSE = ['content-type', 'content-length']; -// Message types -const MESSAGE_TYPE_EMAIL = 'email'; -const MESSAGE_TYPE_SMS = 'sms'; -const MESSAGE_TYPE_PUSH = 'push'; -// API key types -const API_KEY_STANDARD = 'standard'; -const API_KEY_DYNAMIC = 'dynamic'; -// Usage metrics -const METRIC_TEAMS = 'teams'; -const METRIC_USERS = 'users'; - -const METRIC_AUTH_METHOD_PHONE = 'auth.method.phone'; -const METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE = METRIC_AUTH_METHOD_PHONE . '.{countryCode}'; -const METRIC_MESSAGES = 'messages'; -const METRIC_MESSAGES_SENT = METRIC_MESSAGES . '.sent'; -const METRIC_MESSAGES_FAILED = METRIC_MESSAGES . '.failed'; -const METRIC_MESSAGES_TYPE = METRIC_MESSAGES . '.{type}'; -const METRIC_MESSAGES_TYPE_SENT = METRIC_MESSAGES . '.{type}.sent'; -const METRIC_MESSAGES_TYPE_FAILED = METRIC_MESSAGES . '.{type}.failed'; -const METRIC_MESSAGES_TYPE_PROVIDER = METRIC_MESSAGES . '.{type}.{provider}'; -const METRIC_MESSAGES_TYPE_PROVIDER_SENT = METRIC_MESSAGES . '.{type}.{provider}.sent'; -const METRIC_MESSAGES_TYPE_PROVIDER_FAILED = METRIC_MESSAGES . '.{type}.{provider}.failed'; -const METRIC_SESSIONS = 'sessions'; -const METRIC_DATABASES = 'databases'; -const METRIC_COLLECTIONS = 'collections'; -const METRIC_DATABASE_ID_COLLECTIONS = '{databaseInternalId}.collections'; -const METRIC_DOCUMENTS = 'documents'; -const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents'; -const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents'; -const METRIC_BUCKETS = 'buckets'; -const METRIC_FILES = 'files'; -const METRIC_FILES_STORAGE = 'files.storage'; -const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files'; -const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage'; -const METRIC_FUNCTIONS = 'functions'; -const METRIC_DEPLOYMENTS = 'deployments'; -const METRIC_DEPLOYMENTS_STORAGE = 'deployments.storage'; -const METRIC_BUILDS = 'builds'; -const METRIC_BUILDS_SUCCESS = 'builds.success'; -const METRIC_BUILDS_FAILED = 'builds.failed'; -const METRIC_BUILDS_STORAGE = 'builds.storage'; -const METRIC_BUILDS_COMPUTE = 'builds.compute'; -const METRIC_BUILDS_COMPUTE_SUCCESS = 'builds.compute.success'; -const METRIC_BUILDS_COMPUTE_FAILED = 'builds.compute.failed'; -const METRIC_BUILDS_MB_SECONDS = 'builds.mbSeconds'; -const METRIC_FUNCTION_ID_BUILDS = '{functionInternalId}.builds'; -const METRIC_FUNCTION_ID_BUILDS_SUCCESS = '{functionInternalId}.builds.success'; -const METRIC_FUNCTION_ID_BUILDS_FAILED = '{functionInternalId}.builds.failed'; -const METRIC_FUNCTION_ID_BUILDS_STORAGE = '{functionInternalId}.builds.storage'; -const METRIC_FUNCTION_ID_BUILDS_COMPUTE = '{functionInternalId}.builds.compute'; -const METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS = '{functionInternalId}.builds.compute.success'; -const METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED = '{functionInternalId}.builds.compute.failed'; -const METRIC_FUNCTION_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments'; -const METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage'; -const METRIC_FUNCTION_ID_BUILDS_MB_SECONDS = '{functionInternalId}.builds.mbSeconds'; -const METRIC_EXECUTIONS = 'executions'; -const METRIC_EXECUTIONS_COMPUTE = 'executions.compute'; -const METRIC_EXECUTIONS_MB_SECONDS = 'executions.mbSeconds'; -const METRIC_FUNCTION_ID_EXECUTIONS = '{functionInternalId}.executions'; -const METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE = '{functionInternalId}.executions.compute'; -const METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS = '{functionInternalId}.executions.mbSeconds'; -const METRIC_NETWORK_REQUESTS = 'network.requests'; -const METRIC_NETWORK_INBOUND = 'network.inbound'; -const METRIC_NETWORK_OUTBOUND = 'network.outbound'; - -$register = new Registry(); - -App::setMode(System::getEnv('_APP_ENV', App::MODE_TYPE_PRODUCTION)); - -if (!App::isProduction()) { - // Allow specific domains to skip public domain validation in dev environment - // Useful for existing tests involving webhooks - PublicDomain::allow(['request-catcher']); -} - -/* - * ENV vars - */ -Config::load('events', __DIR__ . '/config/events.php'); -Config::load('auth', __DIR__ . '/config/auth.php'); -Config::load('apis', __DIR__ . '/config/apis.php'); // List of APIs -Config::load('errors', __DIR__ . '/config/errors.php'); -Config::load('oAuthProviders', __DIR__ . '/config/oAuthProviders.php'); -Config::load('platforms', __DIR__ . '/config/platforms.php'); -Config::load('collections', __DIR__ . '/config/collections.php'); -Config::load('runtimes', __DIR__ . '/config/runtimes.php'); -Config::load('runtimes-v2', __DIR__ . '/config/runtimes-v2.php'); -Config::load('usage', __DIR__ . '/config/usage.php'); -Config::load('roles', __DIR__ . '/config/roles.php'); // User roles and scopes -Config::load('scopes', __DIR__ . '/config/scopes.php'); // User roles and scopes -Config::load('services', __DIR__ . '/config/services.php'); // List of services -Config::load('variables', __DIR__ . '/config/variables.php'); // List of env variables -Config::load('regions', __DIR__ . '/config/regions.php'); // List of available regions -Config::load('avatar-browsers', __DIR__ . '/config/avatars/browsers.php'); -Config::load('avatar-credit-cards', __DIR__ . '/config/avatars/credit-cards.php'); -Config::load('avatar-flags', __DIR__ . '/config/avatars/flags.php'); -Config::load('locale-codes', __DIR__ . '/config/locale/codes.php'); -Config::load('locale-currencies', __DIR__ . '/config/locale/currencies.php'); -Config::load('locale-eu', __DIR__ . '/config/locale/eu.php'); -Config::load('locale-languages', __DIR__ . '/config/locale/languages.php'); -Config::load('locale-phones', __DIR__ . '/config/locale/phones.php'); -Config::load('locale-countries', __DIR__ . '/config/locale/countries.php'); -Config::load('locale-continents', __DIR__ . '/config/locale/continents.php'); -Config::load('locale-templates', __DIR__ . '/config/locale/templates.php'); -Config::load('storage-logos', __DIR__ . '/config/storage/logos.php'); -Config::load('storage-mimes', __DIR__ . '/config/storage/mimes.php'); -Config::load('storage-inputs', __DIR__ . '/config/storage/inputs.php'); -Config::load('storage-outputs', __DIR__ . '/config/storage/outputs.php'); -Config::load('runtime-specifications', __DIR__ . '/config/runtimes/specifications.php'); -Config::load('function-templates', __DIR__ . '/config/function-templates.php'); - -/** - * New DB Filters - */ -Database::addFilter( - 'casting', - function (mixed $value) { - return json_encode(['value' => $value], JSON_PRESERVE_ZERO_FRACTION); - }, - function (mixed $value) { - if (is_null($value)) { - return; - } - - return json_decode($value, true)['value']; - } -); - -Database::addFilter( - 'enum', - function (mixed $value, Document $attribute) { - if ($attribute->isSet('elements')) { - $attribute->removeAttribute('elements'); - } - - return $value; - }, - function (mixed $value, Document $attribute) { - $formatOptions = \json_decode($attribute->getAttribute('formatOptions', '[]'), true); - if (isset($formatOptions['elements'])) { - $attribute->setAttribute('elements', $formatOptions['elements']); - } - - return $value; - } -); - -Database::addFilter( - 'range', - function (mixed $value, Document $attribute) { - if ($attribute->isSet('min')) { - $attribute->removeAttribute('min'); - } - if ($attribute->isSet('max')) { - $attribute->removeAttribute('max'); - } - - return $value; - }, - function (mixed $value, Document $attribute) { - $formatOptions = json_decode($attribute->getAttribute('formatOptions', '[]'), true); - if (isset($formatOptions['min']) || isset($formatOptions['max'])) { - $attribute - ->setAttribute('min', $formatOptions['min']) - ->setAttribute('max', $formatOptions['max']); - } - - return $value; - } -); - -Database::addFilter( - 'subQueryAttributes', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - $attributes = $database->find('attributes', [ - Query::equal('collectionInternalId', [$document->getInternalId()]), - Query::equal('databaseInternalId', [$document->getAttribute('databaseInternalId')]), - Query::limit($database->getLimitForAttributes()), - ]); - - foreach ($attributes as $attribute) { - if ($attribute->getAttribute('type') === Database::VAR_RELATIONSHIP) { - $options = $attribute->getAttribute('options'); - foreach ($options as $key => $value) { - $attribute->setAttribute($key, $value); - } - $attribute->removeAttribute('options'); - } - } - - return $attributes; - } -); - -Database::addFilter( - 'subQueryIndexes', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - return $database - ->find('indexes', [ - Query::equal('collectionInternalId', [$document->getInternalId()]), - Query::equal('databaseInternalId', [$document->getAttribute('databaseInternalId')]), - Query::limit($database->getLimitForIndexes()), - ]); - } -); - -Database::addFilter( - 'subQueryPlatforms', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - return $database - ->find('platforms', [ - Query::equal('projectInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_SUBQUERY), - ]); - } -); - -Database::addFilter( - 'subQueryKeys', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - return $database - ->find('keys', [ - Query::equal('projectInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_SUBQUERY), - ]); - } -); - -Database::addFilter( - 'subQueryWebhooks', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - return $database - ->find('webhooks', [ - Query::equal('projectInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_SUBQUERY), - ]); - } -); - -Database::addFilter( - 'subQuerySessions', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - return Authorization::skip(fn () => $database->find('sessions', [ - Query::equal('userInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_SUBQUERY), - ])); - } -); - -Database::addFilter( - 'subQueryTokens', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - return Authorization::skip(fn () => $database - ->find('tokens', [ - Query::equal('userInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_SUBQUERY), - ])); - } -); - -Database::addFilter( - 'subQueryChallenges', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - return Authorization::skip(fn () => $database - ->find('challenges', [ - Query::equal('userInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_SUBQUERY), - ])); - } -); - -Database::addFilter( - 'subQueryAuthenticators', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - return Authorization::skip(fn () => $database - ->find('authenticators', [ - Query::equal('userInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_SUBQUERY), - ])); - } -); - -Database::addFilter( - 'subQueryMemberships', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - return Authorization::skip(fn () => $database - ->find('memberships', [ - Query::equal('userInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_SUBQUERY), - ])); - } -); - -Database::addFilter( - 'subQueryVariables', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - return $database - ->find('variables', [ - Query::equal('resourceInternalId', [$document->getInternalId()]), - Query::equal('resourceType', ['function']), - Query::limit(APP_LIMIT_SUBQUERY), - ]); - } -); - -Database::addFilter( - 'encrypt', - function (mixed $value) { - $key = System::getEnv('_APP_OPENSSL_KEY_V1'); - $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); - $tag = null; - - return json_encode([ - 'data' => OpenSSL::encrypt($value, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag), - 'method' => OpenSSL::CIPHER_AES_128_GCM, - 'iv' => \bin2hex($iv), - 'tag' => \bin2hex($tag ?? ''), - 'version' => '1', - ]); - }, - function (mixed $value) { - if (is_null($value)) { - return; - } - $value = json_decode($value, true); - $key = System::getEnv('_APP_OPENSSL_KEY_V' . $value['version']); - - return OpenSSL::decrypt($value['data'], $value['method'], $key, 0, hex2bin($value['iv']), hex2bin($value['tag'])); - } -); - -Database::addFilter( - 'subQueryProjectVariables', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - return $database - ->find('variables', [ - Query::equal('resourceType', ['project']), - Query::limit(APP_LIMIT_SUBQUERY) - ]); - } -); - -Database::addFilter( - 'userSearch', - function (mixed $value, Document $user) { - $searchValues = [ - $user->getId(), - $user->getAttribute('email', ''), - $user->getAttribute('name', ''), - $user->getAttribute('phone', '') - ]; - - foreach ($user->getAttribute('labels', []) as $label) { - $searchValues[] = 'label:' . $label; - } - - $search = implode(' ', \array_filter($searchValues)); - - return $search; - }, - function (mixed $value) { - return $value; - } -); - -Database::addFilter( - 'subQueryTargets', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - return Authorization::skip(fn () => $database - ->find('targets', [ - Query::equal('userInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_SUBQUERY) - ])); - } -); - -Database::addFilter( - 'subQueryTopicTargets', - function (mixed $value) { - return; - }, - function (mixed $value, Document $document, Database $database) { - $targetIds = Authorization::skip(fn () => \array_map( - fn ($document) => $document->getAttribute('targetInternalId'), - $database->find('subscribers', [ - Query::equal('topicInternalId', [$document->getInternalId()]), - Query::limit(APP_LIMIT_SUBSCRIBERS_SUBQUERY) - ]) - )); - if (\count($targetIds) > 0) { - return $database->skipValidation(fn () => $database->find('targets', [ - Query::equal('$internalId', $targetIds) - ])); - } - return []; - } -); - -Database::addFilter( - 'providerSearch', - function (mixed $value, Document $provider) { - $searchValues = [ - $provider->getId(), - $provider->getAttribute('name', ''), - $provider->getAttribute('provider', ''), - $provider->getAttribute('type', '') - ]; - - $search = \implode(' ', \array_filter($searchValues)); - - return $search; - }, - function (mixed $value) { - return $value; - } -); - -Database::addFilter( - 'topicSearch', - function (mixed $value, Document $topic) { - $searchValues = [ - $topic->getId(), - $topic->getAttribute('name', ''), - $topic->getAttribute('description', ''), - ]; - - $search = \implode(' ', \array_filter($searchValues)); - - return $search; - }, - function (mixed $value) { - return $value; - } -); - -Database::addFilter( - 'messageSearch', - function (mixed $value, Document $message) { - $searchValues = [ - $message->getId(), - $message->getAttribute('description', ''), - $message->getAttribute('status', ''), - ]; - - $data = \json_decode($message->getAttribute('data', []), true); - $providerType = $message->getAttribute('providerType', ''); - - if ($providerType === MESSAGE_TYPE_EMAIL) { - $searchValues = \array_merge($searchValues, [$data['subject'], MESSAGE_TYPE_EMAIL]); - } elseif ($providerType === MESSAGE_TYPE_SMS) { - $searchValues = \array_merge($searchValues, [$data['content'], MESSAGE_TYPE_SMS]); - } else { - $searchValues = \array_merge($searchValues, [$data['title'], MESSAGE_TYPE_PUSH]); - } - - $search = \implode(' ', \array_filter($searchValues)); - - return $search; - }, - function (mixed $value) { - return $value; - } -); - -/** - * DB Formats - */ -Structure::addFormat(APP_DATABASE_ATTRIBUTE_EMAIL, function () { - return new Email(); -}, Database::VAR_STRING); - -Structure::addFormat(APP_DATABASE_ATTRIBUTE_DATETIME, function () { - return new DatetimeValidator(); -}, Database::VAR_DATETIME); - -Structure::addFormat(APP_DATABASE_ATTRIBUTE_ENUM, function ($attribute) { - $elements = $attribute['formatOptions']['elements']; - return new WhiteList($elements, true); -}, Database::VAR_STRING); - -Structure::addFormat(APP_DATABASE_ATTRIBUTE_IP, function () { - return new IP(); -}, Database::VAR_STRING); - -Structure::addFormat(APP_DATABASE_ATTRIBUTE_URL, function () { - return new URL(); -}, Database::VAR_STRING); - -Structure::addFormat(APP_DATABASE_ATTRIBUTE_INT_RANGE, function ($attribute) { - $min = $attribute['formatOptions']['min'] ?? -INF; - $max = $attribute['formatOptions']['max'] ?? INF; - return new Range($min, $max, Range::TYPE_INTEGER); -}, Database::VAR_INTEGER); - -Structure::addFormat(APP_DATABASE_ATTRIBUTE_FLOAT_RANGE, function ($attribute) { - $min = $attribute['formatOptions']['min'] ?? -INF; - $max = $attribute['formatOptions']['max'] ?? INF; - return new Range($min, $max, Range::TYPE_FLOAT); -}, Database::VAR_FLOAT); - -/* - * Registry - */ -$register->set('logger', function () { - // Register error logger - $providerName = System::getEnv('_APP_LOGGING_PROVIDER', ''); - $providerConfig = System::getEnv('_APP_LOGGING_CONFIG', ''); - - try { - $loggingProvider = new DSN($providerConfig ?? ''); - - $providerName = $loggingProvider->getScheme(); - $providerConfig = match ($providerName) { - 'sentry' => ['key' => $loggingProvider->getPassword(), 'projectId' => $loggingProvider->getUser() ?? '', 'host' => 'https://' . $loggingProvider->getHost()], - 'logowl' => ['ticket' => $loggingProvider->getUser() ?? '', 'host' => $loggingProvider->getHost()], - default => ['key' => $loggingProvider->getHost()], - }; - } catch (Throwable $th) { - // Fallback for older Appwrite versions up to 1.5.x that use _APP_LOGGING_PROVIDER and _APP_LOGGING_CONFIG environment variables - Console::warning('Using deprecated logging configuration. Please update your configuration to use DSN format.' . $th->getMessage()); - $configChunks = \explode(";", $providerConfig); - - $providerConfig = match ($providerName) { - 'sentry' => [ 'key' => $configChunks[0], 'projectId' => $configChunks[1] ?? '', 'host' => '',], - 'logowl' => ['ticket' => $configChunks[0] ?? '', 'host' => ''], - default => ['key' => $providerConfig], - }; - } - - if (empty($providerName) || empty($providerConfig)) { - return; - } - - if (!Logger::hasProvider($providerName)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Logging provider not supported. Logging is disabled"); - } - - try { - $adapter = match ($providerName) { - 'sentry' => new Sentry($providerConfig['projectId'], $providerConfig['key'], $providerConfig['host']), - 'logowl' => new LogOwl($providerConfig['ticket'], $providerConfig['host']), - 'raygun' => new Raygun($providerConfig['key']), - 'appsignal' => new AppSignal($providerConfig['key']), - default => null - }; - } catch (Throwable $th) { - $adapter = null; - } - - if ($adapter === null) { - Console::error("Logging provider not supported. Logging is disabled"); - return; - } - - return new Logger($adapter); -}); - -$register->set('pools', function () { - $group = new Group(); - - $fallbackForDB = 'db_main=' . AppwriteURL::unparse([ - 'scheme' => 'mariadb', - 'host' => System::getEnv('_APP_DB_HOST', 'mariadb'), - 'port' => System::getEnv('_APP_DB_PORT', '3306'), - 'user' => System::getEnv('_APP_DB_USER', ''), - 'pass' => System::getEnv('_APP_DB_PASS', ''), - 'path' => System::getEnv('_APP_DB_SCHEMA', ''), - ]); - $fallbackForRedis = 'redis_main=' . AppwriteURL::unparse([ - 'scheme' => 'redis', - 'host' => System::getEnv('_APP_REDIS_HOST', 'redis'), - 'port' => System::getEnv('_APP_REDIS_PORT', '6379'), - 'user' => System::getEnv('_APP_REDIS_USER', ''), - 'pass' => System::getEnv('_APP_REDIS_PASS', ''), - ]); - - $connections = [ - 'console' => [ - 'type' => 'database', - 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), - 'multiple' => false, - 'schemes' => ['mariadb', 'mysql'], - ], - 'database' => [ - 'type' => 'database', - 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), - 'multiple' => true, - 'schemes' => ['mariadb', 'mysql'], - ], - 'queue' => [ - 'type' => 'queue', - 'dsns' => System::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), - 'multiple' => false, - 'schemes' => ['redis'], - ], - 'pubsub' => [ - 'type' => 'pubsub', - 'dsns' => System::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), - 'multiple' => false, - 'schemes' => ['redis'], - ], - 'cache' => [ - 'type' => 'cache', - 'dsns' => System::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), - 'multiple' => true, - 'schemes' => ['redis'], - ], - ]; - - $maxConnections = System::getEnv('_APP_CONNECTIONS_MAX', 151); - $instanceConnections = $maxConnections / System::getEnv('_APP_POOL_CLIENTS', 14); - - $multiprocessing = System::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; - - if ($multiprocessing) { - $workerCount = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); - } else { - $workerCount = 1; - } - - if ($workerCount > $instanceConnections) { - throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); - } - - $poolSize = (int)($instanceConnections / $workerCount); - - foreach ($connections as $key => $connection) { - $type = $connection['type'] ?? ''; - $multiple = $connection['multiple'] ?? false; - $schemes = $connection['schemes'] ?? []; - $config = []; - $dsns = explode(',', $connection['dsns'] ?? ''); - foreach ($dsns as &$dsn) { - $dsn = explode('=', $dsn); - $name = ($multiple) ? $key . '_' . $dsn[0] : $key; - $dsn = $dsn[1] ?? ''; - $config[] = $name; - if (empty($dsn)) { - //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); - continue; - } - - $dsn = new DSN($dsn); - $dsnHost = $dsn->getHost(); - $dsnPort = $dsn->getPort(); - $dsnUser = $dsn->getUser(); - $dsnPass = $dsn->getPassword(); - $dsnScheme = $dsn->getScheme(); - $dsnDatabase = $dsn->getPath(); - - if (!in_array($dsnScheme, $schemes)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); - } - - /** - * Get Resource - * - * Creation could be reused across connection types like database, cache, queue, etc. - * - * Resource assignment to an adapter will happen below. - */ - $resource = match ($dsnScheme) { - 'mysql', - 'mariadb' => function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { - return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { - return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true - )); - }); - }, - 'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) { - $redis = new Redis(); - @$redis->pconnect($dsnHost, (int)$dsnPort); - if ($dsnPass) { - $redis->auth($dsnPass); - } - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - - return $redis; - }, - default => throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Invalid scheme'), - }; - - $pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) { - // Get Adapter - switch ($type) { - case 'database': - $adapter = match ($dsn->getScheme()) { - 'mariadb' => new MariaDB($resource()), - 'mysql' => new MySQL($resource()), - default => null - }; - - $adapter->setDatabase($dsn->getPath()); - break; - case 'pubsub': - $adapter = $resource(); - break; - case 'queue': - $adapter = match ($dsn->getScheme()) { - 'redis' => new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()), - default => null - }; - break; - case 'cache': - $adapter = match ($dsn->getScheme()) { - 'redis' => new RedisCache($resource()), - default => null - }; - break; - - default: - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); - } - - return $adapter; - }); - - $group->add($pool); - } - - Config::setParam('pools-' . $key, $config); - } - - return $group; -}); - -$register->set('db', function () { - // This is usually for our workers or CLI commands scope - $dbHost = System::getEnv('_APP_DB_HOST', ''); - $dbPort = System::getEnv('_APP_DB_PORT', ''); - $dbUser = System::getEnv('_APP_DB_USER', ''); - $dbPass = System::getEnv('_APP_DB_PASS', ''); - $dbScheme = System::getEnv('_APP_DB_SCHEMA', ''); - - return new PDO( - "mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", - $dbUser, - $dbPass, - SQL::getPDOAttributes() - ); -}); - -$register->set('smtp', function () { - $mail = new PHPMailer(true); - - $mail->isSMTP(); - - $username = System::getEnv('_APP_SMTP_USERNAME'); - $password = System::getEnv('_APP_SMTP_PASSWORD'); - - $mail->XMailer = 'Appwrite Mailer'; - $mail->Host = System::getEnv('_APP_SMTP_HOST', 'smtp'); - $mail->Port = System::getEnv('_APP_SMTP_PORT', 25); - $mail->SMTPAuth = !empty($username) && !empty($password); - $mail->Username = $username; - $mail->Password = $password; - $mail->SMTPSecure = System::getEnv('_APP_SMTP_SECURE', ''); - $mail->SMTPAutoTLS = false; - $mail->CharSet = 'UTF-8'; - - $from = \urldecode(System::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server')); - $email = System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM); - - $mail->setFrom($email, $from); - $mail->addReplyTo($email, $from); - - $mail->isHTML(true); - - return $mail; -}); -$register->set('geodb', function () { - return new Reader(__DIR__ . '/assets/dbip/dbip-country-lite-2024-09.mmdb'); -}); -$register->set('passwordsDictionary', function () { - $content = \file_get_contents(__DIR__ . '/assets/security/10k-common-passwords'); - $content = explode("\n", $content); - $content = array_flip($content); - return $content; -}); -$register->set('promiseAdapter', function () { - return new Swoole(); -}); -$register->set('hooks', function () { - return new Hooks(); -}); -/* - * Localization - */ -Locale::$exceptions = false; - -$locales = Config::getParam('locale-codes', []); - -foreach ($locales as $locale) { - $code = $locale['code']; - - $path = __DIR__ . '/config/locale/translations/' . $code . '.json'; - - if (!\file_exists($path)) { - $path = __DIR__ . '/config/locale/translations/' . \substr($code, 0, 2) . '.json'; // if `ar-ae` doesn't exist, look for `ar` - if (!\file_exists($path)) { - $path = __DIR__ . '/config/locale/translations/en.json'; // if none translation exists, use default from `en.json` - } - } - - Locale::setLanguageFromJSON($code, $path); -} +require_once __DIR__ . '/init/constants.php'; +require_once __DIR__ . '/init/config.php'; +require_once __DIR__ . '/init/database/filters.php'; +require_once __DIR__ . '/init/database/formats.php'; +require_once __DIR__ . '/init/locale.php'; +require_once __DIR__ . '/init/registers.php'; +require_once __DIR__ . '/init/resources.php'; \stream_context_set_default([ // Set global user agent and http settings 'http' => [ @@ -1075,691 +39,3 @@ foreach ($locales as $locale) { 'timeout' => 2, ], ]); - -// Runtime Execution -App::setResource('log', fn () => new Log()); -App::setResource('logger', function ($register) { - return $register->get('logger'); -}, ['register']); - -App::setResource('hooks', function ($register) { - return $register->get('hooks'); -}, ['register']); - -App::setResource('register', fn () => $register); -App::setResource('locale', fn () => new Locale(System::getEnv('_APP_LOCALE', 'en'))); - -App::setResource('localeCodes', function () { - return array_map(fn ($locale) => $locale['code'], Config::getParam('locale-codes', [])); -}); - -// Queues -App::setResource('queue', function (Group $pools) { - return $pools->get('queue')->pop()->getResource(); -}, ['pools']); -App::setResource('queueForMessaging', function (Connection $queue) { - return new Messaging($queue); -}, ['queue']); -App::setResource('queueForMails', function (Connection $queue) { - return new Mail($queue); -}, ['queue']); -App::setResource('queueForBuilds', function (Connection $queue) { - return new Build($queue); -}, ['queue']); -App::setResource('queueForDatabase', function (Connection $queue) { - return new EventDatabase($queue); -}, ['queue']); -App::setResource('queueForDeletes', function (Connection $queue) { - return new Delete($queue); -}, ['queue']); -App::setResource('queueForEvents', function (Connection $queue) { - return new Event($queue); -}, ['queue']); -App::setResource('queueForAudits', function (Connection $queue) { - return new Audit($queue); -}, ['queue']); -App::setResource('queueForFunctions', function (Connection $queue) { - return new Func($queue); -}, ['queue']); -App::setResource('queueForUsage', function (Connection $queue) { - return new Usage($queue); -}, ['queue']); -App::setResource('queueForCertificates', function (Connection $queue) { - return new Certificate($queue); -}, ['queue']); -App::setResource('queueForMigrations', function (Connection $queue) { - return new Migration($queue); -}, ['queue']); -App::setResource('clients', function ($request, $console, $project) { - $console->setAttribute('platforms', [ // Always allow current host - '$collection' => ID::custom('platforms'), - 'name' => 'Current Host', - 'type' => Origin::CLIENT_TYPE_WEB, - 'hostname' => $request->getHostname(), - ], Document::SET_TYPE_APPEND); - - $hostnames = explode(',', System::getEnv('_APP_CONSOLE_HOSTNAMES', '')); - $validator = new Hostname(); - foreach ($hostnames as $hostname) { - $hostname = trim($hostname); - if (!$validator->isValid($hostname)) { - continue; - } - $console->setAttribute('platforms', [ - '$collection' => ID::custom('platforms'), - 'type' => Origin::CLIENT_TYPE_WEB, - 'name' => $hostname, - 'hostname' => $hostname, - ], Document::SET_TYPE_APPEND); - } - - /** - * Get All verified client URLs for both console and current projects - * + Filter for duplicated entries - */ - $clientsConsole = \array_map( - fn ($node) => $node['hostname'], - \array_filter( - $console->getAttribute('platforms', []), - fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB) && !empty($node['hostname'])) - ) - ); - - $clients = $clientsConsole; - $platforms = $project->getAttribute('platforms', []); - - foreach ($platforms as $node) { - if ( - isset($node['type']) && - ($node['type'] === Origin::CLIENT_TYPE_WEB || - $node['type'] === Origin::CLIENT_TYPE_FLUTTER_WEB) && - !empty($node['hostname']) - ) { - $clients[] = $node['hostname']; - } - } - - return \array_unique($clients); -}, ['request', 'console', 'project']); - -App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForConsole) { - /** @var Appwrite\Utopia\Request $request */ - /** @var Appwrite\Utopia\Response $response */ - /** @var Utopia\Database\Document $project */ - /** @var Utopia\Database\Database $dbForProject */ - /** @var Utopia\Database\Database $dbForConsole */ - /** @var string $mode */ - - Authorization::setDefaultStatus(true); - - Auth::setCookieName('a_session_' . $project->getId()); - - if (APP_MODE_ADMIN === $mode) { - Auth::setCookieName('a_session_' . $console->getId()); - } - - $session = Auth::decodeSession( - $request->getCookie( - Auth::$cookieName, // Get sessions - $request->getCookie(Auth::$cookieName . '_legacy', '') - ) - ); - - // Get session from header for SSR clients - if (empty($session['id']) && empty($session['secret'])) { - $sessionHeader = $request->getHeader('x-appwrite-session', ''); - - if (!empty($sessionHeader)) { - $session = Auth::decodeSession($sessionHeader); - } - } - - // Get fallback session from old clients (no SameSite support) or clients who block 3rd-party cookies - if ($response) { - $response->addHeader('X-Debug-Fallback', 'false'); - } - - if (empty($session['id']) && empty($session['secret'])) { - if ($response) { - $response->addHeader('X-Debug-Fallback', 'true'); - } - $fallback = $request->getHeader('x-fallback-cookies', ''); - $fallback = \json_decode($fallback, true); - $session = Auth::decodeSession(((isset($fallback[Auth::$cookieName])) ? $fallback[Auth::$cookieName] : '')); - } - - Auth::$unique = $session['id'] ?? ''; - Auth::$secret = $session['secret'] ?? ''; - - if (APP_MODE_ADMIN !== $mode) { - if ($project->isEmpty()) { - $user = new Document([]); - } else { - if ($project->getId() === 'console') { - $user = $dbForConsole->getDocument('users', Auth::$unique); - } else { - $user = $dbForProject->getDocument('users', Auth::$unique); - } - } - } else { - $user = $dbForConsole->getDocument('users', Auth::$unique); - } - - if ( - $user->isEmpty() // Check a document has been found in the DB - || !Auth::sessionVerify($user->getAttribute('sessions', []), Auth::$secret) - ) { // Validate user has valid login token - $user = new Document([]); - } - - if (APP_MODE_ADMIN === $mode) { - if ($user->find('teamInternalId', $project->getAttribute('teamInternalId'), 'memberships')) { - Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users. - } else { - $user = new Document([]); - } - } - - $authJWT = $request->getHeader('x-appwrite-jwt', ''); - - if (!empty($authJWT) && !$project->isEmpty()) { // JWT authentication - $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0); - - try { - $payload = $jwt->decode($authJWT); - } catch (JWTException $error) { - throw new Exception(Exception::USER_JWT_INVALID, 'Failed to verify JWT. ' . $error->getMessage()); - } - - $jwtUserId = $payload['userId'] ?? ''; - if (!empty($jwtUserId)) { - $user = $dbForProject->getDocument('users', $jwtUserId); - } - - $jwtSessionId = $payload['sessionId'] ?? ''; - if (!empty($jwtSessionId)) { - if (empty($user->find('$id', $jwtSessionId, 'sessions'))) { // Match JWT to active token - $user = new Document([]); - } - } - } - - $dbForProject->setMetadata('user', $user->getId()); - $dbForConsole->setMetadata('user', $user->getId()); - - return $user; -}, ['mode', 'project', 'console', 'request', 'response', 'dbForProject', 'dbForConsole']); - -App::setResource('project', function ($dbForConsole, $request, $console) { - /** @var Appwrite\Utopia\Request $request */ - /** @var Utopia\Database\Database $dbForConsole */ - /** @var Utopia\Database\Document $console */ - - $projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', '')); - - if (empty($projectId) || $projectId === 'console') { - return $console; - } - - $project = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $projectId)); - - return $project; -}, ['dbForConsole', 'request', 'console']); - -App::setResource('session', function (Document $user) { - if ($user->isEmpty()) { - return; - } - - $sessions = $user->getAttribute('sessions', []); - $sessionId = Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret); - - if (!$sessionId) { - return; - } - - foreach ($sessions as $session) {/** @var Document $session */ - if ($sessionId === $session->getId()) { - return $session; - } - } - - return; -}, ['user']); - -App::setResource('console', function () { - return new Document([ - '$id' => ID::custom('console'), - '$internalId' => ID::custom('console'), - 'name' => 'Appwrite', - '$collection' => ID::custom('projects'), - 'description' => 'Appwrite core engine', - 'logo' => '', - 'teamId' => -1, - 'webhooks' => [], - 'keys' => [], - 'platforms' => [ - [ - '$collection' => ID::custom('platforms'), - 'name' => 'Localhost', - 'type' => Origin::CLIENT_TYPE_WEB, - 'hostname' => 'localhost', - ], // Current host is added on app init - ], - 'legalName' => '', - 'legalCountry' => '', - 'legalState' => '', - 'legalCity' => '', - 'legalAddress' => '', - 'legalTaxId' => '', - 'auths' => [ - 'mockNumbers' => [], - 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', - 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user - 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds - 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' - ], - 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], - 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], - 'oAuthProviders' => [ - 'githubEnabled' => true, - 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), - 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') - ], - ]); -}, []); - -App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { - if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForConsole; - } - - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - $dbAdapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); - - $database - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); - - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) { - $database - ->setSharedTables(true) - ->setTenant($project->getInternalId()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } - - return $database; -}, ['pools', 'dbForConsole', 'cache', 'project']); - -App::setResource('dbForConsole', function (Group $pools, Cache $cache) { - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); - - $database - ->setNamespace('_console') - ->setMetadata('host', \gethostname()) - ->setMetadata('project', 'console') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); - - return $database; -}, ['pools', 'cache']); - -App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { - $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - - return function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { - if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForConsole; - } - - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - $configure = (function (Database $database) use ($project, $dsn) { - $database - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); - - if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) { - $database - ->setSharedTables(true) - ->setTenant($project->getInternalId()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } - }); - - if (isset($databases[$dsn->getHost()])) { - $database = $databases[$dsn->getHost()]; - $configure($database); - return $database; - } - - $dbAdapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); - $databases[$dsn->getHost()] = $database; - $configure($database); - - return $database; - }; -}, ['pools', 'dbForConsole', 'cache']); - -App::setResource('cache', function (Group $pools) { - $list = Config::getParam('pools-cache', []); - $adapters = []; - - foreach ($list as $value) { - $adapters[] = $pools - ->get($value) - ->pop() - ->getResource() - ; - } - - return new Cache(new Sharding($adapters)); -}, ['pools']); - -App::setResource('deviceForLocal', function () { - return new Local(); -}); - -App::setResource('deviceForFiles', function ($project) { - return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); -}, ['project']); - -App::setResource('deviceForFunctions', function ($project) { - return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); -}, ['project']); - -App::setResource('deviceForBuilds', function ($project) { - return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); -}, ['project']); - -function getDevice($root): Device -{ - $connection = System::getEnv('_APP_CONNECTIONS_STORAGE', ''); - - if (!empty($connection)) { - $acl = 'private'; - $device = Storage::DEVICE_LOCAL; - $accessKey = ''; - $accessSecret = ''; - $bucket = ''; - $region = ''; - - try { - $dsn = new DSN($connection); - $device = $dsn->getScheme(); - $accessKey = $dsn->getUser() ?? ''; - $accessSecret = $dsn->getPassword() ?? ''; - $bucket = $dsn->getPath() ?? ''; - $region = $dsn->getParam('region'); - } catch (\Throwable $e) { - Console::warning($e->getMessage() . 'Invalid DSN. Defaulting to Local device.'); - } - - switch ($device) { - case Storage::DEVICE_S3: - return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case STORAGE::DEVICE_DO_SPACES: - $device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); - $device->setHttpVersion(S3::HTTP_VERSION_1_1); - return $device; - case Storage::DEVICE_BACKBLAZE: - return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_LINODE: - return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_WASABI: - return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); - case Storage::DEVICE_LOCAL: - default: - return new Local($root); - } - } else { - switch (strtolower(System::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) ?? '')) { - case Storage::DEVICE_LOCAL: - default: - return new Local($root); - case Storage::DEVICE_S3: - $s3AccessKey = System::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); - $s3SecretKey = System::getEnv('_APP_STORAGE_S3_SECRET', ''); - $s3Region = System::getEnv('_APP_STORAGE_S3_REGION', ''); - $s3Bucket = System::getEnv('_APP_STORAGE_S3_BUCKET', ''); - $s3Acl = 'private'; - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); - case Storage::DEVICE_DO_SPACES: - $doSpacesAccessKey = System::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); - $doSpacesSecretKey = System::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); - $doSpacesRegion = System::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); - $doSpacesBucket = System::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); - $doSpacesAcl = 'private'; - $device = new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); - $device->setHttpVersion(S3::HTTP_VERSION_1_1); - return $device; - case Storage::DEVICE_BACKBLAZE: - $backblazeAccessKey = System::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); - $backblazeSecretKey = System::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); - $backblazeRegion = System::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); - $backblazeBucket = System::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); - $backblazeAcl = 'private'; - return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); - case Storage::DEVICE_LINODE: - $linodeAccessKey = System::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); - $linodeSecretKey = System::getEnv('_APP_STORAGE_LINODE_SECRET', ''); - $linodeRegion = System::getEnv('_APP_STORAGE_LINODE_REGION', ''); - $linodeBucket = System::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); - $linodeAcl = 'private'; - return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); - case Storage::DEVICE_WASABI: - $wasabiAccessKey = System::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); - $wasabiSecretKey = System::getEnv('_APP_STORAGE_WASABI_SECRET', ''); - $wasabiRegion = System::getEnv('_APP_STORAGE_WASABI_REGION', ''); - $wasabiBucket = System::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); - $wasabiAcl = 'private'; - return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); - } - } -} - -App::setResource('mode', function ($request) { - /** @var Appwrite\Utopia\Request $request */ - - /** - * Defines the mode for the request: - * - 'default' => Requests for Client and Server Side - * - 'admin' => Request from the Console on non-console projects - */ - return $request->getParam('mode', $request->getHeader('x-appwrite-mode', APP_MODE_DEFAULT)); -}, ['request']); - -App::setResource('geodb', function ($register) { - /** @var Utopia\Registry\Registry $register */ - return $register->get('geodb'); -}, ['register']); - -App::setResource('passwordsDictionary', function ($register) { - /** @var Utopia\Registry\Registry $register */ - return $register->get('passwordsDictionary'); -}, ['register']); - - -App::setResource('servers', function () { - $platforms = Config::getParam('platforms'); - $server = $platforms[APP_PLATFORM_SERVER]; - - $languages = array_map(function ($language) { - return strtolower($language['name']); - }, $server['sdks']); - - return $languages; -}); - -App::setResource('promiseAdapter', function ($register) { - return $register->get('promiseAdapter'); -}, ['register']); - -App::setResource('schema', function ($utopia, $dbForProject) { - - $complexity = function (int $complexity, array $args) { - $queries = Query::parseQueries($args['queries'] ?? []); - $query = Query::getByType($queries, [Query::TYPE_LIMIT])[0] ?? null; - $limit = $query ? $query->getValue() : APP_LIMIT_LIST_DEFAULT; - - return $complexity * $limit; - }; - - $attributes = function (int $limit, int $offset) use ($dbForProject) { - $attrs = Authorization::skip(fn () => $dbForProject->find('attributes', [ - Query::limit($limit), - Query::offset($offset), - ])); - - return \array_map(function ($attr) { - return $attr->getArrayCopy(); - }, $attrs); - }; - - $urls = [ - 'list' => function (string $databaseId, string $collectionId, array $args) { - return "/v1/databases/$databaseId/collections/$collectionId/documents"; - }, - 'create' => function (string $databaseId, string $collectionId, array $args) { - return "/v1/databases/$databaseId/collections/$collectionId/documents"; - }, - 'read' => function (string $databaseId, string $collectionId, array $args) { - return "/v1/databases/$databaseId/collections/$collectionId/documents/{$args['documentId']}"; - }, - 'update' => function (string $databaseId, string $collectionId, array $args) { - return "/v1/databases/$databaseId/collections/$collectionId/documents/{$args['documentId']}"; - }, - 'delete' => function (string $databaseId, string $collectionId, array $args) { - return "/v1/databases/$databaseId/collections/$collectionId/documents/{$args['documentId']}"; - }, - ]; - - $params = [ - 'list' => function (string $databaseId, string $collectionId, array $args) { - return [ 'queries' => $args['queries']]; - }, - 'create' => function (string $databaseId, string $collectionId, array $args) { - $id = $args['id'] ?? 'unique()'; - $permissions = $args['permissions'] ?? null; - - unset($args['id']); - unset($args['permissions']); - - // Order must be the same as the route params - return [ - 'databaseId' => $databaseId, - 'documentId' => $id, - 'collectionId' => $collectionId, - 'data' => $args, - 'permissions' => $permissions, - ]; - }, - 'update' => function (string $databaseId, string $collectionId, array $args) { - $documentId = $args['id']; - $permissions = $args['permissions'] ?? null; - - unset($args['id']); - unset($args['permissions']); - - // Order must be the same as the route params - return [ - 'databaseId' => $databaseId, - 'collectionId' => $collectionId, - 'documentId' => $documentId, - 'data' => $args, - 'permissions' => $permissions, - ]; - }, - ]; - - return Schema::build( - $utopia, - $complexity, - $attributes, - $urls, - $params, - ); -}, ['utopia', 'dbForProject']); - -App::setResource('contributors', function () { - $path = 'app/config/contributors.json'; - $list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : []; - return $list; -}); - -App::setResource('employees', function () { - $path = 'app/config/employees.json'; - $list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : []; - return $list; -}); - -App::setResource('heroes', function () { - $path = 'app/config/heroes.json'; - $list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : []; - return $list; -}); - -App::setResource('gitHub', function (Cache $cache) { - return new VcsGitHub($cache); -}, ['cache']); - -App::setResource('requestTimestamp', function ($request) { - //TODO: Move this to the Request class itself - $timestampHeader = $request->getHeader('x-appwrite-timestamp'); - $requestTimestamp = null; - if (!empty($timestampHeader)) { - try { - $requestTimestamp = new \DateTime($timestampHeader); - } catch (\Throwable $e) { - throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Invalid X-Appwrite-Timestamp header value'); - } - } - return $requestTimestamp; -}, ['request']); -App::setResource('plan', function (array $plan = []) { - return []; -}); diff --git a/app/init/config.php b/app/init/config.php new file mode 100644 index 0000000000..72e0e5fd7f --- /dev/null +++ b/app/init/config.php @@ -0,0 +1,36 @@ + $value], JSON_PRESERVE_ZERO_FRACTION); + }, + function (mixed $value) { + if (is_null($value)) { + return; + } + + return json_decode($value, true)['value']; + } +); + +Database::addFilter( + 'enum', + function (mixed $value, Document $attribute) { + if ($attribute->isSet('elements')) { + $attribute->removeAttribute('elements'); + } + + return $value; + }, + function (mixed $value, Document $attribute) { + $formatOptions = \json_decode($attribute->getAttribute('formatOptions', '[]'), true); + if (isset($formatOptions['elements'])) { + $attribute->setAttribute('elements', $formatOptions['elements']); + } + + return $value; + } +); + +Database::addFilter( + 'range', + function (mixed $value, Document $attribute) { + if ($attribute->isSet('min')) { + $attribute->removeAttribute('min'); + } + if ($attribute->isSet('max')) { + $attribute->removeAttribute('max'); + } + + return $value; + }, + function (mixed $value, Document $attribute) { + $formatOptions = json_decode($attribute->getAttribute('formatOptions', '[]'), true); + if (isset($formatOptions['min']) || isset($formatOptions['max'])) { + $attribute + ->setAttribute('min', $formatOptions['min']) + ->setAttribute('max', $formatOptions['max']); + } + + return $value; + } +); + +Database::addFilter( + 'subQueryAttributes', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + $attributes = $database->find('attributes', [ + Query::equal('collectionInternalId', [$document->getInternalId()]), + Query::equal('databaseInternalId', [$document->getAttribute('databaseInternalId')]), + Query::limit($database->getLimitForAttributes()), + ]); + + foreach ($attributes as $attribute) { + if ($attribute->getAttribute('type') === Database::VAR_RELATIONSHIP) { + $options = $attribute->getAttribute('options'); + foreach ($options as $key => $value) { + $attribute->setAttribute($key, $value); + } + $attribute->removeAttribute('options'); + } + } + + return $attributes; + } +); + +Database::addFilter( + 'subQueryIndexes', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return $database + ->find('indexes', [ + Query::equal('collectionInternalId', [$document->getInternalId()]), + Query::equal('databaseInternalId', [$document->getAttribute('databaseInternalId')]), + Query::limit($database->getLimitForIndexes()), + ]); + } +); + +Database::addFilter( + 'subQueryPlatforms', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return $database + ->find('platforms', [ + Query::equal('projectInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ]); + } +); + +Database::addFilter( + 'subQueryKeys', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return $database + ->find('keys', [ + Query::equal('projectInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ]); + } +); + +Database::addFilter( + 'subQueryWebhooks', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return $database + ->find('webhooks', [ + Query::equal('projectInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ]); + } +); + +Database::addFilter( + 'subQuerySessions', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return Authorization::skip(fn () => $database->find('sessions', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ])); + } +); + +Database::addFilter( + 'subQueryTokens', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return Authorization::skip(fn () => $database + ->find('tokens', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ])); + } +); + +Database::addFilter( + 'subQueryChallenges', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return Authorization::skip(fn () => $database + ->find('challenges', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ])); + } +); + +Database::addFilter( + 'subQueryAuthenticators', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return Authorization::skip(fn () => $database + ->find('authenticators', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ])); + } +); + +Database::addFilter( + 'subQueryMemberships', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return Authorization::skip(fn () => $database + ->find('memberships', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ])); + } +); + +Database::addFilter( + 'subQueryVariables', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return $database + ->find('variables', [ + Query::equal('resourceInternalId', [$document->getInternalId()]), + Query::equal('resourceType', ['function']), + Query::limit(APP_LIMIT_SUBQUERY), + ]); + } +); + +Database::addFilter( + 'encrypt', + function (mixed $value) { + $key = System::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $tag = null; + + return json_encode([ + 'data' => OpenSSL::encrypt($value, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag), + 'method' => OpenSSL::CIPHER_AES_128_GCM, + 'iv' => \bin2hex($iv), + 'tag' => \bin2hex($tag ?? ''), + 'version' => '1', + ]); + }, + function (mixed $value) { + if (is_null($value)) { + return; + } + $value = json_decode($value, true); + $key = System::getEnv('_APP_OPENSSL_KEY_V' . $value['version']); + + return OpenSSL::decrypt($value['data'], $value['method'], $key, 0, hex2bin($value['iv']), hex2bin($value['tag'])); + } +); + +Database::addFilter( + 'subQueryProjectVariables', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return $database + ->find('variables', [ + Query::equal('resourceType', ['project']), + Query::limit(APP_LIMIT_SUBQUERY) + ]); + } +); + +Database::addFilter( + 'userSearch', + function (mixed $value, Document $user) { + $searchValues = [ + $user->getId(), + $user->getAttribute('email', ''), + $user->getAttribute('name', ''), + $user->getAttribute('phone', '') + ]; + + foreach ($user->getAttribute('labels', []) as $label) { + $searchValues[] = 'label:' . $label; + } + + $search = implode(' ', \array_filter($searchValues)); + + return $search; + }, + function (mixed $value) { + return $value; + } +); + +Database::addFilter( + 'subQueryTargets', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return Authorization::skip(fn () => $database + ->find('targets', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY) + ])); + } +); + +Database::addFilter( + 'subQueryTopicTargets', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + $targetIds = Authorization::skip(fn () => \array_map( + fn ($document) => $document->getAttribute('targetInternalId'), + $database->find('subscribers', [ + Query::equal('topicInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBSCRIBERS_SUBQUERY) + ]) + )); + if (\count($targetIds) > 0) { + return $database->skipValidation(fn () => $database->find('targets', [ + Query::equal('$internalId', $targetIds) + ])); + } + return []; + } +); + +Database::addFilter( + 'providerSearch', + function (mixed $value, Document $provider) { + $searchValues = [ + $provider->getId(), + $provider->getAttribute('name', ''), + $provider->getAttribute('provider', ''), + $provider->getAttribute('type', '') + ]; + + $search = \implode(' ', \array_filter($searchValues)); + + return $search; + }, + function (mixed $value) { + return $value; + } +); + +Database::addFilter( + 'topicSearch', + function (mixed $value, Document $topic) { + $searchValues = [ + $topic->getId(), + $topic->getAttribute('name', ''), + $topic->getAttribute('description', ''), + ]; + + $search = \implode(' ', \array_filter($searchValues)); + + return $search; + }, + function (mixed $value) { + return $value; + } +); + +Database::addFilter( + 'messageSearch', + function (mixed $value, Document $message) { + $searchValues = [ + $message->getId(), + $message->getAttribute('description', ''), + $message->getAttribute('status', ''), + ]; + + $data = \json_decode($message->getAttribute('data', []), true); + $providerType = $message->getAttribute('providerType', ''); + + if ($providerType === MESSAGE_TYPE_EMAIL) { + $searchValues = \array_merge($searchValues, [$data['subject'], MESSAGE_TYPE_EMAIL]); + } elseif ($providerType === MESSAGE_TYPE_SMS) { + $searchValues = \array_merge($searchValues, [$data['content'], MESSAGE_TYPE_SMS]); + } else { + $searchValues = \array_merge($searchValues, [$data['title'], MESSAGE_TYPE_PUSH]); + } + + $search = \implode(' ', \array_filter($searchValues)); + + return $search; + }, + function (mixed $value) { + return $value; + } +); diff --git a/app/init/database/formats.php b/app/init/database/formats.php new file mode 100644 index 0000000000..9a22b9ed2c --- /dev/null +++ b/app/init/database/formats.php @@ -0,0 +1,43 @@ +set('logger', function () { + // Register error logger + $providerName = System::getEnv('_APP_LOGGING_PROVIDER', ''); + $providerConfig = System::getEnv('_APP_LOGGING_CONFIG', ''); + + try { + $loggingProvider = new DSN($providerConfig ?? ''); + + $providerName = $loggingProvider->getScheme(); + $providerConfig = match ($providerName) { + 'sentry' => ['key' => $loggingProvider->getPassword(), 'projectId' => $loggingProvider->getUser() ?? '', 'host' => 'https://' . $loggingProvider->getHost()], + 'logowl' => ['ticket' => $loggingProvider->getUser() ?? '', 'host' => $loggingProvider->getHost()], + default => ['key' => $loggingProvider->getHost()], + }; + } catch (Throwable $th) { + // Fallback for older Appwrite versions up to 1.5.x that use _APP_LOGGING_PROVIDER and _APP_LOGGING_CONFIG environment variables + Console::warning('Using deprecated logging configuration. Please update your configuration to use DSN format.' . $th->getMessage()); + $configChunks = \explode(";", $providerConfig); + + $providerConfig = match ($providerName) { + 'sentry' => [ 'key' => $configChunks[0], 'projectId' => $configChunks[1] ?? '', 'host' => '',], + 'logowl' => ['ticket' => $configChunks[0] ?? '', 'host' => ''], + default => ['key' => $providerConfig], + }; + } + + if (empty($providerName) || empty($providerConfig)) { + return; + } + + if (!Logger::hasProvider($providerName)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Logging provider not supported. Logging is disabled"); + } + + try { + $adapter = match ($providerName) { + 'sentry' => new Sentry($providerConfig['projectId'], $providerConfig['key'], $providerConfig['host']), + 'logowl' => new LogOwl($providerConfig['ticket'], $providerConfig['host']), + 'raygun' => new Raygun($providerConfig['key']), + 'appsignal' => new AppSignal($providerConfig['key']), + default => null + }; + } catch (Throwable $th) { + $adapter = null; + } + + if ($adapter === null) { + Console::error("Logging provider not supported. Logging is disabled"); + return; + } + + return new Logger($adapter); +}); + +$register->set('pools', function () { + $group = new Group(); + + $fallbackForDB = 'db_main=' . AppwriteURL::unparse([ + 'scheme' => 'mariadb', + 'host' => System::getEnv('_APP_DB_HOST', 'mariadb'), + 'port' => System::getEnv('_APP_DB_PORT', '3306'), + 'user' => System::getEnv('_APP_DB_USER', ''), + 'pass' => System::getEnv('_APP_DB_PASS', ''), + 'path' => System::getEnv('_APP_DB_SCHEMA', ''), + ]); + $fallbackForRedis = 'redis_main=' . AppwriteURL::unparse([ + 'scheme' => 'redis', + 'host' => System::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => System::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => System::getEnv('_APP_REDIS_USER', ''), + 'pass' => System::getEnv('_APP_REDIS_PASS', ''), + ]); + + $connections = [ + 'console' => [ + 'type' => 'database', + 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), + 'multiple' => false, + 'schemes' => ['mariadb', 'mysql'], + ], + 'database' => [ + 'type' => 'database', + 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), + 'multiple' => true, + 'schemes' => ['mariadb', 'mysql'], + ], + 'queue' => [ + 'type' => 'queue', + 'dsns' => System::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'pubsub' => [ + 'type' => 'pubsub', + 'dsns' => System::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'cache' => [ + 'type' => 'cache', + 'dsns' => System::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), + 'multiple' => true, + 'schemes' => ['redis'], + ], + ]; + + $maxConnections = System::getEnv('_APP_CONNECTIONS_MAX', 151); + $instanceConnections = $maxConnections / System::getEnv('_APP_POOL_CLIENTS', 14); + + $multiprocessing = System::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; + + if ($multiprocessing) { + $workerCount = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); + } else { + $workerCount = 1; + } + + if ($workerCount > $instanceConnections) { + throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); + } + + $poolSize = (int)($instanceConnections / $workerCount); + + foreach ($connections as $key => $connection) { + $type = $connection['type'] ?? ''; + $multiple = $connection['multiple'] ?? false; + $schemes = $connection['schemes'] ?? []; + $config = []; + $dsns = explode(',', $connection['dsns'] ?? ''); + foreach ($dsns as &$dsn) { + $dsn = explode('=', $dsn); + $name = ($multiple) ? $key . '_' . $dsn[0] : $key; + $dsn = $dsn[1] ?? ''; + $config[] = $name; + if (empty($dsn)) { + //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); + continue; + } + + $dsn = new DSN($dsn); + $dsnHost = $dsn->getHost(); + $dsnPort = $dsn->getPort(); + $dsnUser = $dsn->getUser(); + $dsnPass = $dsn->getPassword(); + $dsnScheme = $dsn->getScheme(); + $dsnDatabase = $dsn->getPath(); + + if (!in_array($dsnScheme, $schemes)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); + } + + /** + * Get Resource + * + * Creation could be reused across connection types like database, cache, queue, etc. + * + * Resource assignment to an adapter will happen below. + */ + $resource = match ($dsnScheme) { + 'mysql', + 'mariadb' => function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + }); + }, + 'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) { + $redis = new Redis(); + @$redis->pconnect($dsnHost, (int)$dsnPort); + if ($dsnPass) { + $redis->auth($dsnPass); + } + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + return $redis; + }, + default => throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Invalid scheme'), + }; + + $pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) { + // Get Adapter + switch ($type) { + case 'database': + $adapter = match ($dsn->getScheme()) { + 'mariadb' => new MariaDB($resource()), + 'mysql' => new MySQL($resource()), + default => null + }; + + $adapter->setDatabase($dsn->getPath()); + break; + case 'pubsub': + $adapter = $resource(); + break; + case 'queue': + $adapter = match ($dsn->getScheme()) { + 'redis' => new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()), + default => null + }; + break; + case 'cache': + $adapter = match ($dsn->getScheme()) { + 'redis' => new RedisCache($resource()), + default => null + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); + } + + return $adapter; + }); + + $group->add($pool); + } + + Config::setParam('pools-' . $key, $config); + } + + return $group; +}); + +$register->set('db', function () { + // This is usually for our workers or CLI commands scope + $dbHost = System::getEnv('_APP_DB_HOST', ''); + $dbPort = System::getEnv('_APP_DB_PORT', ''); + $dbUser = System::getEnv('_APP_DB_USER', ''); + $dbPass = System::getEnv('_APP_DB_PASS', ''); + $dbScheme = System::getEnv('_APP_DB_SCHEMA', ''); + + return new PDO( + "mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", + $dbUser, + $dbPass, + SQL::getPDOAttributes() + ); +}); + +$register->set('smtp', function () { + $mail = new PHPMailer(true); + + $mail->isSMTP(); + + $username = System::getEnv('_APP_SMTP_USERNAME'); + $password = System::getEnv('_APP_SMTP_PASSWORD'); + + $mail->XMailer = 'Appwrite Mailer'; + $mail->Host = System::getEnv('_APP_SMTP_HOST', 'smtp'); + $mail->Port = System::getEnv('_APP_SMTP_PORT', 25); + $mail->SMTPAuth = !empty($username) && !empty($password); + $mail->Username = $username; + $mail->Password = $password; + $mail->SMTPSecure = System::getEnv('_APP_SMTP_SECURE', ''); + $mail->SMTPAutoTLS = false; + $mail->CharSet = 'UTF-8'; + + $from = \urldecode(System::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server')); + $email = System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM); + + $mail->setFrom($email, $from); + $mail->addReplyTo($email, $from); + + $mail->isHTML(true); + + return $mail; +}); + +$register->set('geodb', function () { + return new Reader(__DIR__ . '/../assets/dbip/dbip-country-lite-2024-09.mmdb'); +}); + +$register->set('passwordsDictionary', function () { + $content = \file_get_contents(__DIR__ . '/../assets/security/10k-common-passwords'); + $content = explode("\n", $content); + $content = array_flip($content); + return $content; +}); + +$register->set('promiseAdapter', function () { + return new Swoole(); +}); + +$register->set('hooks', function () { + return new Hooks(); +}); diff --git a/app/init/resources.php b/app/init/resources.php new file mode 100644 index 0000000000..22a4daf333 --- /dev/null +++ b/app/init/resources.php @@ -0,0 +1,732 @@ + new Log()); +App::setResource('logger', function ($register) { + return $register->get('logger'); +}, ['register']); + +App::setResource('hooks', function ($register) { + return $register->get('hooks'); +}, ['register']); + +App::setResource('register', fn () => $register); +App::setResource('locale', fn () => new Locale(System::getEnv('_APP_LOCALE', 'en'))); + +App::setResource('localeCodes', function () { + return array_map(fn ($locale) => $locale['code'], Config::getParam('locale-codes', [])); +}); + +// Queues +App::setResource('queue', function (Group $pools) { + return $pools->get('queue')->pop()->getResource(); +}, ['pools']); +App::setResource('queueForMessaging', function (Connection $queue) { + return new Messaging($queue); +}, ['queue']); +App::setResource('queueForMails', function (Connection $queue) { + return new Mail($queue); +}, ['queue']); +App::setResource('queueForBuilds', function (Connection $queue) { + return new Build($queue); +}, ['queue']); +App::setResource('queueForDatabase', function (Connection $queue) { + return new EventDatabase($queue); +}, ['queue']); +App::setResource('queueForDeletes', function (Connection $queue) { + return new Delete($queue); +}, ['queue']); +App::setResource('queueForEvents', function (Connection $queue) { + return new Event($queue); +}, ['queue']); +App::setResource('queueForAudits', function (Connection $queue) { + return new Audit($queue); +}, ['queue']); +App::setResource('queueForFunctions', function (Connection $queue) { + return new Func($queue); +}, ['queue']); +App::setResource('queueForUsage', function (Connection $queue) { + return new Usage($queue); +}, ['queue']); +App::setResource('queueForCertificates', function (Connection $queue) { + return new Certificate($queue); +}, ['queue']); +App::setResource('queueForMigrations', function (Connection $queue) { + return new Migration($queue); +}, ['queue']); +App::setResource('clients', function ($request, $console, $project) { + $console->setAttribute('platforms', [ // Always allow current host + '$collection' => ID::custom('platforms'), + 'name' => 'Current Host', + 'type' => Origin::CLIENT_TYPE_WEB, + 'hostname' => $request->getHostname(), + ], Document::SET_TYPE_APPEND); + + $hostnames = explode(',', System::getEnv('_APP_CONSOLE_HOSTNAMES', '')); + $validator = new Hostname(); + foreach ($hostnames as $hostname) { + $hostname = trim($hostname); + if (!$validator->isValid($hostname)) { + continue; + } + $console->setAttribute('platforms', [ + '$collection' => ID::custom('platforms'), + 'type' => Origin::CLIENT_TYPE_WEB, + 'name' => $hostname, + 'hostname' => $hostname, + ], Document::SET_TYPE_APPEND); + } + + /** + * Get All verified client URLs for both console and current projects + * + Filter for duplicated entries + */ + $clientsConsole = \array_map( + fn ($node) => $node['hostname'], + \array_filter( + $console->getAttribute('platforms', []), + fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB) && !empty($node['hostname'])) + ) + ); + + $clients = $clientsConsole; + $platforms = $project->getAttribute('platforms', []); + + foreach ($platforms as $node) { + if ( + isset($node['type']) && + ($node['type'] === Origin::CLIENT_TYPE_WEB || + $node['type'] === Origin::CLIENT_TYPE_FLUTTER_WEB) && + !empty($node['hostname']) + ) { + $clients[] = $node['hostname']; + } + } + + return \array_unique($clients); +}, ['request', 'console', 'project']); + +App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForConsole) { + /** @var Appwrite\Utopia\Request $request */ + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Document $project */ + /** @var Utopia\Database\Database $dbForProject */ + /** @var Utopia\Database\Database $dbForConsole */ + /** @var string $mode */ + + Authorization::setDefaultStatus(true); + + Auth::setCookieName('a_session_' . $project->getId()); + + if (APP_MODE_ADMIN === $mode) { + Auth::setCookieName('a_session_' . $console->getId()); + } + + $session = Auth::decodeSession( + $request->getCookie( + Auth::$cookieName, // Get sessions + $request->getCookie(Auth::$cookieName . '_legacy', '') + ) + ); + + // Get session from header for SSR clients + if (empty($session['id']) && empty($session['secret'])) { + $sessionHeader = $request->getHeader('x-appwrite-session', ''); + + if (!empty($sessionHeader)) { + $session = Auth::decodeSession($sessionHeader); + } + } + + // Get fallback session from old clients (no SameSite support) or clients who block 3rd-party cookies + if ($response) { + $response->addHeader('X-Debug-Fallback', 'false'); + } + + if (empty($session['id']) && empty($session['secret'])) { + if ($response) { + $response->addHeader('X-Debug-Fallback', 'true'); + } + $fallback = $request->getHeader('x-fallback-cookies', ''); + $fallback = \json_decode($fallback, true); + $session = Auth::decodeSession(((isset($fallback[Auth::$cookieName])) ? $fallback[Auth::$cookieName] : '')); + } + + Auth::$unique = $session['id'] ?? ''; + Auth::$secret = $session['secret'] ?? ''; + + if (APP_MODE_ADMIN !== $mode) { + if ($project->isEmpty()) { + $user = new Document([]); + } else { + if ($project->getId() === 'console') { + $user = $dbForConsole->getDocument('users', Auth::$unique); + } else { + $user = $dbForProject->getDocument('users', Auth::$unique); + } + } + } else { + $user = $dbForConsole->getDocument('users', Auth::$unique); + } + + if ( + $user->isEmpty() // Check a document has been found in the DB + || !Auth::sessionVerify($user->getAttribute('sessions', []), Auth::$secret) + ) { // Validate user has valid login token + $user = new Document([]); + } + + if (APP_MODE_ADMIN === $mode) { + if ($user->find('teamInternalId', $project->getAttribute('teamInternalId'), 'memberships')) { + Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users. + } else { + $user = new Document([]); + } + } + + $authJWT = $request->getHeader('x-appwrite-jwt', ''); + + if (!empty($authJWT) && !$project->isEmpty()) { // JWT authentication + $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0); + + try { + $payload = $jwt->decode($authJWT); + } catch (JWTException $error) { + throw new Exception(Exception::USER_JWT_INVALID, 'Failed to verify JWT. ' . $error->getMessage()); + } + + $jwtUserId = $payload['userId'] ?? ''; + if (!empty($jwtUserId)) { + $user = $dbForProject->getDocument('users', $jwtUserId); + } + + $jwtSessionId = $payload['sessionId'] ?? ''; + if (!empty($jwtSessionId)) { + if (empty($user->find('$id', $jwtSessionId, 'sessions'))) { // Match JWT to active token + $user = new Document([]); + } + } + } + + $dbForProject->setMetadata('user', $user->getId()); + $dbForConsole->setMetadata('user', $user->getId()); + + return $user; +}, ['mode', 'project', 'console', 'request', 'response', 'dbForProject', 'dbForConsole']); + +App::setResource('project', function ($dbForConsole, $request, $console) { + /** @var Appwrite\Utopia\Request $request */ + /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Document $console */ + + $projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', '')); + + if (empty($projectId) || $projectId === 'console') { + return $console; + } + + $project = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $projectId)); + + return $project; +}, ['dbForConsole', 'request', 'console']); + +App::setResource('session', function (Document $user) { + if ($user->isEmpty()) { + return; + } + + $sessions = $user->getAttribute('sessions', []); + $sessionId = Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret); + + if (!$sessionId) { + return; + } + + foreach ($sessions as $session) {/** @var Document $session */ + if ($sessionId === $session->getId()) { + return $session; + } + } + + return; +}, ['user']); + +App::setResource('console', function () { + return new Document([ + '$id' => ID::custom('console'), + '$internalId' => ID::custom('console'), + 'name' => 'Appwrite', + '$collection' => ID::custom('projects'), + 'description' => 'Appwrite core engine', + 'logo' => '', + 'teamId' => -1, + 'webhooks' => [], + 'keys' => [], + 'platforms' => [ + [ + '$collection' => ID::custom('platforms'), + 'name' => 'Localhost', + 'type' => Origin::CLIENT_TYPE_WEB, + 'hostname' => 'localhost', + ], // Current host is added on app init + ], + 'legalName' => '', + 'legalCountry' => '', + 'legalState' => '', + 'legalCity' => '', + 'legalAddress' => '', + 'legalTaxId' => '', + 'auths' => [ + 'mockNumbers' => [], + 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', + 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user + 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds + 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' + ], + 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], + 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], + 'oAuthProviders' => [ + 'githubEnabled' => true, + 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), + 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') + ], + ]); +}, []); + +App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForConsole; + } + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + $dbAdapter = $pools + ->get($dsn->getHost()) + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); + + $database + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()) + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + + return $database; +}, ['pools', 'dbForConsole', 'cache', 'project']); + +App::setResource('dbForConsole', function (Group $pools, Cache $cache) { + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); + + $database + ->setNamespace('_console') + ->setMetadata('host', \gethostname()) + ->setMetadata('project', 'console') + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); + + return $database; +}, ['pools', 'cache']); + +App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { + $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools + + return function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForConsole; + } + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + $configure = (function (Database $database) use ($project, $dsn) { + $database + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()) + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); + + if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + }); + + if (isset($databases[$dsn->getHost()])) { + $database = $databases[$dsn->getHost()]; + $configure($database); + return $database; + } + + $dbAdapter = $pools + ->get($dsn->getHost()) + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); + $databases[$dsn->getHost()] = $database; + $configure($database); + + return $database; + }; +}, ['pools', 'dbForConsole', 'cache']); + +App::setResource('cache', function (Group $pools) { + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +}, ['pools']); + +App::setResource('deviceForLocal', function () { + return new Local(); +}); + +App::setResource('deviceForFiles', function ($project) { + return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); +}, ['project']); + +App::setResource('deviceForFunctions', function ($project) { + return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); +}, ['project']); + +App::setResource('deviceForBuilds', function ($project) { + return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); +}, ['project']); + +function getDevice($root): Device +{ + $connection = System::getEnv('_APP_CONNECTIONS_STORAGE', ''); + + if (!empty($connection)) { + $acl = 'private'; + $device = Storage::DEVICE_LOCAL; + $accessKey = ''; + $accessSecret = ''; + $bucket = ''; + $region = ''; + + try { + $dsn = new DSN($connection); + $device = $dsn->getScheme(); + $accessKey = $dsn->getUser() ?? ''; + $accessSecret = $dsn->getPassword() ?? ''; + $bucket = $dsn->getPath() ?? ''; + $region = $dsn->getParam('region'); + } catch (\Throwable $e) { + Console::warning($e->getMessage() . 'Invalid DSN. Defaulting to Local device.'); + } + + switch ($device) { + case Storage::DEVICE_S3: + return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE::DEVICE_DO_SPACES: + $device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); + $device->setHttpVersion(S3::HTTP_VERSION_1_1); + return $device; + case Storage::DEVICE_BACKBLAZE: + return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case Storage::DEVICE_LINODE: + return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case Storage::DEVICE_WASABI: + return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case Storage::DEVICE_LOCAL: + default: + return new Local($root); + } + } else { + switch (strtolower(System::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) ?? '')) { + case Storage::DEVICE_LOCAL: + default: + return new Local($root); + case Storage::DEVICE_S3: + $s3AccessKey = System::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); + $s3SecretKey = System::getEnv('_APP_STORAGE_S3_SECRET', ''); + $s3Region = System::getEnv('_APP_STORAGE_S3_REGION', ''); + $s3Bucket = System::getEnv('_APP_STORAGE_S3_BUCKET', ''); + $s3Acl = 'private'; + return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + case Storage::DEVICE_DO_SPACES: + $doSpacesAccessKey = System::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); + $doSpacesSecretKey = System::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); + $doSpacesRegion = System::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); + $doSpacesBucket = System::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); + $doSpacesAcl = 'private'; + $device = new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + $device->setHttpVersion(S3::HTTP_VERSION_1_1); + return $device; + case Storage::DEVICE_BACKBLAZE: + $backblazeAccessKey = System::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); + $backblazeSecretKey = System::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); + $backblazeRegion = System::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); + $backblazeBucket = System::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); + $backblazeAcl = 'private'; + return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); + case Storage::DEVICE_LINODE: + $linodeAccessKey = System::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); + $linodeSecretKey = System::getEnv('_APP_STORAGE_LINODE_SECRET', ''); + $linodeRegion = System::getEnv('_APP_STORAGE_LINODE_REGION', ''); + $linodeBucket = System::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); + $linodeAcl = 'private'; + return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); + case Storage::DEVICE_WASABI: + $wasabiAccessKey = System::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); + $wasabiSecretKey = System::getEnv('_APP_STORAGE_WASABI_SECRET', ''); + $wasabiRegion = System::getEnv('_APP_STORAGE_WASABI_REGION', ''); + $wasabiBucket = System::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); + $wasabiAcl = 'private'; + return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); + } + } +} + +App::setResource('mode', function ($request) { + /** @var Appwrite\Utopia\Request $request */ + + /** + * Defines the mode for the request: + * - 'default' => Requests for Client and Server Side + * - 'admin' => Request from the Console on non-console projects + */ + return $request->getParam('mode', $request->getHeader('x-appwrite-mode', APP_MODE_DEFAULT)); +}, ['request']); + +App::setResource('geodb', function ($register) { + /** @var Utopia\Registry\Registry $register */ + return $register->get('geodb'); +}, ['register']); + +App::setResource('passwordsDictionary', function ($register) { + /** @var Utopia\Registry\Registry $register */ + return $register->get('passwordsDictionary'); +}, ['register']); + + +App::setResource('servers', function () { + $platforms = Config::getParam('platforms'); + $server = $platforms[APP_PLATFORM_SERVER]; + + $languages = array_map(function ($language) { + return strtolower($language['name']); + }, $server['sdks']); + + return $languages; +}); + +App::setResource('promiseAdapter', function ($register) { + return $register->get('promiseAdapter'); +}, ['register']); + +App::setResource('schema', function ($utopia, $dbForProject) { + + $complexity = function (int $complexity, array $args) { + $queries = Query::parseQueries($args['queries'] ?? []); + $query = Query::getByType($queries, [Query::TYPE_LIMIT])[0] ?? null; + $limit = $query ? $query->getValue() : APP_LIMIT_LIST_DEFAULT; + + return $complexity * $limit; + }; + + $attributes = function (int $limit, int $offset) use ($dbForProject) { + $attrs = Authorization::skip(fn () => $dbForProject->find('attributes', [ + Query::limit($limit), + Query::offset($offset), + ])); + + return \array_map(function ($attr) { + return $attr->getArrayCopy(); + }, $attrs); + }; + + $urls = [ + 'list' => function (string $databaseId, string $collectionId, array $args) { + return "/v1/databases/$databaseId/collections/$collectionId/documents"; + }, + 'create' => function (string $databaseId, string $collectionId, array $args) { + return "/v1/databases/$databaseId/collections/$collectionId/documents"; + }, + 'read' => function (string $databaseId, string $collectionId, array $args) { + return "/v1/databases/$databaseId/collections/$collectionId/documents/{$args['documentId']}"; + }, + 'update' => function (string $databaseId, string $collectionId, array $args) { + return "/v1/databases/$databaseId/collections/$collectionId/documents/{$args['documentId']}"; + }, + 'delete' => function (string $databaseId, string $collectionId, array $args) { + return "/v1/databases/$databaseId/collections/$collectionId/documents/{$args['documentId']}"; + }, + ]; + + $params = [ + 'list' => function (string $databaseId, string $collectionId, array $args) { + return [ 'queries' => $args['queries']]; + }, + 'create' => function (string $databaseId, string $collectionId, array $args) { + $id = $args['id'] ?? 'unique()'; + $permissions = $args['permissions'] ?? null; + + unset($args['id']); + unset($args['permissions']); + + // Order must be the same as the route params + return [ + 'databaseId' => $databaseId, + 'documentId' => $id, + 'collectionId' => $collectionId, + 'data' => $args, + 'permissions' => $permissions, + ]; + }, + 'update' => function (string $databaseId, string $collectionId, array $args) { + $documentId = $args['id']; + $permissions = $args['permissions'] ?? null; + + unset($args['id']); + unset($args['permissions']); + + // Order must be the same as the route params + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'documentId' => $documentId, + 'data' => $args, + 'permissions' => $permissions, + ]; + }, + ]; + + return Schema::build( + $utopia, + $complexity, + $attributes, + $urls, + $params, + ); +}, ['utopia', 'dbForProject']); + +App::setResource('contributors', function () { + $path = 'app/config/contributors.json'; + $list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : []; + return $list; +}); + +App::setResource('employees', function () { + $path = 'app/config/employees.json'; + $list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : []; + return $list; +}); + +App::setResource('heroes', function () { + $path = 'app/config/heroes.json'; + $list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : []; + return $list; +}); + +App::setResource('gitHub', function (Cache $cache) { + return new VcsGitHub($cache); +}, ['cache']); + +App::setResource('requestTimestamp', function ($request) { + //TODO: Move this to the Request class itself + $timestampHeader = $request->getHeader('x-appwrite-timestamp'); + $requestTimestamp = null; + if (!empty($timestampHeader)) { + try { + $requestTimestamp = new \DateTime($timestampHeader); + } catch (\Throwable $e) { + throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Invalid X-Appwrite-Timestamp header value'); + } + } + return $requestTimestamp; +}, ['request']); +App::setResource('plan', function (array $plan = []) { + return []; +}); diff --git a/composer.json b/composer.json index 38ead1fbcd..e09c260d5d 100644 --- a/composer.json +++ b/composer.json @@ -96,6 +96,10 @@ "config": { "platform": { "php": "8.3" + }, + "allow-plugins": { + "php-http/discovery": true, + "tbachert/spi": true } } } From 4f0a7de868704c6c336b3a8dff2fe6a0099389f8 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Mar 2025 12:34:49 +0100 Subject: [PATCH 160/275] Fixes --- app/init/config.php | 1 + app/init/constants.php | 2 ++ app/init/locale.php | 3 --- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/init/config.php b/app/init/config.php index 72e0e5fd7f..c329828098 100644 --- a/app/init/config.php +++ b/app/init/config.php @@ -1,5 +1,6 @@ Date: Sat, 15 Mar 2025 12:39:15 +0100 Subject: [PATCH 161/275] Code review fix --- app/init/database/formats.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init/database/formats.php b/app/init/database/formats.php index 9a22b9ed2c..6c73877576 100644 --- a/app/init/database/formats.php +++ b/app/init/database/formats.php @@ -18,7 +18,7 @@ Structure::addFormat(APP_DATABASE_ATTRIBUTE_DATETIME, function () { }, Database::VAR_DATETIME); Structure::addFormat(APP_DATABASE_ATTRIBUTE_ENUM, function ($attribute) { - $elements = $attribute['formatOptions']['elements']; + $elements = $attribute['formatOptions']['elements'] ?? []; return new WhiteList($elements, true); }, Database::VAR_STRING); From 3b7aea22c2cbebcb452eb8e756e61ae5a6355526 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Mar 2025 12:42:34 +0100 Subject: [PATCH 162/275] Fixed test --- app/init/config.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/init/config.php b/app/init/config.php index c329828098..72e0e5fd7f 100644 --- a/app/init/config.php +++ b/app/init/config.php @@ -1,6 +1,5 @@ Date: Sat, 15 Mar 2025 12:58:41 +0100 Subject: [PATCH 163/275] Fixed DB test --- tests/e2e/Services/Databases/DatabasesBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 04f2dbd8c8..3ad553a11d 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -1300,7 +1300,7 @@ trait DatabasesBase ]); $this->assertEquals(400, $unknown['headers']['status-code']); - $this->assertEquals('Unknown attribute: Unknown', $unknown['body']['message']); + $this->assertStringContainsString('Unknown attribute: Unknown', $unknown['body']['message']); $index1 = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $data['moviesId'] . '/indexes', array_merge([ 'content-type' => 'application/json', From 650627bfb4537605f2773c78a688531b898af77c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sun, 16 Mar 2025 01:01:13 +1300 Subject: [PATCH 164/275] Improve deletes queries --- composer.lock | 28 +++++++++++------------ src/Appwrite/Platform/Workers/Deletes.php | 13 ++++++++--- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/composer.lock b/composer.lock index 23937694f8..ed19d10202 100644 --- a/composer.lock +++ b/composer.lock @@ -3705,16 +3705,16 @@ }, { "name": "utopia-php/database", - "version": "0.61.1", + "version": "0.61.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "2e0165bd14a570ec151f400ed381108e81d15b94" + "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/2e0165bd14a570ec151f400ed381108e81d15b94", - "reference": "2e0165bd14a570ec151f400ed381108e81d15b94", + "url": "https://api.github.com/repos/utopia-php/database/zipball/349fbdf4bc088f7775c7dfb8b80239a617a88436", + "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436", "shasum": "" }, "require": { @@ -3755,9 +3755,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.61.1" + "source": "https://github.com/utopia-php/database/tree/0.61.2" }, - "time": "2025-03-14T01:19:38+00:00" + "time": "2025-03-15T11:47:42+00:00" }, { "name": "utopia-php/domains", @@ -5317,16 +5317,16 @@ }, { "name": "laravel/pint", - "version": "v1.21.1", + "version": "v1.21.2", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86" + "reference": "370772e7d9e9da087678a0edf2b11b6960e40558" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/c44bffbb2334e90fba560933c45948fa4a3f3e86", - "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86", + "url": "https://api.github.com/repos/laravel/pint/zipball/370772e7d9e9da087678a0edf2b11b6960e40558", + "reference": "370772e7d9e9da087678a0edf2b11b6960e40558", "shasum": "" }, "require": { @@ -5337,9 +5337,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.70.2", - "illuminate/view": "^11.44.1", - "larastan/larastan": "^3.1.0", + "friendsofphp/php-cs-fixer": "^3.72.0", + "illuminate/view": "^11.44.2", + "larastan/larastan": "^3.2.0", "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3", @@ -5379,7 +5379,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-03-11T03:22:21+00:00" + "time": "2025-03-14T22:31:42+00:00" }, { "name": "matthiasmullie/minify", diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index dd219765c2..9b0590181a 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -387,7 +387,8 @@ class Deletes extends Action $query = [ Query::lessThan('accessedAt', $datetime), - Query::orderDesc('accessedAt') + Query::orderDesc('accessedAt'), + Query::orderDesc('$internalId'), ]; $this->deleteByGroup( @@ -415,24 +416,26 @@ class Deletes extends Action */ private function deleteUsageStats(Document $project, callable $getProjectDB, callable $getLogsDB, string $hourlyUsageRetentionDatetime): void { - /** @var \Utopia\Database\Database $dbForProject*/ + /** @var Database $dbForProject*/ $dbForProject = $getProjectDB($project); // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), + Query::orderDesc('$internalId'), Query::equal('period', ['1h']), ], $dbForProject); if ($project->getId() !== 'console') { - /** @var \Utopia\Database\Database $dbForLogs*/ + /** @var Database $dbForLogs*/ $dbForLogs = call_user_func($getLogsDB, $project); // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), + Query::orderDesc('$internalId'), Query::equal('period', ['1h']), ], $dbForLogs); } @@ -696,6 +699,7 @@ class Deletes extends Action $this->deleteByGroup('executions', [ Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), + Query::orderDesc('$internalId'), ], $dbForProject); } @@ -715,6 +719,7 @@ class Deletes extends Action $this->deleteByGroup('sessions', [ Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), + Query::orderDesc('$internalId'), ], $dbForProject); } @@ -730,6 +735,7 @@ class Deletes extends Action $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime), Query::orderDesc('timestamp'), + Query::orderDesc('$internalId'), ], $dbForPlatform); } @@ -749,6 +755,7 @@ class Deletes extends Action $this->deleteByGroup(Audit::COLLECTION, [ Query::lessThan('time', $auditRetention), Query::orderDesc('time'), + Query::orderDesc('$internalId'), ], $dbForProject); } catch (DatabaseException $e) { Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); From 9dff192508a1c85cd8d2d58201e02550ddf9e7fe Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sun, 16 Mar 2025 01:02:41 +1300 Subject: [PATCH 165/275] Revert "Improve deletes queries" This reverts commit 650627bfb4537605f2773c78a688531b898af77c. --- composer.lock | 28 +++++++++++------------ src/Appwrite/Platform/Workers/Deletes.php | 13 +++-------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/composer.lock b/composer.lock index ed19d10202..23937694f8 100644 --- a/composer.lock +++ b/composer.lock @@ -3705,16 +3705,16 @@ }, { "name": "utopia-php/database", - "version": "0.61.2", + "version": "0.61.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436" + "reference": "2e0165bd14a570ec151f400ed381108e81d15b94" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/349fbdf4bc088f7775c7dfb8b80239a617a88436", - "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436", + "url": "https://api.github.com/repos/utopia-php/database/zipball/2e0165bd14a570ec151f400ed381108e81d15b94", + "reference": "2e0165bd14a570ec151f400ed381108e81d15b94", "shasum": "" }, "require": { @@ -3755,9 +3755,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.61.2" + "source": "https://github.com/utopia-php/database/tree/0.61.1" }, - "time": "2025-03-15T11:47:42+00:00" + "time": "2025-03-14T01:19:38+00:00" }, { "name": "utopia-php/domains", @@ -5317,16 +5317,16 @@ }, { "name": "laravel/pint", - "version": "v1.21.2", + "version": "v1.21.1", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "370772e7d9e9da087678a0edf2b11b6960e40558" + "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/370772e7d9e9da087678a0edf2b11b6960e40558", - "reference": "370772e7d9e9da087678a0edf2b11b6960e40558", + "url": "https://api.github.com/repos/laravel/pint/zipball/c44bffbb2334e90fba560933c45948fa4a3f3e86", + "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86", "shasum": "" }, "require": { @@ -5337,9 +5337,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.72.0", - "illuminate/view": "^11.44.2", - "larastan/larastan": "^3.2.0", + "friendsofphp/php-cs-fixer": "^3.70.2", + "illuminate/view": "^11.44.1", + "larastan/larastan": "^3.1.0", "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3", @@ -5379,7 +5379,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-03-14T22:31:42+00:00" + "time": "2025-03-11T03:22:21+00:00" }, { "name": "matthiasmullie/minify", diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9b0590181a..dd219765c2 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -387,8 +387,7 @@ class Deletes extends Action $query = [ Query::lessThan('accessedAt', $datetime), - Query::orderDesc('accessedAt'), - Query::orderDesc('$internalId'), + Query::orderDesc('accessedAt') ]; $this->deleteByGroup( @@ -416,26 +415,24 @@ class Deletes extends Action */ private function deleteUsageStats(Document $project, callable $getProjectDB, callable $getLogsDB, string $hourlyUsageRetentionDatetime): void { - /** @var Database $dbForProject*/ + /** @var \Utopia\Database\Database $dbForProject*/ $dbForProject = $getProjectDB($project); // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc('$internalId'), Query::equal('period', ['1h']), ], $dbForProject); if ($project->getId() !== 'console') { - /** @var Database $dbForLogs*/ + /** @var \Utopia\Database\Database $dbForLogs*/ $dbForLogs = call_user_func($getLogsDB, $project); // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc('$internalId'), Query::equal('period', ['1h']), ], $dbForLogs); } @@ -699,7 +696,6 @@ class Deletes extends Action $this->deleteByGroup('executions', [ Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), - Query::orderDesc('$internalId'), ], $dbForProject); } @@ -719,7 +715,6 @@ class Deletes extends Action $this->deleteByGroup('sessions', [ Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), - Query::orderDesc('$internalId'), ], $dbForProject); } @@ -735,7 +730,6 @@ class Deletes extends Action $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime), Query::orderDesc('timestamp'), - Query::orderDesc('$internalId'), ], $dbForPlatform); } @@ -755,7 +749,6 @@ class Deletes extends Action $this->deleteByGroup(Audit::COLLECTION, [ Query::lessThan('time', $auditRetention), Query::orderDesc('time'), - Query::orderDesc('$internalId'), ], $dbForProject); } catch (DatabaseException $e) { Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); From 12dbb1d50df24780d26c0a5e0662f886d7a6cfba Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sun, 16 Mar 2025 01:02:44 +1300 Subject: [PATCH 166/275] Reapply "Improve deletes queries" This reverts commit 9dff192508a1c85cd8d2d58201e02550ddf9e7fe. --- composer.lock | 28 +++++++++++------------ src/Appwrite/Platform/Workers/Deletes.php | 13 ++++++++--- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/composer.lock b/composer.lock index 23937694f8..ed19d10202 100644 --- a/composer.lock +++ b/composer.lock @@ -3705,16 +3705,16 @@ }, { "name": "utopia-php/database", - "version": "0.61.1", + "version": "0.61.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "2e0165bd14a570ec151f400ed381108e81d15b94" + "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/2e0165bd14a570ec151f400ed381108e81d15b94", - "reference": "2e0165bd14a570ec151f400ed381108e81d15b94", + "url": "https://api.github.com/repos/utopia-php/database/zipball/349fbdf4bc088f7775c7dfb8b80239a617a88436", + "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436", "shasum": "" }, "require": { @@ -3755,9 +3755,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.61.1" + "source": "https://github.com/utopia-php/database/tree/0.61.2" }, - "time": "2025-03-14T01:19:38+00:00" + "time": "2025-03-15T11:47:42+00:00" }, { "name": "utopia-php/domains", @@ -5317,16 +5317,16 @@ }, { "name": "laravel/pint", - "version": "v1.21.1", + "version": "v1.21.2", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86" + "reference": "370772e7d9e9da087678a0edf2b11b6960e40558" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/c44bffbb2334e90fba560933c45948fa4a3f3e86", - "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86", + "url": "https://api.github.com/repos/laravel/pint/zipball/370772e7d9e9da087678a0edf2b11b6960e40558", + "reference": "370772e7d9e9da087678a0edf2b11b6960e40558", "shasum": "" }, "require": { @@ -5337,9 +5337,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.70.2", - "illuminate/view": "^11.44.1", - "larastan/larastan": "^3.1.0", + "friendsofphp/php-cs-fixer": "^3.72.0", + "illuminate/view": "^11.44.2", + "larastan/larastan": "^3.2.0", "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3", @@ -5379,7 +5379,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-03-11T03:22:21+00:00" + "time": "2025-03-14T22:31:42+00:00" }, { "name": "matthiasmullie/minify", diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index dd219765c2..9b0590181a 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -387,7 +387,8 @@ class Deletes extends Action $query = [ Query::lessThan('accessedAt', $datetime), - Query::orderDesc('accessedAt') + Query::orderDesc('accessedAt'), + Query::orderDesc('$internalId'), ]; $this->deleteByGroup( @@ -415,24 +416,26 @@ class Deletes extends Action */ private function deleteUsageStats(Document $project, callable $getProjectDB, callable $getLogsDB, string $hourlyUsageRetentionDatetime): void { - /** @var \Utopia\Database\Database $dbForProject*/ + /** @var Database $dbForProject*/ $dbForProject = $getProjectDB($project); // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), + Query::orderDesc('$internalId'), Query::equal('period', ['1h']), ], $dbForProject); if ($project->getId() !== 'console') { - /** @var \Utopia\Database\Database $dbForLogs*/ + /** @var Database $dbForLogs*/ $dbForLogs = call_user_func($getLogsDB, $project); // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), + Query::orderDesc('$internalId'), Query::equal('period', ['1h']), ], $dbForLogs); } @@ -696,6 +699,7 @@ class Deletes extends Action $this->deleteByGroup('executions', [ Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), + Query::orderDesc('$internalId'), ], $dbForProject); } @@ -715,6 +719,7 @@ class Deletes extends Action $this->deleteByGroup('sessions', [ Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), + Query::orderDesc('$internalId'), ], $dbForProject); } @@ -730,6 +735,7 @@ class Deletes extends Action $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime), Query::orderDesc('timestamp'), + Query::orderDesc('$internalId'), ], $dbForPlatform); } @@ -749,6 +755,7 @@ class Deletes extends Action $this->deleteByGroup(Audit::COLLECTION, [ Query::lessThan('time', $auditRetention), Query::orderDesc('time'), + Query::orderDesc('$internalId'), ], $dbForProject); } catch (DatabaseException $e) { Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); From 195edd19ac11b833b7a4f36e95d1aa1a97d6e0cf Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Mar 2025 13:18:34 +0100 Subject: [PATCH 167/275] wip --- app/init/config.php | 65 +++---- app/init/constants.php | 55 +++++- app/init/database/filters.php | 19 +- app/init/locale.php | 6 +- app/init/registers.php | 71 ++++---- app/init/resources.php | 315 ++++++++++++++++++++++------------ 6 files changed, 340 insertions(+), 191 deletions(-) diff --git a/app/init/config.php b/app/init/config.php index 72e0e5fd7f..99baade018 100644 --- a/app/init/config.php +++ b/app/init/config.php @@ -2,35 +2,36 @@ use Utopia\Config\Config; -Config::load('events', __DIR__ . '/../config/events.php'); -Config::load('auth', __DIR__ . '/../config/auth.php'); -Config::load('apis', __DIR__ . '/../config/apis.php'); // List of APIs -Config::load('errors', __DIR__ . '/../config/errors.php'); -Config::load('oAuthProviders', __DIR__ . '/../config/oAuthProviders.php'); -Config::load('platforms', __DIR__ . '/../config/platforms.php'); -Config::load('collections', __DIR__ . '/../config/collections.php'); -Config::load('runtimes', __DIR__ . '/../config/runtimes.php'); -Config::load('runtimes-v2', __DIR__ . '/../config/runtimes-v2.php'); -Config::load('usage', __DIR__ . '/../config/usage.php'); -Config::load('roles', __DIR__ . '/../config/roles.php'); // User roles and scopes -Config::load('scopes', __DIR__ . '/../config/scopes.php'); // User roles and scopes -Config::load('services', __DIR__ . '/../config/services.php'); // List of services -Config::load('variables', __DIR__ . '/../config/variables.php'); // List of env variables -Config::load('regions', __DIR__ . '/../config/regions.php'); // List of available regions -Config::load('avatar-browsers', __DIR__ . '/../config/avatars/browsers.php'); -Config::load('avatar-credit-cards', __DIR__ . '/../config/avatars/credit-cards.php'); -Config::load('avatar-flags', __DIR__ . '/../config/avatars/flags.php'); -Config::load('locale-codes', __DIR__ . '/../config/locale/codes.php'); -Config::load('locale-currencies', __DIR__ . '/../config/locale/currencies.php'); -Config::load('locale-eu', __DIR__ . '/../config/locale/eu.php'); -Config::load('locale-languages', __DIR__ . '/../config/locale/languages.php'); -Config::load('locale-phones', __DIR__ . '/../config/locale/phones.php'); -Config::load('locale-countries', __DIR__ . '/../config/locale/countries.php'); -Config::load('locale-continents', __DIR__ . '/../config/locale/continents.php'); -Config::load('locale-templates', __DIR__ . '/../config/locale/templates.php'); -Config::load('storage-logos', __DIR__ . '/../config/storage/logos.php'); -Config::load('storage-mimes', __DIR__ . '/../config/storage/mimes.php'); -Config::load('storage-inputs', __DIR__ . '/../config/storage/inputs.php'); -Config::load('storage-outputs', __DIR__ . '/../config/storage/outputs.php'); -Config::load('runtime-specifications', __DIR__ . '/../config/runtimes/specifications.php'); -Config::load('function-templates', __DIR__ . '/../config/function-templates.php'); +Config::load('events', __DIR__ . '/config/events.php'); +Config::load('auth', __DIR__ . '/config/auth.php'); +Config::load('apis', __DIR__ . '/config/apis.php'); // List of APIs +Config::load('errors', __DIR__ . '/config/errors.php'); +Config::load('oAuthProviders', __DIR__ . '/config/oAuthProviders.php'); +Config::load('platforms', __DIR__ . '/config/platforms.php'); +Config::load('console', __DIR__ . '/config/console.php'); +Config::load('collections', __DIR__ . '/config/collections.php'); +Config::load('runtimes', __DIR__ . '/config/runtimes.php'); +Config::load('runtimes-v2', __DIR__ . '/config/runtimes-v2.php'); +Config::load('usage', __DIR__ . '/config/usage.php'); +Config::load('roles', __DIR__ . '/config/roles.php'); // User roles and scopes +Config::load('scopes', __DIR__ . '/config/scopes.php'); // User roles and scopes +Config::load('services', __DIR__ . '/config/services.php'); // List of services +Config::load('variables', __DIR__ . '/config/variables.php'); // List of env variables +Config::load('regions', __DIR__ . '/config/regions.php'); // List of available regions +Config::load('avatar-browsers', __DIR__ . '/config/avatars/browsers.php'); +Config::load('avatar-credit-cards', __DIR__ . '/config/avatars/credit-cards.php'); +Config::load('avatar-flags', __DIR__ . '/config/avatars/flags.php'); +Config::load('locale-codes', __DIR__ . '/config/locale/codes.php'); +Config::load('locale-currencies', __DIR__ . '/config/locale/currencies.php'); +Config::load('locale-eu', __DIR__ . '/config/locale/eu.php'); +Config::load('locale-languages', __DIR__ . '/config/locale/languages.php'); +Config::load('locale-phones', __DIR__ . '/config/locale/phones.php'); +Config::load('locale-countries', __DIR__ . '/config/locale/countries.php'); +Config::load('locale-continents', __DIR__ . '/config/locale/continents.php'); +Config::load('locale-templates', __DIR__ . '/config/locale/templates.php'); +Config::load('storage-logos', __DIR__ . '/config/storage/logos.php'); +Config::load('storage-mimes', __DIR__ . '/config/storage/mimes.php'); +Config::load('storage-inputs', __DIR__ . '/config/storage/inputs.php'); +Config::load('storage-outputs', __DIR__ . '/config/storage/outputs.php'); +Config::load('runtime-specifications', __DIR__ . '/config/runtimes/specifications.php'); +Config::load('function-templates', __DIR__ . '/config/function-templates.php'); diff --git a/app/init/constants.php b/app/init/constants.php index a1c4387d7a..3383b28f57 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -29,9 +29,10 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCESS = 24 * 60 * 60; // 24 hours const APP_PROJECT_ACCESS = 24 * 60 * 60; // 24 hours +const APP_FILE_ACCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 4318; -const APP_VERSION_STABLE = '1.6.0'; +const APP_VERSION_STABLE = '1.6.2'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; @@ -40,7 +41,10 @@ const APP_DATABASE_ATTRIBUTE_URL = 'url'; const APP_DATABASE_ATTRIBUTE_INT_RANGE = 'intRange'; const APP_DATABASE_ATTRIBUTE_FLOAT_RANGE = 'floatRange'; const APP_DATABASE_ATTRIBUTE_STRING_MAX_LENGTH = 1_073_741_824; // 2^32 bits / 4 bits per char -const APP_DATABASE_TIMEOUT_MILLISECONDS = 15_000; +const APP_DATABASE_TIMEOUT_MILLISECONDS_API = 15 * 1000; // 15 seconds +const APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER = 300 * 1000; // 5 minutes +const APP_DATABASE_TIMEOUT_MILLISECONDS_TASK = 300 * 1000; // 5 minutes +const APP_DATABASE_QUERY_MAX_VALUES = 500; const APP_STORAGE_UPLOADS = '/storage/uploads'; const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_BUILDS = '/storage/builds'; @@ -60,9 +64,12 @@ const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; const APP_SOCIAL_YOUTUBE = 'https://www.youtube.com/c/appwrite?sub_confirmation=1'; const APP_HOSTNAME_INTERNAL = 'appwrite'; -const APP_FUNCTION_SPECIFICATION_DEFAULT = Specification::S_05VCPU_512MB; +const APP_FUNCTION_SPECIFICATION_DEFAULT = Specification::S_1VCPU_512MB; const APP_FUNCTION_CPUS_DEFAULT = 0.5; const APP_FUNCTION_MEMORY_DEFAULT = 512; +const APP_PLATFORM_SERVER = 'server'; +const APP_PLATFORM_CLIENT = 'client'; +const APP_PLATFORM_CONSOLE = 'console'; // Database Reconnect const DATABASE_RECONNECT_SLEEP = 2; @@ -105,6 +112,7 @@ const DELETE_TYPE_TOPIC = 'topic'; const DELETE_TYPE_TARGET = 'target'; const DELETE_TYPE_EXPIRED_TARGETS = 'invalid_targets'; const DELETE_TYPE_SESSION_TARGETS = 'session_targets'; +const DELETE_TYPE_MAINTENANCE = 'maintenance'; // Message types const MESSAGE_SEND_TYPE_INTERNAL = 'internal'; @@ -135,7 +143,10 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; - +const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; +const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; +const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; +const METRIC_WEBHOOK_ID_FAILED = '{webhookInternalId}.webhooks.events.failed'; const METRIC_AUTH_METHOD_PHONE = 'auth.method.phone'; const METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE = METRIC_AUTH_METHOD_PHONE . '.{countryCode}'; const METRIC_MESSAGES = 'messages'; @@ -150,13 +161,24 @@ const METRIC_MESSAGES_TYPE_PROVIDER_FAILED = METRIC_MESSAGES . '.{type}.{provid const METRIC_SESSIONS = 'sessions'; const METRIC_DATABASES = 'databases'; const METRIC_COLLECTIONS = 'collections'; +const METRIC_DATABASES_STORAGE = 'databases.storage'; const METRIC_DATABASE_ID_COLLECTIONS = '{databaseInternalId}.collections'; +const METRIC_DATABASE_ID_STORAGE = '{databaseInternalId}.databases.storage'; const METRIC_DOCUMENTS = 'documents'; const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents'; const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents'; +const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE = '{databaseInternalId}.{collectionInternalId}.databases.storage'; +const METRIC_DATABASES_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; +const METRIC_DATABASES_OPERATIONS_WRITES = 'databases.operations.writes'; +const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_BUCKETS = 'buckets'; const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; +const METRIC_FILES_TRANSFORMATIONS = 'files.transformations'; +const METRIC_BUCKET_ID_FILES_TRANSFORMATIONS = '{bucketInternalId}.files.transformations'; +const METRIC_FILES_IMAGES_TRANSFORMED = 'files.imagesTransformed'; +const METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED = '{bucketInternalId}.files.imagesTransformed'; const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files'; const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage'; const METRIC_FUNCTIONS = 'functions'; @@ -189,3 +211,28 @@ const METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS = '{functionInternalId}.execution const METRIC_NETWORK_REQUESTS = 'network.requests'; const METRIC_NETWORK_INBOUND = 'network.inbound'; const METRIC_NETWORK_OUTBOUND = 'network.outbound'; +const METRIC_MAU = 'users.mau'; +const METRIC_DAU = 'users.dau'; +const METRIC_WAU = 'users.wau'; +const METRIC_WEBHOOKS = 'webhooks'; +const METRIC_PLATFORMS = 'platforms'; +const METRIC_PROVIDERS = 'providers'; +const METRIC_TOPICS = 'topics'; +const METRIC_TARGETS = 'targets'; +const METRIC_PROVIDER_TYPE_TARGETS = '{providerType}.targets'; +const METRIC_KEYS = 'keys'; +const METRIC_RESOURCE_TYPE_ID_BUILDS = '{resourceType}.{resourceInternalId}.builds'; +const METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE = '{resourceType}.{resourceInternalId}.builds.storage'; +const METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments'; +const METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage'; + +// Resource types + +const RESOURCE_TYPE_PROJECTS = 'projects'; +const RESOURCE_TYPE_FUNCTIONS = 'functions'; +const RESOURCE_TYPE_DATABASES = 'databases'; +const RESOURCE_TYPE_BUCKETS = 'buckets'; +const RESOURCE_TYPE_PROVIDERS = 'providers'; +const RESOURCE_TYPE_TOPICS = 'topics'; +const RESOURCE_TYPE_SUBSCRIBERS = 'subscribers'; +const RESOURCE_TYPE_MESSAGES = 'messages'; diff --git a/app/init/database/filters.php b/app/init/database/filters.php index 241914f9ff..8223b1c677 100644 --- a/app/init/database/filters.php +++ b/app/init/database/filters.php @@ -379,12 +379,19 @@ Database::addFilter( $data = \json_decode($message->getAttribute('data', []), true); $providerType = $message->getAttribute('providerType', ''); - if ($providerType === MESSAGE_TYPE_EMAIL) { - $searchValues = \array_merge($searchValues, [$data['subject'], MESSAGE_TYPE_EMAIL]); - } elseif ($providerType === MESSAGE_TYPE_SMS) { - $searchValues = \array_merge($searchValues, [$data['content'], MESSAGE_TYPE_SMS]); - } else { - $searchValues = \array_merge($searchValues, [$data['title'], MESSAGE_TYPE_PUSH]); + switch ($providerType) { + case MESSAGE_TYPE_EMAIL: + $searchValues[] = $data['subject']; + $searchValues[] = MESSAGE_TYPE_EMAIL; + break; + case MESSAGE_TYPE_SMS: + $searchValues[] = $data['content']; + $searchValues[] = MESSAGE_TYPE_SMS; + break; + case MESSAGE_TYPE_PUSH: + $searchValues[] = $data['title'] ?? ''; + $searchValues[] = MESSAGE_TYPE_PUSH; + break; } $search = \implode(' ', \array_filter($searchValues)); diff --git a/app/init/locale.php b/app/init/locale.php index 122dc89692..333dd106e3 100644 --- a/app/init/locale.php +++ b/app/init/locale.php @@ -10,12 +10,12 @@ $locales = Config::getParam('locale-codes', []); foreach ($locales as $locale) { $code = $locale['code']; - $path = __DIR__ . '/../config/locale/translations/' . $code . '.json'; + $path = __DIR__ . '/config/locale/translations/' . $code . '.json'; if (!\file_exists($path)) { - $path = __DIR__ . '/../config/locale/translations/' . \substr($code, 0, 2) . '.json'; // if `ar-ae` doesn't exist, look for `ar` + $path = __DIR__ . '/config/locale/translations/' . \substr($code, 0, 2) . '.json'; // if `ar-ae` doesn't exist, look for `ar` if (!\file_exists($path)) { - $path = __DIR__ . '/../config/locale/translations/en.json'; // if none translation exists, use default from `en.json` + $path = __DIR__ . '/config/locale/translations/en.json'; // if none translation exists, use default from `en.json` } } diff --git a/app/init/registers.php b/app/init/registers.php index 35a309f98c..4ce8adcefd 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -3,19 +3,18 @@ use Appwrite\Extend\Exception; use Appwrite\GraphQL\Promises\Adapter\Swoole; use Appwrite\Hooks\Hooks; +use Appwrite\PubSub\Adapter\Redis as PubSub; use Appwrite\URL\URL as AppwriteURL; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; use Utopia\App; use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Adapter\MySQL; use Utopia\Database\Adapter\SQL; -use Utopia\Database\Database; use Utopia\Domains\Validator\PublicDomain; use Utopia\DSN\DSN; use Utopia\Logger\Adapter\AppSignal; @@ -26,7 +25,6 @@ use Utopia\Logger\Logger; use Utopia\Pools\Group; use Utopia\Pools\Pool; use Utopia\Queue; -use Utopia\Queue\Connection; use Utopia\Registry\Registry; use Utopia\System\System; @@ -39,12 +37,15 @@ if (!App::isProduction()) { // Useful for existing tests involving webhooks PublicDomain::allow(['request-catcher']); } - $register->set('logger', function () { // Register error logger $providerName = System::getEnv('_APP_LOGGING_PROVIDER', ''); $providerConfig = System::getEnv('_APP_LOGGING_CONFIG', ''); + if (empty($providerConfig)) { + return; + } + try { $loggingProvider = new DSN($providerConfig ?? ''); @@ -116,31 +117,43 @@ $register->set('pools', function () { $connections = [ 'console' => [ 'type' => 'database', - 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), + 'dsns' => $fallbackForDB, 'multiple' => false, 'schemes' => ['mariadb', 'mysql'], ], 'database' => [ 'type' => 'database', - 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), + 'dsns' => $fallbackForDB, 'multiple' => true, 'schemes' => ['mariadb', 'mysql'], ], - 'queue' => [ - 'type' => 'queue', - 'dsns' => System::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), + 'logs' => [ + 'type' => 'database', + 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_LOGS', $fallbackForDB), + 'multiple' => false, + 'schemes' => ['mariadb', 'mysql'], + ], + 'publisher' => [ + 'type' => 'publisher', + 'dsns' => $fallbackForRedis, + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'consumer' => [ + 'type' => 'consumer', + 'dsns' => $fallbackForRedis, 'multiple' => false, 'schemes' => ['redis'], ], 'pubsub' => [ 'type' => 'pubsub', - 'dsns' => System::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), + 'dsns' => $fallbackForRedis, 'multiple' => false, 'schemes' => ['redis'], ], 'cache' => [ 'type' => 'cache', - 'dsns' => System::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), + 'dsns' => $fallbackForRedis, 'multiple' => true, 'schemes' => ['redis'], ], @@ -152,7 +165,7 @@ $register->set('pools', function () { $multiprocessing = System::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; if ($multiprocessing) { - $workerCount = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); + $workerCount = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); } else { $workerCount = 1; } @@ -212,12 +225,12 @@ $register->set('pools', function () { }); }, 'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) { - $redis = new Redis(); + $redis = new \Redis(); @$redis->pconnect($dsnHost, (int)$dsnPort); if ($dsnPass) { $redis->auth($dsnPass); } - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); return $redis; }, @@ -235,28 +248,26 @@ $register->set('pools', function () { }; $adapter->setDatabase($dsn->getPath()); - break; + return $adapter; case 'pubsub': - $adapter = $resource(); - break; - case 'queue': - $adapter = match ($dsn->getScheme()) { - 'redis' => new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()), + return match ($dsn->getScheme()) { + 'redis' => new PubSub($resource()), + default => null + }; + case 'publisher': + case 'consumer': + return match ($dsn->getScheme()) { + 'redis' => new Queue\Broker\Redis(new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort())), default => null }; - break; case 'cache': - $adapter = match ($dsn->getScheme()) { + return match ($dsn->getScheme()) { 'redis' => new RedisCache($resource()), default => null }; - break; - default: throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); } - - return $adapter; }); $group->add($pool); @@ -312,22 +323,18 @@ $register->set('smtp', function () { return $mail; }); - $register->set('geodb', function () { - return new Reader(__DIR__ . '/../assets/dbip/dbip-country-lite-2024-09.mmdb'); + return new Reader(__DIR__ . '/assets/dbip/dbip-country-lite-2024-09.mmdb'); }); - $register->set('passwordsDictionary', function () { - $content = \file_get_contents(__DIR__ . '/../assets/security/10k-common-passwords'); + $content = \file_get_contents(__DIR__ . '/assets/security/10k-common-passwords'); $content = explode("\n", $content); $content = array_flip($content); return $content; }); - $register->set('promiseAdapter', function () { return new Swoole(); }); - $register->set('hooks', function () { return new Hooks(); }); diff --git a/app/init/resources.php b/app/init/resources.php index 22a4daf333..0895ef32a5 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -44,6 +44,7 @@ use Utopia\System\System; use Utopia\Validator\Hostname; use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub; +// Runtime Execution App::setResource('log', fn () => new Log()); App::setResource('logger', function ($register) { return $register->get('logger'); @@ -61,42 +62,51 @@ App::setResource('localeCodes', function () { }); // Queues -App::setResource('queue', function (Group $pools) { - return $pools->get('queue')->pop()->getResource(); +App::setResource('publisher', function (Group $pools) { + return $pools->get('publisher')->pop()->getResource(); }, ['pools']); -App::setResource('queueForMessaging', function (Connection $queue) { - return new Messaging($queue); -}, ['queue']); -App::setResource('queueForMails', function (Connection $queue) { - return new Mail($queue); -}, ['queue']); -App::setResource('queueForBuilds', function (Connection $queue) { - return new Build($queue); -}, ['queue']); -App::setResource('queueForDatabase', function (Connection $queue) { - return new EventDatabase($queue); -}, ['queue']); -App::setResource('queueForDeletes', function (Connection $queue) { - return new Delete($queue); -}, ['queue']); -App::setResource('queueForEvents', function (Connection $queue) { - return new Event($queue); -}, ['queue']); -App::setResource('queueForAudits', function (Connection $queue) { - return new Audit($queue); -}, ['queue']); -App::setResource('queueForFunctions', function (Connection $queue) { - return new Func($queue); -}, ['queue']); -App::setResource('queueForUsage', function (Connection $queue) { - return new Usage($queue); -}, ['queue']); -App::setResource('queueForCertificates', function (Connection $queue) { - return new Certificate($queue); -}, ['queue']); -App::setResource('queueForMigrations', function (Connection $queue) { - return new Migration($queue); -}, ['queue']); +App::setResource('consumer', function (Group $pools) { + return $pools->get('consumer')->pop()->getResource(); +}, ['pools']); +App::setResource('queueForMessaging', function (Queue\Publisher $publisher) { + return new Messaging($publisher); +}, ['publisher']); +App::setResource('queueForMails', function (Queue\Publisher $publisher) { + return new Mail($publisher); +}, ['publisher']); +App::setResource('queueForBuilds', function (Queue\Publisher $publisher) { + return new Build($publisher); +}, ['publisher']); +App::setResource('queueForDatabase', function (Queue\Publisher $publisher) { + return new EventDatabase($publisher); +}, ['publisher']); +App::setResource('queueForDeletes', function (Queue\Publisher $publisher) { + return new Delete($publisher); +}, ['publisher']); +App::setResource('queueForEvents', function (Queue\Publisher $publisher) { + return new Event($publisher); +}, ['publisher']); +App::setResource('queueForWebhooks', function (Queue\Publisher $publisher) { + return new Webhook($publisher); +}, ['publisher']); +App::setResource('queueForRealtime', function () { + return new Realtime(); +}, []); +App::setResource('queueForStatsUsage', function (Queue\Publisher $publisher) { + return new StatsUsage($publisher); +}, ['publisher']); +App::setResource('queueForAudits', function (Queue\Publisher $publisher) { + return new Audit($publisher); +}, ['publisher']); +App::setResource('queueForFunctions', function (Queue\Publisher $publisher) { + return new Func($publisher); +}, ['publisher']); +App::setResource('queueForCertificates', function (Queue\Publisher $publisher) { + return new Certificate($publisher); +}, ['publisher']); +App::setResource('queueForMigrations', function (Queue\Publisher $publisher) { + return new Migration($publisher); +}, ['publisher']); App::setResource('clients', function ($request, $console, $project) { $console->setAttribute('platforms', [ // Always allow current host '$collection' => ID::custom('platforms'), @@ -149,12 +159,12 @@ App::setResource('clients', function ($request, $console, $project) { return \array_unique($clients); }, ['request', 'console', 'project']); -App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForConsole) { +App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForPlatform) { /** @var Appwrite\Utopia\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForProject */ - /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Database $dbForPlatform */ /** @var string $mode */ Authorization::setDefaultStatus(true); @@ -203,13 +213,13 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons $user = new Document([]); } else { if ($project->getId() === 'console') { - $user = $dbForConsole->getDocument('users', Auth::$unique); + $user = $dbForPlatform->getDocument('users', Auth::$unique); } else { $user = $dbForProject->getDocument('users', Auth::$unique); } } } else { - $user = $dbForConsole->getDocument('users', Auth::$unique); + $user = $dbForPlatform->getDocument('users', Auth::$unique); } if ( @@ -219,13 +229,13 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons $user = new Document([]); } - if (APP_MODE_ADMIN === $mode) { - if ($user->find('teamInternalId', $project->getAttribute('teamInternalId'), 'memberships')) { - Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users. - } else { - $user = new Document([]); - } - } + // if (APP_MODE_ADMIN === $mode) { + // if ($user->find('teamInternalId', $project->getAttribute('teamInternalId'), 'memberships')) { + // Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users. + // } else { + // $user = new Document([]); + // } + // } $authJWT = $request->getHeader('x-appwrite-jwt', ''); @@ -252,14 +262,14 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons } $dbForProject->setMetadata('user', $user->getId()); - $dbForConsole->setMetadata('user', $user->getId()); + $dbForPlatform->setMetadata('user', $user->getId()); return $user; -}, ['mode', 'project', 'console', 'request', 'response', 'dbForProject', 'dbForConsole']); +}, ['mode', 'project', 'console', 'request', 'response', 'dbForProject', 'dbForPlatform']); -App::setResource('project', function ($dbForConsole, $request, $console) { +App::setResource('project', function ($dbForPlatform, $request, $console) { /** @var Appwrite\Utopia\Request $request */ - /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Database $dbForPlatform */ /** @var Utopia\Database\Document $console */ $projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', '')); @@ -268,10 +278,10 @@ App::setResource('project', function ($dbForConsole, $request, $console) { return $console; } - $project = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $projectId)); + $project = Authorization::skip(fn () => $dbForPlatform->getDocument('projects', $projectId)); return $project; -}, ['dbForConsole', 'request', 'console']); +}, ['dbForPlatform', 'request', 'console']); App::setResource('session', function (Document $user) { if ($user->isEmpty()) { @@ -295,50 +305,12 @@ App::setResource('session', function (Document $user) { }, ['user']); App::setResource('console', function () { - return new Document([ - '$id' => ID::custom('console'), - '$internalId' => ID::custom('console'), - 'name' => 'Appwrite', - '$collection' => ID::custom('projects'), - 'description' => 'Appwrite core engine', - 'logo' => '', - 'teamId' => -1, - 'webhooks' => [], - 'keys' => [], - 'platforms' => [ - [ - '$collection' => ID::custom('platforms'), - 'name' => 'Localhost', - 'type' => Origin::CLIENT_TYPE_WEB, - 'hostname' => 'localhost', - ], // Current host is added on app init - ], - 'legalName' => '', - 'legalCountry' => '', - 'legalState' => '', - 'legalCity' => '', - 'legalAddress' => '', - 'legalTaxId' => '', - 'auths' => [ - 'mockNumbers' => [], - 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', - 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user - 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds - 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' - ], - 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], - 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], - 'oAuthProviders' => [ - 'githubEnabled' => true, - 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), - 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') - ], - ]); + return new Document(Config::getParam('console')); }, []); -App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { +App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForConsole; + return $dbForPlatform; } try { @@ -358,16 +330,12 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, $database ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) + ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) { + if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) ->setTenant($project->getInternalId()) @@ -380,9 +348,9 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, } return $database; -}, ['pools', 'dbForConsole', 'cache', 'project']); +}, ['pools', 'dbForPlatform', 'cache', 'project']); -App::setResource('dbForConsole', function (Group $pools, Cache $cache) { +App::setResource('dbForPlatform', function (Group $pools, Cache $cache) { $dbAdapter = $pools ->get('console') ->pop() @@ -394,17 +362,18 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { ->setNamespace('_console') ->setMetadata('host', \gethostname()) ->setMetadata('project', 'console') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) + ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); return $database; }, ['pools', 'cache']); -App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { +App::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - return function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { + return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases) { if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForConsole; + return $dbForPlatform; } try { @@ -418,9 +387,12 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $database ->setMetadata('host', \gethostname()) ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) + ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) { + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) ->setTenant($project->getInternalId()) @@ -450,7 +422,40 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, return $database; }; -}, ['pools', 'dbForConsole', 'cache']); +}, ['pools', 'dbForPlatform', 'cache']); + +App::setResource('getLogsDB', function (Group $pools, Cache $cache) { + $database = null; + return function (?Document $project = null) use ($pools, $cache, $database) { + if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant($project->getInternalId()); + return $database; + } + + $dbAdapter = $pools + ->get('logs') + ->pop() + ->getResource(); + + $database = new Database( + $dbAdapter, + $cache + ); + + $database + ->setSharedTables(true) + ->setNamespace('logsV1') + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) + ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); + + // set tenant + if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant($project->getInternalId()); + } + + return $database; + }; +}, ['pools', 'cache']); App::setResource('cache', function (Group $pools) { $list = Config::getParam('pools-cache', []); @@ -467,6 +472,27 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); +App::setResource('redis', function () { + $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); + $port = System::getEnv('_APP_REDIS_PORT', 6379); + $pass = System::getEnv('_APP_REDIS_PASS', ''); + + $redis = new \Redis(); + @$redis->pconnect($host, (int)$port); + if ($pass) { + $redis->auth($pass); + } + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); + + return $redis; +}); + +App::setResource('timelimit', function (\Redis $redis) { + return function (string $key, int $limit, int $time) use ($redis) { + return new TimeLimitRedis($key, $limit, $time, $redis); + }; +}, ['redis']); + App::setResource('deviceForLocal', function () { return new Local(); }); @@ -483,9 +509,9 @@ App::setResource('deviceForBuilds', function ($project) { return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); }, ['project']); -function getDevice($root): Device +function getDevice(string $root, string $connection = ''): Device { - $connection = System::getEnv('_APP_CONNECTIONS_STORAGE', ''); + $connection = !empty($connection) ? $connection : System::getEnv('_APP_CONNECTIONS_STORAGE', ''); if (!empty($connection)) { $acl = 'private'; @@ -494,6 +520,7 @@ function getDevice($root): Device $accessSecret = ''; $bucket = ''; $region = ''; + $url = App::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); try { $dsn = new DSN($connection); @@ -508,7 +535,7 @@ function getDevice($root): Device switch ($device) { case Storage::DEVICE_S3: - return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); + return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl, $url); case STORAGE::DEVICE_DO_SPACES: $device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); $device->setHttpVersion(S3::HTTP_VERSION_1_1); @@ -534,7 +561,8 @@ function getDevice($root): Device $s3Region = System::getEnv('_APP_STORAGE_S3_REGION', ''); $s3Bucket = System::getEnv('_APP_STORAGE_S3_BUCKET', ''); $s3Acl = 'private'; - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + $s3EndpointUrl = App::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); + return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl, $s3EndpointUrl); case Storage::DEVICE_DO_SPACES: $doSpacesAccessKey = System::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); $doSpacesSecretKey = System::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); @@ -727,6 +755,65 @@ App::setResource('requestTimestamp', function ($request) { } return $requestTimestamp; }, ['request']); + App::setResource('plan', function (array $plan = []) { return []; }); + +App::setResource('smsRates', function () { + return []; +}); + +App::setResource('team', function (Document $project, Database $dbForPlatform, App $utopia, Request $request) { + $teamInternalId = ''; + if ($project->getId() !== 'console') { + $teamInternalId = $project->getAttribute('teamInternalId', ''); + } else { + $route = $utopia->match($request); + $path = $route->getPath(); + if (str_starts_with($path, '/v1/projects/:projectId')) { + $uri = $request->getURI(); + $pid = explode('/', $uri)[3]; + $p = Authorization::skip(fn () => $dbForPlatform->getDocument('projects', $pid)); + $teamInternalId = $p->getAttribute('teamInternalId', ''); + } elseif ($path === '/v1/projects') { + $teamId = $request->getParam('teamId', ''); + $team = Authorization::skip(fn () => $dbForPlatform->getDocument('teams', $teamId)); + return $team; + } + } + + $team = Authorization::skip(function () use ($dbForPlatform, $teamInternalId) { + return $dbForPlatform->findOne('teams', [ + Query::equal('$internalId', [$teamInternalId]), + ]); + }); + + return $team; +}, ['project', 'dbForPlatform', 'utopia', 'request']); + +App::setResource( + 'isResourceBlocked', + fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false +); + +App::setResource('previewHostname', function (Request $request) { + if (App::isDevelopment()) { + $host = $request->getQuery('appwrite-hostname') ?? ''; + if (!empty($host)) { + return $host; + } + } + + return ''; +}, ['request']); + +App::setResource('apiKey', function (Request $request, Document $project): ?Key { + $key = $request->getHeader('x-appwrite-key'); + + if (empty($key)) { + return null; + } + + return Key::decode($project, $key); +}, ['request', 'project']); From a2a0a1405f71582ba3a9bb8d5476e378587f0b0a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Mar 2025 13:23:02 +0100 Subject: [PATCH 168/275] plurals --- app/init.php | 4 ++-- app/init/{config.php => configs.php} | 0 app/init/{locale.php => locales.php} | 0 app/init/resources.php | 2 -- 4 files changed, 2 insertions(+), 4 deletions(-) rename app/init/{config.php => configs.php} (100%) rename app/init/{locale.php => locales.php} (100%) diff --git a/app/init.php b/app/init.php index 15ef1f6361..fd1a67b28d 100644 --- a/app/init.php +++ b/app/init.php @@ -19,10 +19,10 @@ if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { \error_reporting(E_ALL); require_once __DIR__ . '/init/constants.php'; -require_once __DIR__ . '/init/config.php'; +require_once __DIR__ . '/init/configs.php'; require_once __DIR__ . '/init/database/filters.php'; require_once __DIR__ . '/init/database/formats.php'; -require_once __DIR__ . '/init/locale.php'; +require_once __DIR__ . '/init/locales.php'; require_once __DIR__ . '/init/registers.php'; require_once __DIR__ . '/init/resources.php'; diff --git a/app/init/config.php b/app/init/configs.php similarity index 100% rename from app/init/config.php rename to app/init/configs.php diff --git a/app/init/locale.php b/app/init/locales.php similarity index 100% rename from app/init/locale.php rename to app/init/locales.php diff --git a/app/init/resources.php b/app/init/resources.php index 0895ef32a5..0764ed6e76 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -13,7 +13,6 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; -use Appwrite\Event\Usage; use Appwrite\Extend\Exception; use Appwrite\GraphQL\Schema; use Appwrite\Network\Validator\Origin; @@ -31,7 +30,6 @@ use Utopia\DSN\DSN; use Utopia\Locale\Locale; use Utopia\Logger\Log; use Utopia\Pools\Group; -use Utopia\Queue\Connection; use Utopia\Storage\Device; use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\DOSpaces; From 07926534bf04dced17d0576c32199a21550d17b3 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Mar 2025 13:24:56 +0100 Subject: [PATCH 169/275] Fixed namespace --- app/init.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/init.php b/app/init.php index fd1a67b28d..c32f1eb9a8 100644 --- a/app/init.php +++ b/app/init.php @@ -8,6 +8,8 @@ * */ +use Utopia\System\System; + if (\file_exists(__DIR__ . '/../vendor/autoload.php')) { require_once __DIR__ . '/../vendor/autoload.php'; } From 875a60fe0797c60a4d692c01bb9e99126a883df9 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Mar 2025 13:53:00 +0100 Subject: [PATCH 170/275] fixed paths --- app/init/configs.php | 66 ++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/app/init/configs.php b/app/init/configs.php index 99baade018..7d2d858351 100644 --- a/app/init/configs.php +++ b/app/init/configs.php @@ -2,36 +2,36 @@ use Utopia\Config\Config; -Config::load('events', __DIR__ . '/config/events.php'); -Config::load('auth', __DIR__ . '/config/auth.php'); -Config::load('apis', __DIR__ . '/config/apis.php'); // List of APIs -Config::load('errors', __DIR__ . '/config/errors.php'); -Config::load('oAuthProviders', __DIR__ . '/config/oAuthProviders.php'); -Config::load('platforms', __DIR__ . '/config/platforms.php'); -Config::load('console', __DIR__ . '/config/console.php'); -Config::load('collections', __DIR__ . '/config/collections.php'); -Config::load('runtimes', __DIR__ . '/config/runtimes.php'); -Config::load('runtimes-v2', __DIR__ . '/config/runtimes-v2.php'); -Config::load('usage', __DIR__ . '/config/usage.php'); -Config::load('roles', __DIR__ . '/config/roles.php'); // User roles and scopes -Config::load('scopes', __DIR__ . '/config/scopes.php'); // User roles and scopes -Config::load('services', __DIR__ . '/config/services.php'); // List of services -Config::load('variables', __DIR__ . '/config/variables.php'); // List of env variables -Config::load('regions', __DIR__ . '/config/regions.php'); // List of available regions -Config::load('avatar-browsers', __DIR__ . '/config/avatars/browsers.php'); -Config::load('avatar-credit-cards', __DIR__ . '/config/avatars/credit-cards.php'); -Config::load('avatar-flags', __DIR__ . '/config/avatars/flags.php'); -Config::load('locale-codes', __DIR__ . '/config/locale/codes.php'); -Config::load('locale-currencies', __DIR__ . '/config/locale/currencies.php'); -Config::load('locale-eu', __DIR__ . '/config/locale/eu.php'); -Config::load('locale-languages', __DIR__ . '/config/locale/languages.php'); -Config::load('locale-phones', __DIR__ . '/config/locale/phones.php'); -Config::load('locale-countries', __DIR__ . '/config/locale/countries.php'); -Config::load('locale-continents', __DIR__ . '/config/locale/continents.php'); -Config::load('locale-templates', __DIR__ . '/config/locale/templates.php'); -Config::load('storage-logos', __DIR__ . '/config/storage/logos.php'); -Config::load('storage-mimes', __DIR__ . '/config/storage/mimes.php'); -Config::load('storage-inputs', __DIR__ . '/config/storage/inputs.php'); -Config::load('storage-outputs', __DIR__ . '/config/storage/outputs.php'); -Config::load('runtime-specifications', __DIR__ . '/config/runtimes/specifications.php'); -Config::load('function-templates', __DIR__ . '/config/function-templates.php'); +Config::load('events',__DIR__ . '/../config/events.php'); +Config::load('auth',__DIR__ . '/../config/auth.php'); +Config::load('apis',__DIR__ . '/../config/apis.php'); // List of APIs +Config::load('errors',__DIR__ . '/../config/errors.php'); +Config::load('oAuthProviders',__DIR__ . '/../config/oAuthProviders.php'); +Config::load('platforms',__DIR__ . '/../config/platforms.php'); +Config::load('console',__DIR__ . '/../config/console.php'); +Config::load('collections',__DIR__ . '/../config/collections.php'); +Config::load('runtimes',__DIR__ . '/../config/runtimes.php'); +Config::load('runtimes-v2',__DIR__ . '/../config/runtimes-v2.php'); +Config::load('usage',__DIR__ . '/../config/usage.php'); +Config::load('roles',__DIR__ . '/../config/roles.php'); // User roles and scopes +Config::load('scopes',__DIR__ . '/../config/scopes.php'); // User roles and scopes +Config::load('services',__DIR__ . '/../config/services.php'); // List of services +Config::load('variables',__DIR__ . '/../config/variables.php'); // List of env variables +Config::load('regions',__DIR__ . '/../config/regions.php'); // List of available regions +Config::load('avatar-browsers',__DIR__ . '/../config/avatars/browsers.php'); +Config::load('avatar-credit-cards',__DIR__ . '/../config/avatars/credit-cards.php'); +Config::load('avatar-flags',__DIR__ . '/../config/avatars/flags.php'); +Config::load('locale-codes',__DIR__ . '/../config/locale/codes.php'); +Config::load('locale-currencies',__DIR__ . '/../config/locale/currencies.php'); +Config::load('locale-eu',__DIR__ . '/../config/locale/eu.php'); +Config::load('locale-languages',__DIR__ . '/../config/locale/languages.php'); +Config::load('locale-phones',__DIR__ . '/../config/locale/phones.php'); +Config::load('locale-countries',__DIR__ . '/../config/locale/countries.php'); +Config::load('locale-continents',__DIR__ . '/../config/locale/continents.php'); +Config::load('locale-templates',__DIR__ . '/../config/locale/templates.php'); +Config::load('storage-logos',__DIR__ . '/../config/storage/logos.php'); +Config::load('storage-mimes',__DIR__ . '/../config/storage/mimes.php'); +Config::load('storage-inputs',__DIR__ . '/../config/storage/inputs.php'); +Config::load('storage-outputs',__DIR__ . '/../config/storage/outputs.php'); +Config::load('runtime-specifications',__DIR__ . '/../config/runtimes/specifications.php'); +Config::load('function-templates',__DIR__ . '/../config/function-templates.php'); From 56aa503146f160923c6bef57ae7ad12d54db7e4a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Mar 2025 13:57:12 +0100 Subject: [PATCH 171/275] Fixed paths --- app/init/locales.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/init/locales.php b/app/init/locales.php index 333dd106e3..dce95729e4 100644 --- a/app/init/locales.php +++ b/app/init/locales.php @@ -10,12 +10,12 @@ $locales = Config::getParam('locale-codes', []); foreach ($locales as $locale) { $code = $locale['code']; - $path = __DIR__ . '/config/locale/translations/' . $code . '.json'; + $path =__DIR__ . '/../config/locale/translations/' . $code . '.json'; if (!\file_exists($path)) { - $path = __DIR__ . '/config/locale/translations/' . \substr($code, 0, 2) . '.json'; // if `ar-ae` doesn't exist, look for `ar` + $path =__DIR__ . '/../config/locale/translations/' . \substr($code, 0, 2) . '.json'; // if `ar-ae` doesn't exist, look for `ar` if (!\file_exists($path)) { - $path = __DIR__ . '/config/locale/translations/en.json'; // if none translation exists, use default from `en.json` + $path =__DIR__ . '/../config/locale/translations/en.json'; // if none translation exists, use default from `en.json` } } From aac90ea8b8f4ad9ed4a2511565d98a097e5e735f Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Mar 2025 14:17:47 +0100 Subject: [PATCH 172/275] Fixed format --- app/init/configs.php | 62 ++++++++++++++++++++++---------------------- app/init/locales.php | 6 ++--- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/app/init/configs.php b/app/init/configs.php index 7d2d858351..bc50777df2 100644 --- a/app/init/configs.php +++ b/app/init/configs.php @@ -2,36 +2,36 @@ use Utopia\Config\Config; -Config::load('events',__DIR__ . '/../config/events.php'); -Config::load('auth',__DIR__ . '/../config/auth.php'); -Config::load('apis',__DIR__ . '/../config/apis.php'); // List of APIs -Config::load('errors',__DIR__ . '/../config/errors.php'); -Config::load('oAuthProviders',__DIR__ . '/../config/oAuthProviders.php'); -Config::load('platforms',__DIR__ . '/../config/platforms.php'); -Config::load('console',__DIR__ . '/../config/console.php'); -Config::load('collections',__DIR__ . '/../config/collections.php'); -Config::load('runtimes',__DIR__ . '/../config/runtimes.php'); -Config::load('runtimes-v2',__DIR__ . '/../config/runtimes-v2.php'); -Config::load('usage',__DIR__ . '/../config/usage.php'); -Config::load('roles',__DIR__ . '/../config/roles.php'); // User roles and scopes -Config::load('scopes',__DIR__ . '/../config/scopes.php'); // User roles and scopes -Config::load('services',__DIR__ . '/../config/services.php'); // List of services -Config::load('variables',__DIR__ . '/../config/variables.php'); // List of env variables -Config::load('regions',__DIR__ . '/../config/regions.php'); // List of available regions -Config::load('avatar-browsers',__DIR__ . '/../config/avatars/browsers.php'); -Config::load('avatar-credit-cards',__DIR__ . '/../config/avatars/credit-cards.php'); -Config::load('avatar-flags',__DIR__ . '/../config/avatars/flags.php'); -Config::load('locale-codes',__DIR__ . '/../config/locale/codes.php'); -Config::load('locale-currencies',__DIR__ . '/../config/locale/currencies.php'); -Config::load('locale-eu',__DIR__ . '/../config/locale/eu.php'); -Config::load('locale-languages',__DIR__ . '/../config/locale/languages.php'); -Config::load('locale-phones',__DIR__ . '/../config/locale/phones.php'); -Config::load('locale-countries',__DIR__ . '/../config/locale/countries.php'); -Config::load('locale-continents',__DIR__ . '/../config/locale/continents.php'); -Config::load('locale-templates',__DIR__ . '/../config/locale/templates.php'); -Config::load('storage-logos',__DIR__ . '/../config/storage/logos.php'); -Config::load('storage-mimes',__DIR__ . '/../config/storage/mimes.php'); -Config::load('storage-inputs',__DIR__ . '/../config/storage/inputs.php'); -Config::load('storage-outputs',__DIR__ . '/../config/storage/outputs.php'); +Config::load('events', __DIR__ . '/../config/events.php'); +Config::load('auth', __DIR__ . '/../config/auth.php'); +Config::load('apis', __DIR__ . '/../config/apis.php'); // List of APIs +Config::load('errors', __DIR__ . '/../config/errors.php'); +Config::load('oAuthProviders', __DIR__ . '/../config/oAuthProviders.php'); +Config::load('platforms', __DIR__ . '/../config/platforms.php'); +Config::load('console', __DIR__ . '/../config/console.php'); +Config::load('collections', __DIR__ . '/../config/collections.php'); +Config::load('runtimes', __DIR__ . '/../config/runtimes.php'); +Config::load('runtimes-v2', __DIR__ . '/../config/runtimes-v2.php'); +Config::load('usage', __DIR__ . '/../config/usage.php'); +Config::load('roles', __DIR__ . '/../config/roles.php'); // User roles and scopes +Config::load('scopes', __DIR__ . '/../config/scopes.php'); // User roles and scopes +Config::load('services', __DIR__ . '/../config/services.php'); // List of services +Config::load('variables', __DIR__ . '/../config/variables.php'); // List of env variables +Config::load('regions', __DIR__ . '/../config/regions.php'); // List of available regions +Config::load('avatar-browsers', __DIR__ . '/../config/avatars/browsers.php'); +Config::load('avatar-credit-cards', __DIR__ . '/../config/avatars/credit-cards.php'); +Config::load('avatar-flags', __DIR__ . '/../config/avatars/flags.php'); +Config::load('locale-codes', __DIR__ . '/../config/locale/codes.php'); +Config::load('locale-currencies', __DIR__ . '/../config/locale/currencies.php'); +Config::load('locale-eu', __DIR__ . '/../config/locale/eu.php'); +Config::load('locale-languages', __DIR__ . '/../config/locale/languages.php'); +Config::load('locale-phones', __DIR__ . '/../config/locale/phones.php'); +Config::load('locale-countries', __DIR__ . '/../config/locale/countries.php'); +Config::load('locale-continents', __DIR__ . '/../config/locale/continents.php'); +Config::load('locale-templates', __DIR__ . '/../config/locale/templates.php'); +Config::load('storage-logos', __DIR__ . '/../config/storage/logos.php'); +Config::load('storage-mimes', __DIR__ . '/../config/storage/mimes.php'); +Config::load('storage-inputs', __DIR__ . '/../config/storage/inputs.php'); +Config::load('storage-outputs', __DIR__ . '/../config/storage/outputs.php'); Config::load('runtime-specifications',__DIR__ . '/../config/runtimes/specifications.php'); Config::load('function-templates',__DIR__ . '/../config/function-templates.php'); diff --git a/app/init/locales.php b/app/init/locales.php index dce95729e4..122dc89692 100644 --- a/app/init/locales.php +++ b/app/init/locales.php @@ -10,12 +10,12 @@ $locales = Config::getParam('locale-codes', []); foreach ($locales as $locale) { $code = $locale['code']; - $path =__DIR__ . '/../config/locale/translations/' . $code . '.json'; + $path = __DIR__ . '/../config/locale/translations/' . $code . '.json'; if (!\file_exists($path)) { - $path =__DIR__ . '/../config/locale/translations/' . \substr($code, 0, 2) . '.json'; // if `ar-ae` doesn't exist, look for `ar` + $path = __DIR__ . '/../config/locale/translations/' . \substr($code, 0, 2) . '.json'; // if `ar-ae` doesn't exist, look for `ar` if (!\file_exists($path)) { - $path =__DIR__ . '/../config/locale/translations/en.json'; // if none translation exists, use default from `en.json` + $path = __DIR__ . '/../config/locale/translations/en.json'; // if none translation exists, use default from `en.json` } } From 3d69a35d5ae2b440bc2a6c102d3eaddeadcbada7 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Mar 2025 14:32:18 +0100 Subject: [PATCH 173/275] Fixed tests --- app/init/registers.php | 4 ++-- app/init/resources.php | 31 +++++++++++++++++++------------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/app/init/registers.php b/app/init/registers.php index 4ce8adcefd..9c90eb54d2 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -324,10 +324,10 @@ $register->set('smtp', function () { return $mail; }); $register->set('geodb', function () { - return new Reader(__DIR__ . '/assets/dbip/dbip-country-lite-2024-09.mmdb'); + return new Reader(__DIR__ . '/../assets/dbip/dbip-country-lite-2024-09.mmdb'); }); $register->set('passwordsDictionary', function () { - $content = \file_get_contents(__DIR__ . '/assets/security/10k-common-passwords'); + $content = \file_get_contents(__DIR__ . '/../assets/security/10k-common-passwords'); $content = explode("\n", $content); $content = array_flip($content); return $content; diff --git a/app/init/resources.php b/app/init/resources.php index 0764ed6e76..9211320fdf 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -3,6 +3,7 @@ use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; +use Appwrite\Auth\Key; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Certificate; @@ -13,9 +14,14 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +use Appwrite\Event\Realtime; +use Appwrite\Event\StatsUsage; +use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\GraphQL\Schema; use Appwrite\Network\Validator\Origin; +use Appwrite\Utopia\Request; +use Utopia\Abuse\Adapters\TimeLimit as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; @@ -30,6 +36,7 @@ use Utopia\DSN\DSN; use Utopia\Locale\Locale; use Utopia\Logger\Log; use Utopia\Pools\Group; +use Utopia\Queue\Publisher; use Utopia\Storage\Device; use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\DOSpaces; @@ -66,43 +73,43 @@ App::setResource('publisher', function (Group $pools) { App::setResource('consumer', function (Group $pools) { return $pools->get('consumer')->pop()->getResource(); }, ['pools']); -App::setResource('queueForMessaging', function (Queue\Publisher $publisher) { +App::setResource('queueForMessaging', function (Publisher $publisher) { return new Messaging($publisher); }, ['publisher']); -App::setResource('queueForMails', function (Queue\Publisher $publisher) { +App::setResource('queueForMails', function (Publisher $publisher) { return new Mail($publisher); }, ['publisher']); -App::setResource('queueForBuilds', function (Queue\Publisher $publisher) { +App::setResource('queueForBuilds', function (Publisher $publisher) { return new Build($publisher); }, ['publisher']); -App::setResource('queueForDatabase', function (Queue\Publisher $publisher) { +App::setResource('queueForDatabase', function (Publisher $publisher) { return new EventDatabase($publisher); }, ['publisher']); -App::setResource('queueForDeletes', function (Queue\Publisher $publisher) { +App::setResource('queueForDeletes', function (Publisher $publisher) { return new Delete($publisher); }, ['publisher']); -App::setResource('queueForEvents', function (Queue\Publisher $publisher) { +App::setResource('queueForEvents', function (Publisher $publisher) { return new Event($publisher); }, ['publisher']); -App::setResource('queueForWebhooks', function (Queue\Publisher $publisher) { +App::setResource('queueForWebhooks', function (Publisher $publisher) { return new Webhook($publisher); }, ['publisher']); App::setResource('queueForRealtime', function () { return new Realtime(); }, []); -App::setResource('queueForStatsUsage', function (Queue\Publisher $publisher) { +App::setResource('queueForStatsUsage', function (Publisher $publisher) { return new StatsUsage($publisher); }, ['publisher']); -App::setResource('queueForAudits', function (Queue\Publisher $publisher) { +App::setResource('queueForAudits', function (Publisher $publisher) { return new Audit($publisher); }, ['publisher']); -App::setResource('queueForFunctions', function (Queue\Publisher $publisher) { +App::setResource('queueForFunctions', function (Publisher $publisher) { return new Func($publisher); }, ['publisher']); -App::setResource('queueForCertificates', function (Queue\Publisher $publisher) { +App::setResource('queueForCertificates', function (Publisher $publisher) { return new Certificate($publisher); }, ['publisher']); -App::setResource('queueForMigrations', function (Queue\Publisher $publisher) { +App::setResource('queueForMigrations', function (Publisher $publisher) { return new Migration($publisher); }, ['publisher']); App::setResource('clients', function ($request, $console, $project) { From e80bc2bb398af17e755d3bd1fe5cdbac02195509 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 15 Mar 2025 13:36:08 +0000 Subject: [PATCH 174/275] chore: shifted migration to 1.7.0 --- src/Appwrite/Migration/Migration.php | 1 + src/Appwrite/Migration/Version/V22.php | 8 --- src/Appwrite/Migration/Version/V23.php | 69 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 src/Appwrite/Migration/Version/V23.php diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 56016f1057..17e93f43f5 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -93,6 +93,7 @@ abstract class Migration '1.6.0' => 'V21', '1.6.1' => 'V21', '1.6.2' => 'V22', + '1.7.0' => 'V23', ]; /** diff --git a/src/Appwrite/Migration/Version/V22.php b/src/Appwrite/Migration/Version/V22.php index c5fdfc9ed6..4d15662112 100644 --- a/src/Appwrite/Migration/Version/V22.php +++ b/src/Appwrite/Migration/Version/V22.php @@ -75,14 +75,6 @@ class V22 extends Migration Console::warning("'personalRefreshToken' from {$id}: {$th->getMessage()}"); } break; - case 'memberships': - // Create roles index - try { - $this->createIndexFromCollection($this->projectDB, $id, '_key_roles'); - } catch (Throwable $th) { - Console::warning("'_key_roles' from {$id}: {$th->getMessage()}"); - } - break; } usleep(50000); diff --git a/src/Appwrite/Migration/Version/V23.php b/src/Appwrite/Migration/Version/V23.php new file mode 100644 index 0000000000..dec7e8e9d3 --- /dev/null +++ b/src/Appwrite/Migration/Version/V23.php @@ -0,0 +1,69 @@ + null, + fn () => [] + ); + } + + Console::info('Migrating Collections'); + $this->migrateCollections(); + } + + /** + * Migrate Collections. + * + * @return void + * @throws Exception|Throwable + */ + private function migrateCollections(): void + { + $internalProjectId = $this->project->getInternalId(); + $collectionType = match ($internalProjectId) { + 'console' => 'console', + default => 'projects', + }; + + $collections = $this->collections[$collectionType]; + foreach ($collections as $collection) { + $id = $collection['$id']; + + Console::log("Migrating Collection \"{$id}\""); + + $this->projectDB->setNamespace("_$internalProjectId"); + + switch ($id) { + case 'memberships': + // Create roles index + try { + $this->createIndexFromCollection($this->projectDB, $id, '_key_roles'); + } catch (Throwable $th) { + Console::warning("'_key_roles' from {$id}: {$th->getMessage()}"); + } + break; + } + + usleep(50000); + } + } +} From 167011ce872521a85888cb6df15c5be59b10fa1f Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Mar 2025 14:38:01 +0100 Subject: [PATCH 175/275] Fixed timelimit --- app/init/resources.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init/resources.php b/app/init/resources.php index 9211320fdf..96c52a2350 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -21,7 +21,7 @@ use Appwrite\Extend\Exception; use Appwrite\GraphQL\Schema; use Appwrite\Network\Validator\Origin; use Appwrite\Utopia\Request; -use Utopia\Abuse\Adapters\TimeLimit as TimeLimitRedis; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; From 418185b9147c387bf0c6aed79c94cf938eee05c4 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 15 Mar 2025 14:43:51 +0100 Subject: [PATCH 176/275] fixed format --- app/init/configs.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/init/configs.php b/app/init/configs.php index bc50777df2..1a5dbced1b 100644 --- a/app/init/configs.php +++ b/app/init/configs.php @@ -33,5 +33,5 @@ Config::load('storage-logos', __DIR__ . '/../config/storage/logos.php'); Config::load('storage-mimes', __DIR__ . '/../config/storage/mimes.php'); Config::load('storage-inputs', __DIR__ . '/../config/storage/inputs.php'); Config::load('storage-outputs', __DIR__ . '/../config/storage/outputs.php'); -Config::load('runtime-specifications',__DIR__ . '/../config/runtimes/specifications.php'); -Config::load('function-templates',__DIR__ . '/../config/function-templates.php'); +Config::load('runtime-specifications', __DIR__ . '/../config/runtimes/specifications.php'); +Config::load('function-templates', __DIR__ . '/../config/function-templates.php'); From 4eece99768a78ea1f8c64cd76711aae6cab63397 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 16 Mar 2025 02:35:10 +0000 Subject: [PATCH 177/275] Feat: domains count --- app/init/constants.php | 1 + src/Appwrite/Platform/Workers/StatsResources.php | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/app/init/constants.php b/app/init/constants.php index 3383b28f57..5e4edfd97d 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -221,6 +221,7 @@ const METRIC_TOPICS = 'topics'; const METRIC_TARGETS = 'targets'; const METRIC_PROVIDER_TYPE_TARGETS = '{providerType}.targets'; const METRIC_KEYS = 'keys'; +const METRIC_DOMAINS = 'domains'; const METRIC_RESOURCE_TYPE_ID_BUILDS = '{resourceType}.{resourceInternalId}.builds'; const METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE = '{resourceType}.{resourceInternalId}.builds.storage'; const METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments'; diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 1140698342..3c0e772bd4 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -114,6 +114,13 @@ class StatsResources extends Action $keys = $dbForPlatform->count('keys', [ Query::equal('projectInternalId', [$project->getInternalId()]) ]); + + $domains = $dbForPlatform->count('rules', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::equal('owner', ['']), + ]); + + $databases = $dbForProject->count('databases'); $buckets = $dbForProject->count('buckets'); $users = $dbForProject->count('users'); @@ -162,6 +169,7 @@ class StatsResources extends Action METRIC_PROVIDERS => $providers, METRIC_TOPICS => $topics, METRIC_KEYS => $keys, + METRIC_DOMAINS => $domains, METRIC_TARGETS => $targets, str_replace('{providerType}', MESSAGE_TYPE_EMAIL, METRIC_PROVIDER_TYPE_TARGETS) => $emailTargets, str_replace('{providerType}', MESSAGE_TYPE_PUSH, METRIC_PROVIDER_TYPE_TARGETS) => $pushTargets, From 90899b92e3f78b2184db17b892ef633f651bf4d9 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 16 Mar 2025 10:36:32 +0100 Subject: [PATCH 178/275] Change envs to system lib --- app/init/resources.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/init/resources.php b/app/init/resources.php index 96c52a2350..4e53b24c06 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -525,7 +525,7 @@ function getDevice(string $root, string $connection = ''): Device $accessSecret = ''; $bucket = ''; $region = ''; - $url = App::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); + $url = System::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); try { $dsn = new DSN($connection); @@ -566,7 +566,7 @@ function getDevice(string $root, string $connection = ''): Device $s3Region = System::getEnv('_APP_STORAGE_S3_REGION', ''); $s3Bucket = System::getEnv('_APP_STORAGE_S3_BUCKET', ''); $s3Acl = 'private'; - $s3EndpointUrl = App::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); + $s3EndpointUrl = System::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl, $s3EndpointUrl); case Storage::DEVICE_DO_SPACES: $doSpacesAccessKey = System::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); From 04711d8c02a341ca9e18e3b820f31a27a19f6c61 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 19:21:31 +1300 Subject: [PATCH 179/275] Add doc --- src/Appwrite/Platform/Workers/Migrations.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index f21a846a0d..4939dc8143 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -34,6 +34,9 @@ class Migrations extends Action protected Document $project; + /** + * @var callable + */ protected $logError; public static function getName(): string From 9cb860d4467f8e087bc1e16e1f6f515c10ad7cee Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 17 Mar 2025 13:52:17 +0200 Subject: [PATCH 180/275] _app_region --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 1cd9df243d..ea796682ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,6 +93,7 @@ services: - app/http.php environment: - _APP_ENV + - _APP_REGION - _APP_EDITION - _APP_WORKER_PER_CORE - _APP_LOCALE From bbfd22ce2e68c86ee862ff450ee294d442554dec Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 17 Mar 2025 17:05:40 +0200 Subject: [PATCH 181/275] Index check for delete worker --- src/Appwrite/Platform/Workers/Deletes.php | 147 +++++++++++++++++----- 1 file changed, 113 insertions(+), 34 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9b0590181a..3c8bcdd635 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -248,7 +248,8 @@ class Deletes extends Action $this->deleteByGroup( 'subscribers', [ - Query::equal('topicInternalId', [$topic->getInternalId()]) + Query::equal('topicInternalId', [$topic->getInternalId()]), + Query::orderAsc(), ], $getProjectDB($project) ); @@ -269,7 +270,8 @@ class Deletes extends Action $this->deleteByGroup( 'subscribers', [ - Query::equal('targetInternalId', [$target->getInternalId()]) + Query::equal('targetInternalId', [$target->getInternalId()]), + Query::orderAsc(), ], $dbForProject, function (Document $subscriber) use ($dbForProject, $target) { @@ -303,10 +305,14 @@ class Deletes extends Action */ private function deleteExpiredTargets(Document $project, callable $getProjectDB): void { + /** + * todo: No index found for `expired` attribute + */ $this->deleteByGroup( 'targets', [ - Query::equal('expired', [true]) + Query::equal('expired', [true]), + Query::orderAsc(), ], $getProjectDB($project), function (Document $target) use ($getProjectDB, $project) { @@ -317,10 +323,14 @@ class Deletes extends Action private function deleteSessionTargets(Document $project, callable $getProjectDB, Document $session): void { + /** + * todo: No index found for `sessionInternalId` attribute + */ $this->deleteByGroup( 'targets', [ - Query::equal('sessionInternalId', [$session->getInternalId()]) + Query::equal('sessionInternalId', [$session->getInternalId()]), + Query::orderAsc(), ], $getProjectDB($project), function (Document $target) use ($getProjectDB, $project) { @@ -348,10 +358,17 @@ class Deletes extends Action ); $query[] = Query::equal('resource', [$resource]); + if (!empty($resourceType)) { $query[] = Query::equal('resourceType', [$resourceType]); } + $query[] = Query::orderAsc(); + + /** + * todo: missing index on `resource`, `resourceType` + */ + $this->deleteByGroup( 'cache', $query, @@ -388,7 +405,7 @@ class Deletes extends Action $query = [ Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), - Query::orderDesc('$internalId'), + Query::orderDesc(), ]; $this->deleteByGroup( @@ -421,10 +438,10 @@ class Deletes extends Action // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ + Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc('$internalId'), - Query::equal('period', ['1h']), + Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC ], $dbForProject); if ($project->getId() !== 'console') { @@ -433,10 +450,10 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ + Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc('$internalId'), - Query::equal('period', ['1h']), + Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC ], $dbForLogs); } } @@ -457,7 +474,8 @@ class Deletes extends Action $this->deleteByGroup( 'memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) + Query::equal('teamInternalId', [$teamInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $membership) use ($dbForProject) { @@ -547,7 +565,13 @@ class Deletes extends Action if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { $dbForProject->deleteCollection($collection->getId()); } else { - $this->deleteByGroup($collection->getId(), [], database: $dbForProject); + $this->deleteByGroup( + $collection->getId(), + [ + Query::orderAsc() + ], + database: $dbForProject + ); } } catch (Throwable $e) { Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); @@ -567,58 +591,82 @@ class Deletes extends Action // Delete Platforms $this->deleteByGroup('platforms', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete project and function rules $this->deleteByGroup('rules', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($dbForPlatform, $certificates) { $this->deleteRule($dbForPlatform, $document, $certificates); }); // Delete Keys $this->deleteByGroup('keys', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete Webhooks $this->deleteByGroup('webhooks', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete VCS Installations $this->deleteByGroup('installations', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete VCS Repositories $this->deleteByGroup('repositories', [ Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete VCS comments $this->deleteByGroup('vcsComments', [ Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); - // Delete Schedules (No projectInternalId in this collection) + /** + * No projectInternalId in this collection + * todo: No index for `projectId` attribute + */ + // Delete Schedules () $this->deleteByGroup('schedules', [ Query::equal('projectId', [$projectId]), + Query::orderAsc() ], $dbForPlatform); // Delete metadata table if ($projectTables) { $dbForProject->deleteCollection(Database::METADATA); } elseif ($sharedTablesV1) { - $this->deleteByGroup(Database::METADATA, [], $dbForProject); + $this->deleteByGroup( + Database::METADATA, + [ + Query::orderAsc() + ], + $dbForProject + ); } elseif ($sharedTablesV2) { $queries = \array_map( fn ($id) => Query::notEqual('$id', $id), $projectCollectionIds ); - $this->deleteByGroup(Database::METADATA, $queries, $dbForProject); + $queries[] = Query::orderAsc(); + + $this->deleteByGroup( + Database::METADATA, + $queries, + $dbForProject + ); } // Delete all storage directories @@ -643,14 +691,16 @@ class Deletes extends Action // Delete all sessions of this user from the sessions table and update the sessions field of the user record $this->deleteByGroup('sessions', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject); $dbForProject->purgeCachedDocument('users', $userId); // Delete Memberships and decrement team membership counts $this->deleteByGroup('memberships', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($dbForProject) { if ($document->getAttribute('confirm')) { // Count only confirmed members $teamId = $document->getAttribute('teamId'); @@ -663,19 +713,25 @@ class Deletes extends Action // Delete tokens $this->deleteByGroup('tokens', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject); // Delete identities + /** + * todo: Remove Duplication index `_key_userInternalId` + */ $this->deleteByGroup('identities', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject); // Delete targets $this->deleteByGroup( 'targets', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $target) use ($getProjectDB, $project) { @@ -699,7 +755,7 @@ class Deletes extends Action $this->deleteByGroup('executions', [ Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), - Query::orderDesc('$internalId'), + Query::orderDesc(), ], $dbForProject); } @@ -719,7 +775,7 @@ class Deletes extends Action $this->deleteByGroup('sessions', [ Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), - Query::orderDesc('$internalId'), + Query::orderDesc(), ], $dbForProject); } @@ -735,7 +791,7 @@ class Deletes extends Action $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime), Query::orderDesc('timestamp'), - Query::orderDesc('$internalId'), + Query::orderAsc(), // KEY "_key_timestamp" ("timestamp" DESC), ], $dbForPlatform); } @@ -755,7 +811,7 @@ class Deletes extends Action $this->deleteByGroup(Audit::COLLECTION, [ Query::lessThan('time', $auditRetention), Query::orderDesc('time'), - Query::orderDesc('$internalId'), + Query::orderAsc(), // KEY "index-time" ("time" DESC) ], $dbForProject); } catch (DatabaseException $e) { Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); @@ -780,33 +836,39 @@ class Deletes extends Action /** * Delete rules + * todo: No index for this query */ Console::info("Deleting rules for function " . $functionId); $this->deleteByGroup('rules', [ Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), - Query::equal('projectInternalId', [$project->getInternalId()]) + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($project, $dbForPlatform, $certificates) { $this->deleteRule($dbForPlatform, $document, $certificates); }); /** * Delete Variables + * todo: No index for this query */ Console::info("Deleting variables for function " . $functionId); $this->deleteByGroup('variables', [ Query::equal('resourceType', ['function']), - Query::equal('resourceInternalId', [$functionInternalId]) + Query::equal('resourceInternalId', [$functionInternalId]), + Query::orderAsc() ], $dbForProject); /** * Delete Deployments + * todo: No index for `resourceInternalId`, perhaps use resourceId until fixed */ Console::info("Deleting deployments for function " . $functionId); $deploymentInternalIds = []; $this->deleteByGroup('deployments', [ - Query::equal('resourceInternalId', [$functionInternalId]) + Query::equal('resourceInternalId', [$functionInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($deviceForFunctions, &$deploymentInternalIds) { $deploymentInternalIds[] = $document->getInternalId(); $this->deleteDeploymentFiles($deviceForFunctions, $document); @@ -814,12 +876,14 @@ class Deletes extends Action /** * Delete builds + * todo: No index for `deploymentInternalId`, perhaps use deploymentId until fixed */ Console::info("Deleting builds for function " . $functionId); foreach ($deploymentInternalIds as $deploymentInternalId) { $this->deleteByGroup('builds', [ - Query::equal('deploymentInternalId', [$deploymentInternalId]) + Query::equal('deploymentInternalId', [$deploymentInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($deviceForBuilds) { $this->deleteBuildFiles($deviceForBuilds, $document); }); @@ -827,26 +891,35 @@ class Deletes extends Action /** * Delete Executions + * todo: No index for `functionInternalId` , perhaps use functionId until fixed */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ - Query::equal('functionInternalId', [$functionInternalId]) + Query::equal('functionInternalId', [$functionInternalId]), + Query::orderAsc() ], $dbForProject); /** * Delete VCS Repositories and VCS Comments + * todo: no index for this query */ Console::info("Deleting VCS repositories and comments linked to function " . $functionId); $this->deleteByGroup('repositories', [ Query::equal('projectInternalId', [$project->getInternalId()]), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('resourceType', ['function']), + Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($dbForPlatform) { $providerRepositoryId = $document->getAttribute('providerRepositoryId', ''); $projectInternalId = $document->getAttribute('projectInternalId', ''); + + /** + * todo: add index to this query + */ $this->deleteByGroup('vcsComments', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); }); @@ -942,11 +1015,13 @@ class Deletes extends Action /** * Delete builds + * todo: no index for `deploymentInternalId` change to temporary to deploymentId? */ Console::info("Deleting builds for deployment " . $deploymentId); $this->deleteByGroup('builds', [ - Query::equal('deploymentInternalId', [$deploymentInternalId]) + Query::equal('deploymentInternalId', [$deploymentInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($deviceForBuilds) { $this->deleteBuildFiles($deviceForBuilds, $document); }); @@ -974,6 +1049,10 @@ class Deletes extends Action ): void { $start = \microtime(true); + /** + * deleteDocuments uses a cursor, we need to add a unique order by field or use default + */ + try { $documents = $database->deleteDocuments($collection, $queries); } catch (Throwable $th) { From 8184bc8c78df84dc04a8c0514edda828a26f061d Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 17 Mar 2025 17:31:24 +0200 Subject: [PATCH 182/275] _app_region --- .env | 2 +- app/config/variables.php | 4 ++-- app/controllers/api/projects.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 4 ++-- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 9b4aef4ddd..ae4522c898 100644 --- a/.env +++ b/.env @@ -1,7 +1,7 @@ _APP_ENV=development _APP_EDITION=self-hosted _APP_LOCALE=en -_APP_REGION=fra +_APP_REGION=default _APP_WORKER_PER_CORE=6 _APP_COMPRESSION_MIN_SIZE_BYTES=1024 _APP_CONSOLE_WHITELIST_ROOT=disabled diff --git a/app/config/variables.php b/app/config/variables.php index 2f7a10221d..592d398062 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -9,9 +9,9 @@ return [ 'variables' => [ [ 'name' => '_APP_REGION', - 'description' => 'Set your server running geo region. By default, the var is set to \'fra\'.', + 'description' => 'Set your server running geo region. By default, the var is set to \'default\'.', 'introduction' => '', - 'default' => 'fra', + 'default' => 'default', 'required' => false, 'question' => '', 'filter' => '' diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 5099393e68..2f46c3c338 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -81,7 +81,7 @@ App::post('/v1/projects') ->param('projectId', '', new ProjectId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, and hyphen. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Project name. Max length: 128 chars.') ->param('teamId', '', new UID(), 'Team unique ID.') - ->param('region', System::getEnv('_APP_REGION'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) + ->param('region', System::getEnv('_APP_REGION','default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) ->param('description', '', new Text(256), 'Project description. Max length: 256 chars.', true) ->param('logo', '', new Text(1024), 'Project logo.', true) ->param('url', '', new URL(), 'Project URL.', true) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index b62b95b066..b96fd5622f 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -104,7 +104,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION')]), + Query::equal('region', [System::getEnv('_APP_REGION','default')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); @@ -154,7 +154,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION')]), + Query::equal('region', [System::getEnv('_APP_REGION','default')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index e8195d175d..624948e2f5 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -183,7 +183,7 @@ class Deletes extends Action $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION')]), + Query::equal('region', [System::getEnv('_APP_REGION','default')]), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], From 59f613f9bbda26a3019feb283b6de0f27b88af93 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 17 Mar 2025 17:33:48 +0200 Subject: [PATCH 183/275] _app_region --- app/controllers/api/projects.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 4 ++-- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 2f46c3c338..48d20cd17f 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -81,7 +81,7 @@ App::post('/v1/projects') ->param('projectId', '', new ProjectId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, and hyphen. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Project name. Max length: 128 chars.') ->param('teamId', '', new UID(), 'Team unique ID.') - ->param('region', System::getEnv('_APP_REGION','default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) + ->param('region', System::getEnv('_APP_REGION', 'default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) ->param('description', '', new Text(256), 'Project description. Max length: 256 chars.', true) ->param('logo', '', new Text(1024), 'Project logo.', true) ->param('url', '', new URL(), 'Project URL.', true) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index b96fd5622f..dad2db0d9a 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -104,7 +104,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION','default')]), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); @@ -154,7 +154,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION','default')]), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 624948e2f5..9b0590181a 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -183,7 +183,7 @@ class Deletes extends Action $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION','default')]), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], From 7149deeacb498429a756987d1a5990ca82a164d6 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 17 Mar 2025 18:11:49 +0200 Subject: [PATCH 184/275] _app_region --- .env | 2 +- app/config/regions.php | 7 ------ app/config/variables.php | 2 +- app/controllers/api/functions.php | 2 +- app/controllers/api/messaging.php | 12 +++++----- app/controllers/api/projects.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 4 ++-- src/Appwrite/Platform/Workers/Deletes.php | 2 +- .../Platform/Workers/StatsUsageDump.php | 4 ++-- tests/e2e/Scopes/ProjectCustom.php | 2 +- .../Projects/ProjectsConsoleClientTest.php | 22 +++++++++---------- 11 files changed, 27 insertions(+), 34 deletions(-) diff --git a/.env b/.env index ae4522c898..9b4aef4ddd 100644 --- a/.env +++ b/.env @@ -1,7 +1,7 @@ _APP_ENV=development _APP_EDITION=self-hosted _APP_LOCALE=en -_APP_REGION=default +_APP_REGION=fra _APP_WORKER_PER_CORE=6 _APP_COMPRESSION_MIN_SIZE_BYTES=1024 _APP_CONSOLE_WHITELIST_ROOT=disabled diff --git a/app/config/regions.php b/app/config/regions.php index 61029a9996..0b32d2ba4b 100644 --- a/app/config/regions.php +++ b/app/config/regions.php @@ -1,13 +1,6 @@ [ - '$id' => 'default', - 'name' => 'Frankfurt', - 'disabled' => false, - 'flag' => 'de', - 'default' => true, - ], 'fra' => [ '$id' => 'fra', 'name' => 'Frankfurt', diff --git a/app/config/variables.php b/app/config/variables.php index 592d398062..7c774e8d6a 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -9,7 +9,7 @@ return [ 'variables' => [ [ 'name' => '_APP_REGION', - 'description' => 'Set your server running geo region. By default, the var is set to \'default\'.', + 'description' => 'Set your server running geo region. By default, the var is set to \'fra\'.', 'introduction' => '', 'default' => 'default', 'required' => false, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 1483c3bf03..37932e4165 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -2079,7 +2079,7 @@ App::post('/v1/functions/:functionId/executions') ]; $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => ScheduleExecutions::getSupportedResource(), 'resourceId' => $execution->getId(), 'resourceInternalId' => $execution->getInternalId(), diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 9393f1fbfe..4812fab0f3 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -2936,7 +2936,7 @@ App::post('/v1/messaging/messages/email') break; case MessageStatus::SCHEDULED: $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -3058,7 +3058,7 @@ App::post('/v1/messaging/messages/sms') break; case MessageStatus::SCHEDULED: $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => System::getEnv('_APP_REGION'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -3275,7 +3275,7 @@ App::post('/v1/messaging/messages/push') break; case MessageStatus::SCHEDULED: $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -3661,7 +3661,7 @@ App::patch('/v1/messaging/messages/email/:messageId') if (\is_null($currentScheduledAt) && !\is_null($scheduledAt)) { $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -3862,7 +3862,7 @@ App::patch('/v1/messaging/messages/sms/:messageId') if (\is_null($currentScheduledAt) && !\is_null($scheduledAt)) { $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -4035,7 +4035,7 @@ App::patch('/v1/messaging/messages/push/:messageId') if (\is_null($currentScheduledAt) && !\is_null($scheduledAt)) { $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 48d20cd17f..5099393e68 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -81,7 +81,7 @@ App::post('/v1/projects') ->param('projectId', '', new ProjectId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, and hyphen. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Project name. Max length: 128 chars.') ->param('teamId', '', new UID(), 'Team unique ID.') - ->param('region', System::getEnv('_APP_REGION', 'default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) + ->param('region', System::getEnv('_APP_REGION'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) ->param('description', '', new Text(256), 'Project description. Max length: 256 chars.', true) ->param('logo', '', new Text(1024), 'Project logo.', true) ->param('url', '', new URL(), 'Project URL.', true) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index dad2db0d9a..b62b95b066 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -104,7 +104,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', [System::getEnv('_APP_REGION')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); @@ -154,7 +154,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', [System::getEnv('_APP_REGION')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9b0590181a..e8195d175d 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -183,7 +183,7 @@ class Deletes extends Action $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', [System::getEnv('_APP_REGION')]), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 119a9e7288..3100a39dea 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -157,7 +157,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => System::getEnv('_APP_REGION'), ]); $documentClone = new Document($document->getArrayCopy()); @@ -191,7 +191,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => System::getEnv('_APP_REGION'), ]); $documentClone = new Document($document->getArrayCopy()); $dbForProject->createOrUpdateDocumentsWithIncrease( diff --git a/tests/e2e/Scopes/ProjectCustom.php b/tests/e2e/Scopes/ProjectCustom.php index 7f84ace6f2..2799e29772 100644 --- a/tests/e2e/Scopes/ProjectCustom.php +++ b/tests/e2e/Scopes/ProjectCustom.php @@ -42,7 +42,7 @@ trait ProjectCustom 'x-appwrite-project' => 'console', ], [ 'projectId' => ID::unique(), - 'region' => 'default', + 'region' => 'fra', 'name' => 'Demo Project', 'teamId' => $team['body']['$id'], 'description' => 'Demo Project Description', diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index a6498b8a4e..16a18d27c1 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -49,7 +49,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default', + 'region' => 'fra', ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -89,7 +89,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => '', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(400, $response['headers']['status-code']); @@ -100,7 +100,7 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders()), [ 'projectId' => ID::unique(), 'name' => 'Project Test', - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(401, $response['headers']['status-code']); @@ -129,7 +129,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => $projectId, 'name' => 'Project Duplicate', 'teamId' => $teamId, - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(409, $response['headers']['status-code']); @@ -178,7 +178,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Team 1 Project', 'teamId' => $team1, - 'region' => 'default', + 'region' => 'fra', ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -277,7 +277,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test 2', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -2042,7 +2042,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -2135,7 +2135,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3749,7 +3749,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project', 'teamId' => $teamId, - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3820,7 +3820,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 1', 'teamId' => $teamId, - 'region' => 'default' + 'region' => 'fra' ]); $project2 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ @@ -3830,7 +3830,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 2', 'teamId' => $teamId, - 'region' => 'default' + 'region' => 'fra' ]); $project1Id = $project1['body']['$id']; From 60912cb0845bb5787126171608a54bcb11619049 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 17 Mar 2025 18:14:58 +0200 Subject: [PATCH 185/275] _app_region --- app/controllers/api/messaging.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 4812fab0f3..178266db60 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -3058,7 +3058,7 @@ App::post('/v1/messaging/messages/sms') break; case MessageStatus::SCHEDULED: $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), From e5261e7d70d62ff0ef398facb58f65f1a64247d1 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 17 Mar 2025 18:31:14 +0200 Subject: [PATCH 186/275] _app_region --- src/Appwrite/Migration/Version/V20.php | 2 +- tests/e2e/Services/Functions/FunctionsCustomClientTest.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Version/V20.php b/src/Appwrite/Migration/Version/V20.php index 5a0807cedf..cca0d9244b 100644 --- a/src/Appwrite/Migration/Version/V20.php +++ b/src/Appwrite/Migration/Version/V20.php @@ -426,7 +426,7 @@ class V20 extends Migration 'period' => 'inf', 'value' => $value, 'time' => null, - 'region' => 'default', + 'region' => 'fra', ])); } catch (Duplicate $th) { Console::warning("Error while creating inf metric: duplicate id {$metric} {$id}"); diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 914a255663..5e24dc4d89 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -111,7 +111,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('default', $output['APPWRITE_REGION']); + $this->assertEquals('fra', $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); @@ -221,7 +221,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('default', $output['APPWRITE_REGION']); + $this->assertEquals('fra', $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); From 5f7ceecb30e64f11fa6bb0a5cb746007cc983b8d Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Mar 2025 03:46:14 +0000 Subject: [PATCH 187/275] chore: stop tests on failure --- phpunit.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpunit.xml b/phpunit.xml index 4c4e55ea4e..598b730908 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -6,7 +6,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false" + stopOnFailure="true" > From a7165e6a32a0c731e0a10939935b863fcb824573 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 16:51:00 +1300 Subject: [PATCH 188/275] Disable PDO persistence since we manage our own pool --- app/init/registers.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init/registers.php b/app/init/registers.php index 9c90eb54d2..1ebcbc1691 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -217,7 +217,7 @@ $register->set('pools', function () { return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, + PDO::ATTR_PERSISTENT => false, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_EMULATE_PREPARES => true, PDO::ATTR_STRINGIFY_FETCHES => true From e43a3109eb98b46400c19f119d81220f6ea8b81e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 20:05:12 +1300 Subject: [PATCH 189/275] Update migration lib --- composer.json | 2 +- composer.lock | 40 ++++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/composer.json b/composer.json index 53c2a6d482..b604d6d1b8 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.16.*", - "utopia-php/migration": "0.6.*", + "utopia-php/migration": "0.8.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.5.*", diff --git a/composer.lock b/composer.lock index ed19d10202..2e5e6738e6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "44c6436ced36b0b026139edba252052e", + "content-hash": "ef36941f461409608974a804999f6787", "packages": [ { "name": "adhocore/jwt", @@ -4159,16 +4159,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.22", + "version": "0.8.0", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "a0269746bd318ff0993f5aa008675b971689d5b5" + "reference": "d8245090a7c55ea3c1fc411815b06bad0d5bcdd8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/a0269746bd318ff0993f5aa008675b971689d5b5", - "reference": "a0269746bd318ff0993f5aa008675b971689d5b5", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/d8245090a7c55ea3c1fc411815b06bad0d5bcdd8", + "reference": "d8245090a7c55ea3c1fc411815b06bad0d5bcdd8", "shasum": "" }, "require": { @@ -4209,9 +4209,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.22" + "source": "https://github.com/utopia-php/migration/tree/0.8.0" }, - "time": "2025-03-13T07:35:55+00:00" + "time": "2025-03-18T06:52:33+00:00" }, { "name": "utopia-php/mongo", @@ -4760,16 +4760,16 @@ }, { "name": "utopia-php/telemetry", - "version": "0.1.0", + "version": "0.1.1", "source": { "type": "git", "url": "https://github.com/utopia-php/telemetry.git", - "reference": "d35f2f0632f4ee0be63fb7ace6a94a6adda71a80" + "reference": "437f0021777f0e575dfb9e8a1a081b3aed75e33f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/d35f2f0632f4ee0be63fb7ace6a94a6adda71a80", - "reference": "d35f2f0632f4ee0be63fb7ace6a94a6adda71a80", + "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/437f0021777f0e575dfb9e8a1a081b3aed75e33f", + "reference": "437f0021777f0e575dfb9e8a1a081b3aed75e33f", "shasum": "" }, "require": { @@ -4790,7 +4790,7 @@ "type": "library", "autoload": { "psr-4": { - "Utopia\\": "src/" + "Utopia\\Telemetry\\": "src/Telemetry" } }, "notification-url": "https://packagist.org/downloads/", @@ -4804,9 +4804,9 @@ ], "support": { "issues": "https://github.com/utopia-php/telemetry/issues", - "source": "https://github.com/utopia-php/telemetry/tree/0.1.0" + "source": "https://github.com/utopia-php/telemetry/tree/0.1.1" }, - "time": "2024-11-13T10:29:53+00:00" + "time": "2025-03-17T11:57:52+00:00" }, { "name": "utopia-php/vcs", @@ -5043,16 +5043,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.7", + "version": "0.40.9", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "9e89b0bc4d8e6c81817d27096629f34a149fa873" + "reference": "dbb45a5db22cdc3368fe2573c07ba6088f188fa4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/9e89b0bc4d8e6c81817d27096629f34a149fa873", - "reference": "9e89b0bc4d8e6c81817d27096629f34a149fa873", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/dbb45a5db22cdc3368fe2573c07ba6088f188fa4", + "reference": "dbb45a5db22cdc3368fe2573c07ba6088f188fa4", "shasum": "" }, "require": { @@ -5088,9 +5088,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.7" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.9" }, - "time": "2025-03-12T08:43:55+00:00" + "time": "2025-03-17T18:39:14+00:00" }, { "name": "doctrine/annotations", From fe308b4f1f86ad97bdb6ef0e3f46becb6b4c5bf2 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 20:49:19 +1300 Subject: [PATCH 190/275] Update lib --- composer.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/composer.lock b/composer.lock index 2e5e6738e6..07e77a9037 100644 --- a/composer.lock +++ b/composer.lock @@ -4159,16 +4159,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.0", + "version": "0.8.1", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "d8245090a7c55ea3c1fc411815b06bad0d5bcdd8" + "reference": "36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/d8245090a7c55ea3c1fc411815b06bad0d5bcdd8", - "reference": "d8245090a7c55ea3c1fc411815b06bad0d5bcdd8", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab", + "reference": "36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab", "shasum": "" }, "require": { @@ -4209,9 +4209,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.0" + "source": "https://github.com/utopia-php/migration/tree/0.8.1" }, - "time": "2025-03-18T06:52:33+00:00" + "time": "2025-03-18T07:48:08+00:00" }, { "name": "utopia-php/mongo", From aa5ea7f3a10f632d87c1dbbf334104c2c38d4135 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 18 Mar 2025 10:01:06 +0100 Subject: [PATCH 191/275] feat: add pool telemetry --- composer.json | 2 +- composer.lock | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/composer.json b/composer.json index b604d6d1b8..d7b8505b5c 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/migration": "0.8.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "0.5.*", + "utopia-php/pools": "0.7.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.9.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index 07e77a9037..cdba756c72 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ef36941f461409608974a804999f6787", + "content-hash": "e0d7f21b681e4591144fec16c4f0d6aa", "packages": [ { "name": "adhocore/jwt", @@ -4375,25 +4375,26 @@ }, { "name": "utopia-php/pools", - "version": "0.5.0", + "version": "0.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "6f716a213a08db95eda1b5dddfa90983c1834817" + "reference": "ad64d45afda08ec8b29e2642a8d18075964d40bf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/6f716a213a08db95eda1b5dddfa90983c1834817", - "reference": "6f716a213a08db95eda1b5dddfa90983c1834817", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/ad64d45afda08ec8b29e2642a8d18075964d40bf", + "reference": "ad64d45afda08ec8b29e2642a8d18075964d40bf", "shasum": "" }, "require": { - "php": ">=8.0" + "php": ">=8.3", + "utopia-php/telemetry": "0.1.*" }, "require-dev": { - "laravel/pint": "1.2.*", - "phpstan/phpstan": "1.8.*", - "phpunit/phpunit": "^9.3" + "laravel/pint": "1.*", + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "11.*" }, "type": "library", "autoload": { @@ -4420,9 +4421,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.5.0" + "source": "https://github.com/utopia-php/pools/tree/0.7.0" }, - "time": "2024-04-19T11:11:54+00:00" + "time": "2025-03-18T03:55:33+00:00" }, { "name": "utopia-php/preloader", @@ -8402,7 +8403,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 70e27748be399a05cacf1bc222b49dac1198731e Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Mar 2025 12:52:41 +0000 Subject: [PATCH 192/275] chore: set min operations to 1 for reads and writes --- app/controllers/api/databases.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index d8d496089c..2a1ec4cba4 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3355,7 +3355,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, min($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection @@ -3520,7 +3520,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, min($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -3661,7 +3661,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, min($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -3959,7 +3959,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, min($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); $response->addHeader('X-Debug-Operations', $operations); From aab8bca91e98fcdc42fc854853e6d0725fee86ce Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Mar 2025 14:20:25 +0000 Subject: [PATCH 193/275] fix: logic --- app/controllers/api/databases.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 2a1ec4cba4..0c37e1a765 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3355,7 +3355,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, min($operations, 1)) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection @@ -3520,7 +3520,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, min($operations, 1)) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -3661,7 +3661,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, min($operations, 1)) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -3959,7 +3959,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, min($operations, 1)) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); $response->addHeader('X-Debug-Operations', $operations); From 795928a037644ca8aca6a3ad6fd07b2f6ffffef5 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 18 Mar 2025 16:59:05 +0200 Subject: [PATCH 194/275] Fix $id --- src/Appwrite/Platform/Workers/Deletes.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 3c8bcdd635..a14e54caab 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -1045,7 +1045,8 @@ class Deletes extends Action string $collection, array $queries, Database $database, - ?callable $callback = null + ?callable $callback = null, + bool $shortSelect = false ): void { $start = \microtime(true); @@ -1053,6 +1054,12 @@ class Deletes extends Action * deleteDocuments uses a cursor, we need to add a unique order by field or use default */ + if (!\is_callable($callback) && $shortSelect) { + $queries = array_merge($queries, [ + Query::select(['$internalId', '$id', '$permissions', '$updatedAt']) + ]); + } + try { $documents = $database->deleteDocuments($collection, $queries); } catch (Throwable $th) { From e9841e53b9995091d64b665addcd1d2389740d25 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 18 Mar 2025 21:40:46 +0200 Subject: [PATCH 195/275] revert --- .env | 1 - app/config/regions.php | 63 +------------------ app/config/variables.php | 9 --- app/controllers/api/projects.php | 2 +- app/views/install/compose.phtml | 37 +++++------ docker-compose.yml | 40 ++++++------ src/Appwrite/Migration/Version/V20.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 4 +- src/Appwrite/Platform/Workers/Deletes.php | 2 +- .../Platform/Workers/StatsUsageDump.php | 4 +- tests/e2e/Scopes/ProjectCustom.php | 2 +- .../Functions/FunctionsCustomClientTest.php | 4 +- .../Projects/ProjectsConsoleClientTest.php | 22 +++---- 13 files changed, 60 insertions(+), 132 deletions(-) diff --git a/.env b/.env index 9b4aef4ddd..1893e023ba 100644 --- a/.env +++ b/.env @@ -1,7 +1,6 @@ _APP_ENV=development _APP_EDITION=self-hosted _APP_LOCALE=en -_APP_REGION=fra _APP_WORKER_PER_CORE=6 _APP_COMPRESSION_MIN_SIZE_BYTES=1024 _APP_CONSOLE_WHITELIST_ROOT=disabled diff --git a/app/config/regions.php b/app/config/regions.php index 0b32d2ba4b..05e04930fe 100644 --- a/app/config/regions.php +++ b/app/config/regions.php @@ -1,67 +1,10 @@ [ - '$id' => 'fra', - 'name' => 'Frankfurt', + 'default' => [ + '$id' => 'default', + 'name' => 'default', 'disabled' => false, - 'flag' => 'de', - 'default' => true, - ], - 'nyc' => [ - '$id' => 'nyc', - 'name' => 'New York', - 'disabled' => true, - 'flag' => 'us', - 'default' => true, - ], - 'sfo' => [ - '$id' => 'sfo', - 'name' => 'San Francisco', - 'disabled' => true, - 'flag' => 'us', - 'default' => true, - ], - 'blr' => [ - '$id' => 'blr', - 'name' => 'Bengaluru', - 'disabled' => true, - 'flag' => 'in', - 'default' => true, - ], - 'lon' => [ - '$id' => 'lon', - 'name' => 'London', - 'disabled' => true, - 'flag' => 'gb', - 'default' => true, - ], - 'ams' => [ - '$id' => 'ams', - 'name' => 'Amsterdam', - 'disabled' => true, - 'flag' => 'nl', - 'default' => true, - ], - 'sgp' => [ - '$id' => 'sgp', - 'name' => 'Singapore', - 'disabled' => true, - 'flag' => 'sg', - 'default' => true, - ], - 'tor' => [ - '$id' => 'tor', - 'name' => 'Toronto', - 'disabled' => true, - 'flag' => 'ca', - 'default' => true, - ], - 'syd' => [ - '$id' => 'syd', - 'name' => 'Sydney', - 'disabled' => true, - 'flag' => 'au', 'default' => true, ], ]; diff --git a/app/config/variables.php b/app/config/variables.php index 7c774e8d6a..98dd9ffec1 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -7,15 +7,6 @@ return [ 'category' => 'General', 'description' => '', 'variables' => [ - [ - 'name' => '_APP_REGION', - 'description' => 'Set your server running geo region. By default, the var is set to \'fra\'.', - 'introduction' => '', - 'default' => 'default', - 'required' => false, - 'question' => '', - 'filter' => '' - ], [ 'name' => '_APP_ENV', 'description' => 'Set your server running environment. By default, the var is set to \'development\'. When deploying to production, change it to: \'production\'.', diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 5099393e68..48d20cd17f 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -81,7 +81,7 @@ App::post('/v1/projects') ->param('projectId', '', new ProjectId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, and hyphen. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Project name. Max length: 128 chars.') ->param('teamId', '', new UID(), 'Team unique ID.') - ->param('region', System::getEnv('_APP_REGION'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) + ->param('region', System::getEnv('_APP_REGION', 'default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) ->param('description', '', new Text(256), 'Project description. Max length: 256 chars.', true) ->param('logo', '', new Text(1024), 'Project logo.', true) ->param('url', '', new URL(), 'Project URL.', true) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 344a433103..77cf21efd2 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -164,7 +164,6 @@ $image = $this->getParam('image', ''); - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_ASSISTANT_OPENAI_API_KEY - - _APP_REGION appwrite-console: <<: *x-logging container_name: appwrite-console @@ -229,7 +228,6 @@ $image = $this->getParam('image', ''); - _APP_DB_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_REGION appwrite-worker-audits: image: /: @@ -256,7 +254,6 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - _APP_LOGGING_CONFIG - - _APP_REGION appwrite-worker-webhooks: image: /: @@ -285,7 +282,6 @@ $image = $this->getParam('image', ''); - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_LOGGING_CONFIG - - _APP_REGION appwrite-worker-deletes: image: /: @@ -347,7 +343,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_EMAIL_CERTIFICATES - - _APP_REGION + appwrite-worker-databases: image: /: @@ -374,7 +370,7 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - _APP_LOGGING_CONFIG - - _APP_REGION + appwrite-worker-builds: image: /: @@ -439,7 +435,7 @@ $image = $this->getParam('image', ''); - _APP_STORAGE_WASABI_SECRET - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - - _APP_REGION + appwrite-worker-certificates: image: /: @@ -473,7 +469,7 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - _APP_LOGGING_CONFIG - - _APP_REGION + appwrite-worker-functions: image: /: @@ -512,7 +508,7 @@ $image = $this->getParam('image', ''); - _APP_DOCKER_HUB_USERNAME - _APP_DOCKER_HUB_PASSWORD - _APP_LOGGING_CONFIG - - _APP_REGION + appwrite-worker-mails: image: /: @@ -547,7 +543,7 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_DOMAIN - _APP_OPTIONS_FORCE_HTTPS - - _APP_REGION + appwrite-worker-messaging: image: /: @@ -599,7 +595,7 @@ $image = $this->getParam('image', ''); - _APP_STORAGE_WASABI_SECRET - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - - _APP_REGION + appwrite-worker-migrations: image: /: @@ -630,7 +626,7 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - - _APP_REGION + appwrite-task-maintenance: image: /: @@ -665,7 +661,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - - _APP_REGION + appwrite-task-stats-resources: image: /: @@ -697,7 +693,7 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL - - _APP_REGION + appwrite-worker-stats-resources: image: /: @@ -726,7 +722,7 @@ $image = $this->getParam('image', ''); - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_STATS_RESOURCES_INTERVAL - - _APP_REGION + appwrite-worker-stats-usage: image: /: @@ -755,7 +751,7 @@ $image = $this->getParam('image', ''); - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - - _APP_REGION + appwrite-worker-stats-usage-dump: image: /: @@ -784,7 +780,7 @@ $image = $this->getParam('image', ''); - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - - _APP_REGION + appwrite-task-scheduler-functions: image: /: @@ -810,7 +806,7 @@ $image = $this->getParam('image', ''); - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_REGION + appwrite-task-scheduler-executions: image: /: @@ -836,7 +832,7 @@ $image = $this->getParam('image', ''); - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_REGION + appwrite-task-scheduler-messages: image: /: @@ -862,7 +858,7 @@ $image = $this->getParam('image', ''); - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_REGION + appwrite-assistant: image: appwrite/assistant:0.4.0 @@ -898,7 +894,6 @@ $image = $this->getParam('image', ''); - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_ENV=$_APP_ENV - - OPR_EXECUTOR_REGION=$_APP_REGION - OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG diff --git a/docker-compose.yml b/docker-compose.yml index ea796682ba..f1451426e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,7 +93,7 @@ services: - app/http.php environment: - _APP_ENV - - _APP_REGION + - _APP_EDITION - _APP_WORKER_PER_CORE - _APP_LOCALE @@ -269,7 +269,7 @@ services: - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-audits: entrypoint: worker-audits @@ -299,7 +299,7 @@ services: - _APP_DB_PASS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-webhooks: entrypoint: worker-webhooks @@ -332,7 +332,7 @@ services: - _APP_LOGGING_CONFIG - _APP_WEBHOOK_MAX_FAILED_ATTEMPTS - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-deletes: entrypoint: worker-deletes @@ -393,7 +393,7 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES - - _APP_REGION + appwrite-worker-databases: entrypoint: worker-databases @@ -425,7 +425,7 @@ services: - _APP_WORKERS_NUM - _APP_QUEUE_NAME - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-builds: entrypoint: worker-builds @@ -492,7 +492,7 @@ services: - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-certificates: entrypoint: worker-certificates @@ -528,7 +528,7 @@ services: - _APP_DB_PASS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-functions: entrypoint: worker-functions @@ -571,7 +571,7 @@ services: - _APP_LOGGING_CONFIG - _APP_LOGGING_PROVIDER - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-mails: entrypoint: worker-mails @@ -606,7 +606,7 @@ services: - _APP_DOMAIN - _APP_OPTIONS_FORCE_HTTPS - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-messaging: entrypoint: worker-messaging @@ -662,7 +662,7 @@ services: - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-migrations: entrypoint: worker-migrations @@ -698,7 +698,7 @@ services: - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-task-maintenance: entrypoint: maintenance @@ -737,7 +737,7 @@ services: - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_MAINTENANCE_DELAY - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-task-stats-resources: container_name: appwrite-task-stats-resources @@ -769,7 +769,7 @@ services: - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL - - _APP_REGION + appwrite-worker-stats-resources: entrypoint: worker-stats-resources @@ -801,7 +801,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-stats-usage: entrypoint: worker-stats-usage @@ -833,7 +833,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-stats-usage-dump: entrypoint: worker-stats-usage-dump @@ -866,7 +866,7 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - _APP_STATS_USAGE_DUAL_WRITING_DBS - - _APP_REGION + appwrite-task-scheduler-functions: entrypoint: schedule-functions @@ -895,7 +895,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-task-scheduler-executions: entrypoint: schedule-executions @@ -923,7 +923,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_REGION + appwrite-task-scheduler-messages: entrypoint: schedule-messages @@ -952,7 +952,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-assistant: container_name: appwrite-assistant diff --git a/src/Appwrite/Migration/Version/V20.php b/src/Appwrite/Migration/Version/V20.php index cca0d9244b..5a0807cedf 100644 --- a/src/Appwrite/Migration/Version/V20.php +++ b/src/Appwrite/Migration/Version/V20.php @@ -426,7 +426,7 @@ class V20 extends Migration 'period' => 'inf', 'value' => $value, 'time' => null, - 'region' => 'fra', + 'region' => 'default', ])); } catch (Duplicate $th) { Console::warning("Error while creating inf metric: duplicate id {$metric} {$id}"); diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index b62b95b066..dad2db0d9a 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -104,7 +104,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION')]), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); @@ -154,7 +154,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION')]), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index e8195d175d..9b0590181a 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -183,7 +183,7 @@ class Deletes extends Action $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION')]), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 3100a39dea..c0ff93e9b9 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -157,7 +157,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION'), + 'region' => System::getEnv('_APP_REGION', 'default') ]); $documentClone = new Document($document->getArrayCopy()); @@ -191,7 +191,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION'), + 'region' => System::getEnv('_APP_REGION', 'default'), ]); $documentClone = new Document($document->getArrayCopy()); $dbForProject->createOrUpdateDocumentsWithIncrease( diff --git a/tests/e2e/Scopes/ProjectCustom.php b/tests/e2e/Scopes/ProjectCustom.php index 2799e29772..7f84ace6f2 100644 --- a/tests/e2e/Scopes/ProjectCustom.php +++ b/tests/e2e/Scopes/ProjectCustom.php @@ -42,7 +42,7 @@ trait ProjectCustom 'x-appwrite-project' => 'console', ], [ 'projectId' => ID::unique(), - 'region' => 'fra', + 'region' => 'default', 'name' => 'Demo Project', 'teamId' => $team['body']['$id'], 'description' => 'Demo Project Description', diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 5e24dc4d89..914a255663 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -111,7 +111,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('fra', $output['APPWRITE_REGION']); + $this->assertEquals('default', $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); @@ -221,7 +221,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('fra', $output['APPWRITE_REGION']); + $this->assertEquals('default', $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 16a18d27c1..a6498b8a4e 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -49,7 +49,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'fra', + 'region' => 'default', ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -89,7 +89,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => '', 'teamId' => $team['body']['$id'], - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(400, $response['headers']['status-code']); @@ -100,7 +100,7 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders()), [ 'projectId' => ID::unique(), 'name' => 'Project Test', - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(401, $response['headers']['status-code']); @@ -129,7 +129,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => $projectId, 'name' => 'Project Duplicate', 'teamId' => $teamId, - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(409, $response['headers']['status-code']); @@ -178,7 +178,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Team 1 Project', 'teamId' => $team1, - 'region' => 'fra', + 'region' => 'default', ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -277,7 +277,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test 2', 'teamId' => $team['body']['$id'], - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -2042,7 +2042,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -2135,7 +2135,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3749,7 +3749,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project', 'teamId' => $teamId, - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3820,7 +3820,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 1', 'teamId' => $teamId, - 'region' => 'fra' + 'region' => 'default' ]); $project2 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ @@ -3830,7 +3830,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 2', 'teamId' => $teamId, - 'region' => 'fra' + 'region' => 'default' ]); $project1Id = $project1['body']['$id']; From e6e7c5c8161ca02f1c9fefb23285c98ad10eb7b9 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 18 Mar 2025 21:44:34 +0200 Subject: [PATCH 196/275] revert --- app/views/install/compose.phtml | 16 ---------------- docker-compose.yml | 19 ------------------- 2 files changed, 35 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 77cf21efd2..fd05d2a0b6 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -344,7 +344,6 @@ $image = $this->getParam('image', ''); - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_EMAIL_CERTIFICATES - appwrite-worker-databases: image: /: entrypoint: worker-databases @@ -371,7 +370,6 @@ $image = $this->getParam('image', ''); - _APP_DB_PASS - _APP_LOGGING_CONFIG - appwrite-worker-builds: image: /: entrypoint: worker-builds @@ -436,7 +434,6 @@ $image = $this->getParam('image', ''); - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - appwrite-worker-certificates: image: /: entrypoint: worker-certificates @@ -470,7 +467,6 @@ $image = $this->getParam('image', ''); - _APP_DB_PASS - _APP_LOGGING_CONFIG - appwrite-worker-functions: image: /: entrypoint: worker-functions @@ -509,7 +505,6 @@ $image = $this->getParam('image', ''); - _APP_DOCKER_HUB_PASSWORD - _APP_LOGGING_CONFIG - appwrite-worker-mails: image: /: entrypoint: worker-mails @@ -544,7 +539,6 @@ $image = $this->getParam('image', ''); - _APP_DOMAIN - _APP_OPTIONS_FORCE_HTTPS - appwrite-worker-messaging: image: /: entrypoint: worker-messaging @@ -596,7 +590,6 @@ $image = $this->getParam('image', ''); - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - appwrite-worker-migrations: image: /: entrypoint: worker-migrations @@ -627,7 +620,6 @@ $image = $this->getParam('image', ''); - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - appwrite-task-maintenance: image: /: entrypoint: maintenance @@ -662,7 +654,6 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - appwrite-task-stats-resources: image: /: container_name: appwrite-task-stats-resources @@ -694,7 +685,6 @@ $image = $this->getParam('image', ''); - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL - appwrite-worker-stats-resources: image: /: entrypoint: worker-stats-resources @@ -723,7 +713,6 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_STATS_RESOURCES_INTERVAL - appwrite-worker-stats-usage: image: /: entrypoint: worker-stats-usage @@ -752,7 +741,6 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - appwrite-worker-stats-usage-dump: image: /: entrypoint: worker-stats-usage-dump @@ -781,7 +769,6 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - appwrite-task-scheduler-functions: image: /: entrypoint: schedule-functions @@ -807,7 +794,6 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - appwrite-task-scheduler-executions: image: /: entrypoint: schedule-executions @@ -833,7 +819,6 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - appwrite-task-scheduler-messages: image: /: entrypoint: schedule-messages @@ -859,7 +844,6 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - appwrite-assistant: image: appwrite/assistant:0.4.0 container_name: appwrite-assistant diff --git a/docker-compose.yml b/docker-compose.yml index f1451426e8..5933e37611 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -270,7 +270,6 @@ services: - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - appwrite-worker-audits: entrypoint: worker-audits <<: *x-logging @@ -300,7 +299,6 @@ services: - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - appwrite-worker-webhooks: entrypoint: worker-webhooks <<: *x-logging @@ -333,7 +331,6 @@ services: - _APP_WEBHOOK_MAX_FAILED_ATTEMPTS - _APP_DATABASE_SHARED_TABLES - appwrite-worker-deletes: entrypoint: worker-deletes <<: *x-logging @@ -394,7 +391,6 @@ services: - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES - appwrite-worker-databases: entrypoint: worker-databases <<: *x-logging @@ -426,7 +422,6 @@ services: - _APP_QUEUE_NAME - _APP_DATABASE_SHARED_TABLES - appwrite-worker-builds: entrypoint: worker-builds <<: *x-logging @@ -493,7 +488,6 @@ services: - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES - appwrite-worker-certificates: entrypoint: worker-certificates <<: *x-logging @@ -529,7 +523,6 @@ services: - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - appwrite-worker-functions: entrypoint: worker-functions <<: *x-logging @@ -572,7 +565,6 @@ services: - _APP_LOGGING_PROVIDER - _APP_DATABASE_SHARED_TABLES - appwrite-worker-mails: entrypoint: worker-mails <<: *x-logging @@ -607,7 +599,6 @@ services: - _APP_OPTIONS_FORCE_HTTPS - _APP_DATABASE_SHARED_TABLES - appwrite-worker-messaging: entrypoint: worker-messaging <<: *x-logging @@ -663,7 +654,6 @@ services: - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES - appwrite-worker-migrations: entrypoint: worker-migrations <<: *x-logging @@ -699,7 +689,6 @@ services: - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_DATABASE_SHARED_TABLES - appwrite-task-maintenance: entrypoint: maintenance <<: *x-logging @@ -738,7 +727,6 @@ services: - _APP_MAINTENANCE_DELAY - _APP_DATABASE_SHARED_TABLES - appwrite-task-stats-resources: container_name: appwrite-task-stats-resources entrypoint: stats-resources @@ -770,7 +758,6 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL - appwrite-worker-stats-resources: entrypoint: worker-stats-resources <<: *x-logging @@ -802,7 +789,6 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - appwrite-worker-stats-usage: entrypoint: worker-stats-usage <<: *x-logging @@ -834,7 +820,6 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - appwrite-worker-stats-usage-dump: entrypoint: worker-stats-usage-dump <<: *x-logging @@ -867,7 +852,6 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_STATS_USAGE_DUAL_WRITING_DBS - appwrite-task-scheduler-functions: entrypoint: schedule-functions <<: *x-logging @@ -896,7 +880,6 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - appwrite-task-scheduler-executions: entrypoint: schedule-executions <<: *x-logging @@ -924,7 +907,6 @@ services: - _APP_DB_USER - _APP_DB_PASS - appwrite-task-scheduler-messages: entrypoint: schedule-messages <<: *x-logging @@ -953,7 +935,6 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - appwrite-assistant: container_name: appwrite-assistant image: appwrite/assistant:0.7.0 From ee09e2c91b18c280b2ebbf082df06a9e9351fa83 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 18 Mar 2025 21:47:54 +0200 Subject: [PATCH 197/275] revert --- docker-compose.yml | 5 +---- src/Appwrite/Platform/Workers/StatsUsageDump.php | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5933e37611..3f9f54bc1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,7 +93,6 @@ services: - app/http.php environment: - _APP_ENV - - _APP_EDITION - _APP_WORKER_PER_CORE - _APP_LOCALE @@ -788,7 +787,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - + - appwrite-worker-stats-usage: entrypoint: worker-stats-usage <<: *x-logging @@ -967,7 +966,6 @@ services: - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_ENV=$_APP_ENV - - OPR_EXECUTOR_REGION=$_APP_REGION - OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - OPR_EXECUTOR_RUNTIME_VERSIONS=v2,v4 @@ -1007,7 +1005,6 @@ services: environment: - OPR_PROXY_WORKER_PER_CORE=$_APP_WORKER_PER_CORE - OPR_PROXY_ENV=$_APP_ENV - - OPR_PROXY_REGION=$_APP_REGION - OPR_PROXY_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - OPR_PROXY_SECRET=$_APP_EXECUTOR_SECRET - OPR_PROXY_LOGGING_CONFIG=$_APP_LOGGING_CONFIG diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index c0ff93e9b9..747e6fdbfd 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -191,7 +191,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => System::getEnv('_APP_REGION', 'default') ]); $documentClone = new Document($document->getArrayCopy()); $dbForProject->createOrUpdateDocumentsWithIncrease( From eb4340cf4c3f3a445ced961f2e75f0957035d11b Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 18 Mar 2025 21:49:28 +0200 Subject: [PATCH 198/275] revert --- docker-compose.yml | 2 +- src/Appwrite/Platform/Workers/StatsUsageDump.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3f9f54bc1e..b6dc80df6a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -787,7 +787,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - - + appwrite-worker-stats-usage: entrypoint: worker-stats-usage <<: *x-logging diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 747e6fdbfd..119a9e7288 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -157,7 +157,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default') + 'region' => System::getEnv('_APP_REGION', 'default'), ]); $documentClone = new Document($document->getArrayCopy()); @@ -191,7 +191,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default') + 'region' => System::getEnv('_APP_REGION', 'default'), ]); $documentClone = new Document($document->getArrayCopy()); $dbForProject->createOrUpdateDocumentsWithIncrease( From faed019d921847e05ff9ff5f7d2b42684dc8424e Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 19 Mar 2025 09:32:48 +0200 Subject: [PATCH 199/275] revert --- tests/e2e/Scopes/ProjectCustom.php | 3 ++- .../Projects/ProjectsConsoleClientTest.php | 23 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/e2e/Scopes/ProjectCustom.php b/tests/e2e/Scopes/ProjectCustom.php index 7f84ace6f2..533fccd87d 100644 --- a/tests/e2e/Scopes/ProjectCustom.php +++ b/tests/e2e/Scopes/ProjectCustom.php @@ -4,6 +4,7 @@ namespace Tests\E2E\Scopes; use Tests\E2E\Client; use Utopia\Database\Helpers\ID; +use Utopia\System\System; trait ProjectCustom { @@ -42,7 +43,7 @@ trait ProjectCustom 'x-appwrite-project' => 'console', ], [ 'projectId' => ID::unique(), - 'region' => 'default', + 'region' => System::getEnv('_APP_REGION', 'default'), 'name' => 'Demo Project', 'teamId' => $team['body']['$id'], 'description' => 'Demo Project Description', diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index a6498b8a4e..8e2ab03880 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -14,6 +14,7 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; +use Utopia\System\System; class ProjectsConsoleClientTest extends Scope { @@ -49,7 +50,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default', + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -89,7 +90,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => '', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(400, $response['headers']['status-code']); @@ -100,7 +101,7 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders()), [ 'projectId' => ID::unique(), 'name' => 'Project Test', - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(401, $response['headers']['status-code']); @@ -129,7 +130,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => $projectId, 'name' => 'Project Duplicate', 'teamId' => $teamId, - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(409, $response['headers']['status-code']); @@ -178,7 +179,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Team 1 Project', 'teamId' => $team1, - 'region' => 'default', + 'region' => System::getEnv('_APP_REGION', 'default'), ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -277,7 +278,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test 2', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -2042,7 +2043,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -2135,7 +2136,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3749,7 +3750,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project', 'teamId' => $teamId, - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3820,7 +3821,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 1', 'teamId' => $teamId, - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $project2 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ @@ -3830,7 +3831,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 2', 'teamId' => $teamId, - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $project1Id = $project1['body']['$id']; From 88d6150db2f67ffaf1a4918f0427d444fca770d1 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 19 Mar 2025 10:29:18 +0200 Subject: [PATCH 200/275] revert --- tests/e2e/Services/Functions/FunctionsCustomClientTest.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 914a255663..55db081720 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -8,6 +8,7 @@ use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideClient; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Role; +use Utopia\System\System; class FunctionsCustomClientTest extends Scope { @@ -111,7 +112,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('default', $output['APPWRITE_REGION']); + $this->assertEquals(System::getEnv('_APP_REGION', 'default'), $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); @@ -221,7 +222,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('default', $output['APPWRITE_REGION']); + $this->assertEquals(System::getEnv('_APP_REGION', 'default'), $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); From cc0a97b10e30290fce85fc1a84cf5f9948c8f391 Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 19 Mar 2025 16:37:15 +0200 Subject: [PATCH 201/275] Fixes --- app/config/collections/common.php | 6 +++--- src/Appwrite/Platform/Workers/Deletes.php | 24 ++++++++++++----------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/app/config/collections/common.php b/app/config/collections/common.php index f68400e226..3f6245dfdd 100644 --- a/app/config/collections/common.php +++ b/app/config/collections/common.php @@ -1038,7 +1038,7 @@ return [ '$id' => ID::custom('providerUid'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 2048, + 'size' => 2048, // Decrease to 128 as in index length? 'signed' => true, 'required' => false, 'default' => null, @@ -1107,14 +1107,14 @@ return [ '$id' => ID::custom('_key_userInternalId_provider_providerUid'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['userInternalId', 'provider', 'providerUid'], - 'lengths' => [11, 128, 128], + 'lengths' => [11, 128, 128], // providerUid is length 2000! 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ '$id' => ID::custom('_key_provider_providerUid'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['provider', 'providerUid'], - 'lengths' => [128, 128], + 'lengths' => [128, 128], // providerUid is length 2000! 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index a14e54caab..27ff27fe31 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -357,21 +357,23 @@ class Deletes extends Action new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) ); - $query[] = Query::equal('resource', [$resource]); + $queries = [ + Query::equal('resource', [$resource]) + ]; if (!empty($resourceType)) { - $query[] = Query::equal('resourceType', [$resourceType]); + $queries[] = Query::equal('resourceType', [$resourceType]); } - $query[] = Query::orderAsc(); + $queries[] = Query::orderAsc(); /** - * todo: missing index on `resource`, `resourceType` + * todo: No index on `resource`, `resourceType`, But it is fine since resource index is a strong index! */ $this->deleteByGroup( 'cache', - $query, + $queries, $dbForProject, function (Document $document) use ($cache, $projectId) { $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); @@ -719,7 +721,7 @@ class Deletes extends Action // Delete identities /** - * todo: Remove Duplication index `_key_userInternalId` + * todo: Remove Duplication index `_key_userInternalId` , But lets leave because of index length issue in unique index */ $this->deleteByGroup('identities', [ Query::equal('userInternalId', [$userInternalId]), @@ -836,13 +838,13 @@ class Deletes extends Action /** * Delete rules - * todo: No index for this query + * todo: No index for this query, drop _key_projectInternalId and create _key_projectInternalId, resourceInternalId, resourceType */ Console::info("Deleting rules for function " . $functionId); $this->deleteByGroup('rules', [ - Query::equal('resourceType', ['function']), - Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('projectInternalId', [$project->getInternalId()]), + Query::equal('resourceInternalId', [$functionInternalId]), + Query::equal('resourceType', ['function']), Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($project, $dbForPlatform, $certificates) { $this->deleteRule($dbForPlatform, $document, $certificates); @@ -850,12 +852,12 @@ class Deletes extends Action /** * Delete Variables - * todo: No index for this query + * todo: No index for this query , drop _key_resourceInternalId and create new one with {resourceInternalId, resourceType} */ Console::info("Deleting variables for function " . $functionId); $this->deleteByGroup('variables', [ - Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), + Query::equal('resourceType', ['function']), Query::orderAsc() ], $dbForProject); From 31ca35782baa46d73a2ad75cdb209f6c08d7bc97 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 20 Mar 2025 15:21:24 +1300 Subject: [PATCH 202/275] Use cursor pagination with bigger limit for maintenance project loop --- src/Appwrite/Platform/Tasks/Maintenance.php | 33 +++------------------ 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 98a3f4d295..b9e312a3fb 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -47,13 +47,15 @@ class Maintenance extends Action Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); - $this->foreachProject($dbForPlatform, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) { + $dbForPlatform->foreach('projects', function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) { $queueForDeletes ->setType(DELETE_TYPE_MAINTENANCE) ->setProject($project) ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) ->trigger(); - }); + }, [ + Query::limit(100), + ]); $queueForDeletes ->setType(DELETE_TYPE_MAINTENANCE) @@ -68,33 +70,6 @@ class Maintenance extends Action }, $interval, $delay); } - protected function foreachProject(Database $dbForPlatform, callable $callback): void - { - // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document - $count = 0; - $chunk = 0; - $limit = 50; - $sum = $limit; - $executionStart = \microtime(true); - - while ($sum === $limit) { - $projects = $dbForPlatform->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]); - - $chunk++; - - /** @var string[] $projectIds */ - $sum = count($projects); - - foreach ($projects as $project) { - $callback($project); - $count++; - } - } - - $executionEnd = \microtime(true); - Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds"); - } - private function notifyDeleteConnections(Delete $queueForDeletes): void { $queueForDeletes From 9c54cce5bd8c61d26a549273dfe30b6bcfd94751 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 20 Mar 2025 09:07:32 +0200 Subject: [PATCH 203/275] Queries fixes --- src/Appwrite/Platform/Workers/Deletes.php | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 27ff27fe31..b7fce0405d 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -365,6 +365,7 @@ class Deletes extends Action $queries[] = Query::equal('resourceType', [$resourceType]); } + $queries[] = Query::select(['$internalId', '$id', '$updatedAt']); $queries[] = Query::orderAsc(); /** @@ -404,7 +405,8 @@ class Deletes extends Action new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) ); - $query = [ + $queries = [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), Query::orderDesc(), @@ -412,7 +414,7 @@ class Deletes extends Action $this->deleteByGroup( 'cache', - $query, + $queries, $dbForProject, function (Document $document) use ($cache, $projectId) { $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); @@ -440,6 +442,7 @@ class Deletes extends Action // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -452,6 +455,7 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -755,6 +759,7 @@ class Deletes extends Action // Delete Executions $this->deleteByGroup('executions', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -775,6 +780,7 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -811,6 +817,7 @@ class Deletes extends Action try { $this->deleteByGroup(Audit::COLLECTION, [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderAsc(), // KEY "index-time" ("time" DESC) @@ -897,6 +904,7 @@ class Deletes extends Action */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::equal('functionInternalId', [$functionInternalId]), Query::orderAsc() ], $dbForProject); @@ -1017,7 +1025,7 @@ class Deletes extends Action /** * Delete builds - * todo: no index for `deploymentInternalId` change to temporary to deploymentId? + * todo: no index for `deploymentInternalId` Same as above index , no need to handle again... */ Console::info("Deleting builds for deployment " . $deploymentId); @@ -1047,8 +1055,7 @@ class Deletes extends Action string $collection, array $queries, Database $database, - ?callable $callback = null, - bool $shortSelect = false + ?callable $callback = null ): void { $start = \microtime(true); @@ -1056,12 +1063,6 @@ class Deletes extends Action * deleteDocuments uses a cursor, we need to add a unique order by field or use default */ - if (!\is_callable($callback) && $shortSelect) { - $queries = array_merge($queries, [ - Query::select(['$internalId', '$id', '$permissions', '$updatedAt']) - ]); - } - try { $documents = $database->deleteDocuments($collection, $queries); } catch (Throwable $th) { From 5eb7c36f50e2713aee76beaff1bb207eca130116 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 10:32:36 +0000 Subject: [PATCH 204/275] chore: add console audit retention --- .env | 2 +- app/worker.php | 2 +- docker-compose.yml | 1 + src/Appwrite/Platform/Workers/Deletes.php | 9 ++++++++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 1893e023ba..c68f0a0d52 100644 --- a/.env +++ b/.env @@ -85,7 +85,7 @@ _APP_MAINTENANCE_DELAY= _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 -_APP_MAINTENANCE_RETENTION_AUDIT=1209600 +_APP_MAINTENANCE_RETENTION_AUDIT=project=1209600,console=15778800 _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=3600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 diff --git a/app/worker.php b/app/worker.php index eeefe80000..5d7bd4de41 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,7 +219,7 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); + return System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800'); // project = 14 days, console = 6 months }); Server::setResource('executionRetention', function () { diff --git a/docker-compose.yml b/docker-compose.yml index b6dc80df6a..af6a323ec8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -389,6 +389,7 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES + - _APP_MAINTENANCE_RETENTION_AUDIT appwrite-worker-databases: entrypoint: worker-databases diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9b0590181a..75380ed3fe 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -751,9 +751,16 @@ class Deletes extends Action $projectId = $project->getId(); $dbForProject = $getProjectDB($project); + [$projectAuditRetention, $consoleAuditRetention] = array_map( + function ($part) { + return DateTime::addSeconds(new \DateTime(), -1 * (int)explode('=', $part)[1]); + }, + explode(',', $auditRetention) + ); + try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::lessThan('time', $auditRetention), + Query::lessThan('time', ($projectId === 'console' ? $consoleAuditRetention : $projectAuditRetention)), Query::orderDesc('time'), Query::orderDesc('$internalId'), ], $dbForProject); From 9d8289221925ddf7eeebbd42f56ba9be311dd2a0 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 11:32:06 +0000 Subject: [PATCH 205/275] chore: update logic --- app/worker.php | 10 +++++++++- src/Appwrite/Platform/Workers/Deletes.php | 15 ++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/app/worker.php b/app/worker.php index 5d7bd4de41..18b51eda25 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,7 +219,15 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800'); // project = 14 days, console = 6 months + return array_map( + function ($part) { + [$key, $value] = explode('=', $part); + return [ + $key => DateTime::addSeconds(new \DateTime(), -1 * (int)$value) + ]; + }, + explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')) + ); }); Server::setResource('executionRetention', function () { diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 75380ed3fe..93f8748e49 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -57,7 +57,7 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log) => $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -746,21 +746,14 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void + private function deleteAuditLogs(Document $project, callable $getProjectDB, array $auditRetention): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); - [$projectAuditRetention, $consoleAuditRetention] = array_map( - function ($part) { - return DateTime::addSeconds(new \DateTime(), -1 * (int)explode('=', $part)[1]); - }, - explode(',', $auditRetention) - ); - try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::lessThan('time', ($projectId === 'console' ? $consoleAuditRetention : $projectAuditRetention)), + Query::lessThan('time', ($projectId === 'console' ? $auditRetention['console'] : $auditRetention['project'])), Query::orderDesc('time'), Query::orderDesc('$internalId'), ], $dbForProject); From 7c3f8504bb65e598f50d07912b5d753c86f5e1f6 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 13:15:14 +0000 Subject: [PATCH 206/275] chore: fix initialization --- app/worker.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/worker.php b/app/worker.php index 18b51eda25..e305ec42d6 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,14 +219,14 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return array_map( - function ($part) { + return array_reduce( + explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), + function ($carry, $part) { [$key, $value] = explode('=', $part); - return [ - $key => DateTime::addSeconds(new \DateTime(), -1 * (int)$value) - ]; + $carry[$key] = DateTime::addSeconds(new \DateTime(), -1 * (int)$value); + return $carry; }, - explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')) + [] ); }); From b0b57935da0e54074766a3f9a144f41fe2081a55 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 13:42:19 +0000 Subject: [PATCH 207/275] chore: add comments --- app/worker.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/worker.php b/app/worker.php index e305ec42d6..d0fa5cf4f8 100644 --- a/app/worker.php +++ b/app/worker.php @@ -215,12 +215,12 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache) { }, ['pools', 'cache']); Server::setResource('abuseRetention', function () { - return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); + return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); // 1 day }); Server::setResource('auditRetention', function () { return array_reduce( - explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), + explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), // project = 14 days, console = 6 months function ($carry, $part) { [$key, $value] = explode('=', $part); $carry[$key] = DateTime::addSeconds(new \DateTime(), -1 * (int)$value); @@ -231,7 +231,7 @@ Server::setResource('auditRetention', function () { }); Server::setResource('executionRetention', function () { - return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); // 14 days }); Server::setResource('cache', function (Registry $register) { From 13750f7dbd5b5850075eefab4bacf3e782752004 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 07:42:10 +0000 Subject: [PATCH 208/275] feat: introduce new env --- .env | 3 ++- app/config/variables.php | 11 ++++++++++- app/views/install/compose.phtml | 3 +++ app/worker.php | 13 ++++--------- docker-compose.yml | 3 +++ 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.env b/.env index c68f0a0d52..c10c12613b 100644 --- a/.env +++ b/.env @@ -85,7 +85,8 @@ _APP_MAINTENANCE_DELAY= _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 -_APP_MAINTENANCE_RETENTION_AUDIT=project=1209600,console=15778800 +_APP_MAINTENANCE_RETENTION_AUDIT=1209600 +_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE=15778800 _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=3600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 diff --git a/app/config/variables.php b/app/config/variables.php index 98dd9ffec1..27463d2fee 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -1048,13 +1048,22 @@ return [ ], [ 'name' => '_APP_MAINTENANCE_RETENTION_AUDIT', - 'description' => 'IThe maximum duration (in seconds) upto which to retain audit logs. The default value is 1209600 seconds (14 days).', + 'description' => 'The maximum duration (in seconds) upto which to retain audit logs. The default value is 1209600 seconds (14 days).', 'introduction' => '0.7.0', 'default' => '1209600', 'required' => false, 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', + 'description' => 'The maximum duration (in seconds) upto which to retain console audit logs. The default value is 15778800 seconds (6 months).', + 'introduction' => '1.6.2', + 'default' => '15778800', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_MAINTENANCE_RETENTION_ABUSE', 'description' => 'The maximum duration (in seconds) upto which to retain abuse logs. The default value is 86400 seconds (1 day).', diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index fd05d2a0b6..7dfe14fcef 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -148,6 +148,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_SMS_PROVIDER @@ -340,6 +341,7 @@ $image = $this->getParam('image', ''); - _APP_EXECUTOR_HOST - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_EMAIL_CERTIFICATES @@ -651,6 +653,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES diff --git a/app/worker.php b/app/worker.php index d0fa5cf4f8..3f99089eef 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,15 +219,10 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return array_reduce( - explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), // project = 14 days, console = 6 months - function ($carry, $part) { - [$key, $value] = explode('=', $part); - $carry[$key] = DateTime::addSeconds(new \DateTime(), -1 * (int)$value); - return $carry; - }, - [] - ); + return [ + 'project' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', '1209600')), + 'console' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', '15778800')) + ]; }); Server::setResource('executionRetention', function () { diff --git a/docker-compose.yml b/docker-compose.yml index af6a323ec8..05ddba967a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -171,6 +171,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_SMS_PROVIDER @@ -390,6 +391,7 @@ services: - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE appwrite-worker-databases: entrypoint: worker-databases @@ -722,6 +724,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_MAINTENANCE_DELAY From 7c795da72fd7aa05b280f27012f87357a08d2814 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 07:48:50 +0000 Subject: [PATCH 209/275] chore: fix type --- app/worker.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/worker.php b/app/worker.php index 3f99089eef..744c758c69 100644 --- a/app/worker.php +++ b/app/worker.php @@ -220,8 +220,8 @@ Server::setResource('abuseRetention', function () { Server::setResource('auditRetention', function () { return [ - 'project' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', '1209600')), - 'console' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', '15778800')) + 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), + 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) ]; }); From c0c0c921dc249f2f235a9bd7e8efdce66ae3c137 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 08:08:30 +0000 Subject: [PATCH 210/275] chore: fix naming --- app/worker.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/worker.php b/app/worker.php index 744c758c69..491f266a18 100644 --- a/app/worker.php +++ b/app/worker.php @@ -220,8 +220,8 @@ Server::setResource('abuseRetention', function () { Server::setResource('auditRetention', function () { return [ - 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), - 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) + 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), // 14 days + 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) // 6 months ]; }); From d661195a5f4bf375f7b72e5629e1a1dfd3fe8d3d Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 15:59:30 +0000 Subject: [PATCH 211/275] chore: only update total count for prviledged users --- app/controllers/api/teams.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index b45c9fd3b9..dfdccf5e99 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -615,7 +615,10 @@ App::post('/v1/teams/:teamId/memberships') $membership = ($isPrivilegedUser || $isAppUser) ? Authorization::skip(fn () => $dbForProject->createDocument('memberships', $membership)) : $dbForProject->createDocument('memberships', $membership); - Authorization::skip(fn () => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1)); + + if ($isPrivilegedUser || $isAppUser) { + Authorization::skip(fn () => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1)); + } } elseif ($membership->getAttribute('confirm') === false) { $membership->setAttribute('secret', Auth::hash($secret)); From 7c2ab4fcf652b2823a42c04daf8ce4c0233eb609 Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 22 Mar 2025 13:47:09 +0530 Subject: [PATCH 212/275] update: change sample rate for 4xx via env. --- app/controllers/general.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index a374aa3c39..1423763cec 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -871,17 +871,18 @@ App::error() if (!empty($providerConfig) && $error->getCode() >= 400 && $error->getCode() < 500) { // Register error logger try { - $loggingProvider = new DSN($providerConfig ?? ''); + $loggingProvider = new DSN($providerConfig); $providerName = $loggingProvider->getScheme(); if (!empty($providerName) && $providerName === 'sentry') { $key = $loggingProvider->getPassword(); $projectId = $loggingProvider->getUser() ?? ''; $host = 'https://' . $loggingProvider->getHost(); + $sampleRate = $loggingProvider->getParam('sample', 0.01); $adapter = new Sentry($projectId, $key, $host); $logger = new Logger($adapter); - $logger->setSample(0.01); + $logger->setSample($sampleRate); $publish = true; } else { throw new \Exception('Invalid experimental logging provider'); From 86856bf87d1afa58543411eda105fb975009e5db Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 22 Mar 2025 13:55:23 +0000 Subject: [PATCH 213/275] chore: update gitpod.yml config --- .gitpod.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 478b62fc8d..e9c0928229 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,6 +1,7 @@ tasks: - name: Run Appwrite Docker Stack init: | + git config --global --add safe.directory /app git submodule update --init docker compose build docker compose pull @@ -23,11 +24,3 @@ vscode: extensions: - ms-azuretools.vscode-docker - zobo.php-intellisense - -github: - # https://www.gitpod.io/docs/prebuilds#github-specific-configuration - prebuilds: - # enable for pull requests coming from forks (defaults to false) - pullRequestsFromForks: true - # add a check to pull requests (defaults to true) - addCheck: false From 37dfe0d719b49f32b2540b3d6b3310fcd110e976 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 22 Mar 2025 14:05:39 +0000 Subject: [PATCH 214/275] refactor: initialization of audit retention --- app/worker.php | 14 ++++++-------- src/Appwrite/Platform/Workers/Deletes.php | 10 +++++----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/app/worker.php b/app/worker.php index 491f266a18..90496c0430 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,8 +16,6 @@ use Appwrite\Event\Migration; use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; use Appwrite\Event\StatsUsageDump; -/** remove */ -/** /remove */ use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; use Swoole\Runtime; @@ -218,12 +216,12 @@ Server::setResource('abuseRetention', function () { return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); // 1 day }); -Server::setResource('auditRetention', function () { - return [ - 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), // 14 days - 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) // 6 months - ]; -}); +Server::setResource('auditRetention', function (Document $project) { + if ($project->getId() === 'console') { + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)); // 6 months + } + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); // 14 days +}, ['project']); Server::setResource('executionRetention', function () { return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); // 14 days diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 93f8748e49..bb5cc81fcc 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -57,7 +57,7 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log) => + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -742,18 +742,18 @@ class Deletes extends Action /** * @param Database $dbForPlatform * @param callable $getProjectDB - * @param string $datetime + * @param string $auditRetention * @return void * @throws Exception */ - private function deleteAuditLogs(Document $project, callable $getProjectDB, array $auditRetention): void + private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::lessThan('time', ($projectId === 'console' ? $auditRetention['console'] : $auditRetention['project'])), + Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderDesc('$internalId'), ], $dbForProject); From 26821c83726db7506fbf2d0828f34a908cb5d1d3 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 11:08:39 +0200 Subject: [PATCH 215/275] Bump database 0.62 --- composer.json | 2 +- composer.lock | 311 ++++---------------------------------------------- 2 files changed, 22 insertions(+), 291 deletions(-) diff --git a/composer.json b/composer.json index d7b8505b5c..b1b6aed539 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.61.*", + "utopia-php/database": "0.62.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index cdba756c72..1ea887fdbc 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e0d7f21b681e4591144fec16c4f0d6aa", + "content-hash": "d2bec8137dcd84994121f89a29932d31", "packages": [ { "name": "adhocore/jwt", @@ -751,65 +751,6 @@ }, "time": "2025-03-13T21:08:17+00:00" }, - { - "name": "jean85/pretty-package-versions", - "version": "2.1.0", - "source": { - "type": "git", - "url": "https://github.com/Jean85/pretty-package-versions.git", - "reference": "3c4e5f62ba8d7de1734312e4fff32f67a8daaf10" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/Jean85/pretty-package-versions/zipball/3c4e5f62ba8d7de1734312e4fff32f67a8daaf10", - "reference": "3c4e5f62ba8d7de1734312e4fff32f67a8daaf10", - "shasum": "" - }, - "require": { - "composer-runtime-api": "^2.1.0", - "php": "^7.4|^8.0" - }, - "require-dev": { - "friendsofphp/php-cs-fixer": "^3.2", - "jean85/composer-provided-replaced-stub-package": "^1.0", - "phpstan/phpstan": "^1.4", - "phpunit/phpunit": "^7.5|^8.5|^9.6", - "vimeo/psalm": "^4.3 || ^5.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.x-dev" - } - }, - "autoload": { - "psr-4": { - "Jean85\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Alessandro Lai", - "email": "alessandro.lai85@gmail.com" - } - ], - "description": "A library to get pretty versions strings of installed dependencies", - "keywords": [ - "composer", - "package", - "release", - "versions" - ], - "support": { - "issues": "https://github.com/Jean85/pretty-package-versions/issues", - "source": "https://github.com/Jean85/pretty-package-versions/tree/2.1.0" - }, - "time": "2024-11-18T16:19:46+00:00" - }, { "name": "league/csv", "version": "9.14.0", @@ -968,75 +909,6 @@ }, "time": "2023-10-02T10:01:54+00:00" }, - { - "name": "mongodb/mongodb", - "version": "1.10.0", - "source": { - "type": "git", - "url": "https://github.com/mongodb/mongo-php-library.git", - "reference": "b0bbd657f84219212487d01a8ffe93a789e1e488" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/b0bbd657f84219212487d01a8ffe93a789e1e488", - "reference": "b0bbd657f84219212487d01a8ffe93a789e1e488", - "shasum": "" - }, - "require": { - "ext-hash": "*", - "ext-json": "*", - "ext-mongodb": "^1.11.0", - "jean85/pretty-package-versions": "^1.2 || ^2.0.1", - "php": "^7.1 || ^8.0", - "symfony/polyfill-php80": "^1.19" - }, - "require-dev": { - "doctrine/coding-standard": "^9.0", - "squizlabs/php_codesniffer": "^3.6", - "symfony/phpunit-bridge": "^5.2" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.10.x-dev" - } - }, - "autoload": { - "files": [ - "src/functions.php" - ], - "psr-4": { - "MongoDB\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "Apache-2.0" - ], - "authors": [ - { - "name": "Andreas Braun", - "email": "andreas.braun@mongodb.com" - }, - { - "name": "Jeremy Mikola", - "email": "jmikola@gmail.com" - } - ], - "description": "MongoDB driver library", - "homepage": "https://jira.mongodb.org/browse/PHPLIB", - "keywords": [ - "database", - "driver", - "mongodb", - "persistence" - ], - "support": { - "issues": "https://github.com/mongodb/mongo-php-library/issues", - "source": "https://github.com/mongodb/mongo-php-library/tree/1.10.0" - }, - "time": "2021-10-20T22:22:37+00:00" - }, { "name": "mustangostang/spyc", "version": "0.6.3", @@ -2371,16 +2243,16 @@ }, { "name": "ramsey/collection", - "version": "2.1.0", + "version": "2.1.1", "source": { "type": "git", "url": "https://github.com/ramsey/collection.git", - "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109" + "reference": "344572933ad0181accbf4ba763e85a0306a8c5e2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ramsey/collection/zipball/3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", - "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", + "url": "https://api.github.com/repos/ramsey/collection/zipball/344572933ad0181accbf4ba763e85a0306a8c5e2", + "reference": "344572933ad0181accbf4ba763e85a0306a8c5e2", "shasum": "" }, "require": { @@ -2441,9 +2313,9 @@ ], "support": { "issues": "https://github.com/ramsey/collection/issues", - "source": "https://github.com/ramsey/collection/tree/2.1.0" + "source": "https://github.com/ramsey/collection/tree/2.1.1" }, - "time": "2025-03-02T04:48:29+00:00" + "time": "2025-03-22T05:38:12+00:00" }, { "name": "ramsey/uuid", @@ -2932,86 +2804,6 @@ ], "time": "2024-09-09T11:45:10+00:00" }, - { - "name": "symfony/polyfill-php80", - "version": "v1.31.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "60328e362d4c2c802a54fcbf04f9d3fb892b4cf8" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/60328e362d4c2c802a54fcbf04f9d3fb892b4cf8", - "reference": "60328e362d4c2c802a54fcbf04f9d3fb892b4cf8", - "shasum": "" - }, - "require": { - "php": ">=7.2" - }, - "type": "library", - "extra": { - "thanks": { - "url": "https://github.com/symfony/polyfill", - "name": "symfony/polyfill" - } - }, - "autoload": { - "files": [ - "bootstrap.php" - ], - "psr-4": { - "Symfony\\Polyfill\\Php80\\": "" - }, - "classmap": [ - "Resources/stubs" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Ion Bazan", - "email": "ion.bazan@gmail.com" - }, - { - "name": "Nicolas Grekas", - "email": "p@tchwork.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill backporting some PHP 8.0+ features to lower PHP versions", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "polyfill", - "portable", - "shim" - ], - "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.31.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2024-09-09T11:45:10+00:00" - }, { "name": "symfony/polyfill-php82", "version": "v1.31.0", @@ -3705,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.61.2", + "version": "0.62.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436" + "reference": "65dc51466c12552add10395900cdbb4728da4068" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/349fbdf4bc088f7775c7dfb8b80239a617a88436", - "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436", + "url": "https://api.github.com/repos/utopia-php/database/zipball/65dc51466c12552add10395900cdbb4728da4068", + "reference": "65dc51466c12552add10395900cdbb4728da4068", "shasum": "" }, "require": { @@ -3722,8 +3514,7 @@ "ext-pdo": "*", "php": ">=8.1", "utopia-php/cache": "0.12.*", - "utopia-php/framework": "0.33.*", - "utopia-php/mongo": "0.3.*" + "utopia-php/framework": "0.33.*" }, "require-dev": { "fakerphp/faker": "1.23.*", @@ -3755,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.61.2" + "source": "https://github.com/utopia-php/database/tree/0.62.1" }, - "time": "2025-03-15T11:47:42+00:00" + "time": "2025-03-24T08:27:18+00:00" }, { "name": "utopia-php/domains", @@ -4159,16 +3950,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.1", + "version": "0.8.2", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab" + "reference": "aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab", - "reference": "36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1", + "reference": "aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1", "shasum": "" }, "require": { @@ -4176,7 +3967,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.61.*", + "utopia-php/database": "0.62.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4209,69 +4000,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.1" + "source": "https://github.com/utopia-php/migration/tree/0.8.2" }, - "time": "2025-03-18T07:48:08+00:00" - }, - { - "name": "utopia-php/mongo", - "version": "0.3.1", - "source": { - "type": "git", - "url": "https://github.com/utopia-php/mongo.git", - "reference": "52326a9a43e2d27ff0c15c48ba746dacbe9a7aee" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/52326a9a43e2d27ff0c15c48ba746dacbe9a7aee", - "reference": "52326a9a43e2d27ff0c15c48ba746dacbe9a7aee", - "shasum": "" - }, - "require": { - "ext-mongodb": "*", - "mongodb/mongodb": "1.10.0", - "php": ">=8.0" - }, - "require-dev": { - "fakerphp/faker": "^1.14", - "laravel/pint": "1.2.*", - "phpstan/phpstan": "1.8.*", - "phpunit/phpunit": "^9.4", - "swoole/ide-helper": "4.8.0" - }, - "type": "library", - "autoload": { - "psr-4": { - "Utopia\\Mongo\\": "src" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - }, - { - "name": "Wess", - "email": "wess@appwrite.io" - } - ], - "description": "A simple library to manage Mongo database", - "keywords": [ - "database", - "mongo", - "php", - "upf", - "utopia" - ], - "support": { - "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/0.3.1" - }, - "time": "2023-09-01T17:25:28+00:00" + "time": "2025-03-24T09:05:31+00:00" }, { "name": "utopia-php/orchestration", From 5479e6f8f1f51691e168564b2592d15df917a421 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 18 Mar 2025 12:29:32 +0100 Subject: [PATCH 216/275] chore(audits): return queue pre-fetch results --- src/Appwrite/Platform/Workers/Audits.php | 49 +++++++++++++----------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index ed5ff8010a..962bc622d1 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -12,12 +12,12 @@ use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Queue\Result\Commit; +use Utopia\Queue\Result\NoCommit; use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development - protected const BATCH_SIZE_PRODUCTION = 5_000; protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds private int $lastTriggeredTime = 0; @@ -27,9 +27,7 @@ class Audits extends Action protected function getBatchSize(): int { - return System::getEnv('_APP_ENV', 'development') === 'development' - ? self::BATCH_SIZE_DEVELOPMENT - : self::BATCH_SIZE_PRODUCTION; + return intval(System::getEnv('_APP_QUEUE_PREFETCH_COUNT', 1)); } public static function getName(): string @@ -57,13 +55,13 @@ class Audits extends Action * @param Message $message * @param callable $getProjectDB * @param Document $project - * @return void + * @return Commit|NoCommit * @throws Throwable * @throws \Utopia\Database\Exception * @throws Authorization * @throws Structure */ - public function action(Message $message, callable $getProjectDB, Document $project): void + public function action(Message $message, callable $getProjectDB, Document $project): Commit|NoCommit { $payload = $message->getPayload() ?? []; @@ -119,29 +117,34 @@ class Audits extends Action // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); - $shouldProcessBatch = \count($this->logs) >= $batchSize; - if (!$shouldProcessBatch && \count($this->logs) > 0) { + $logCount = array_reduce($this->logs, fn (int $current, $logs) => $current + count($logs), 0); + $shouldProcessBatch = $logCount >= $batchSize; + if (!$shouldProcessBatch && $logCount > 0) { $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } - if ($shouldProcessBatch) { - try { - foreach ($this->logs as $internalId => $projectLogs) { - $dbForProject = $getProjectDB($projectLogs['project']); + if (!$shouldProcessBatch) { + return new NoCommit(); + } - Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); - $audit = new Audit($dbForProject); + try { + foreach ($this->logs as $internalId => $projectLogs) { + $dbForProject = $getProjectDB($projectLogs['project']); - $audit->logBatch($projectLogs['logs']); - Console::success('Audit logs processed successfully'); + Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); + $audit = new Audit($dbForProject); - unset($this->logs[$internalId]); - } - } catch (Throwable $e) { - Console::error('Error processing audit logs: ' . $e->getMessage()); - } finally { - $this->lastTriggeredTime = time(); + $audit->logBatch($projectLogs['logs']); + Console::success('Audit logs processed successfully'); + + unset($this->logs[$internalId]); } + return new Commit(); + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + return new NoCommit(); + } finally { + $this->lastTriggeredTime = time(); } } } From 25b6d59e3071b6d223cb53c8e9453631d5f3ec43 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 12:54:19 +0200 Subject: [PATCH 217/275] Add internalId --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index b8cff64214..d1b143ec9b 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -60,11 +60,19 @@ class Base extends Queries 'type' => Database::VAR_STRING, 'array' => false, ]); + + $attributes[] = new Document([ + 'key' => '$internalId', + 'type' => Database::VAR_STRING, + 'array' => false, + ]); + $attributes[] = new Document([ 'key' => '$createdAt', 'type' => Database::VAR_DATETIME, 'array' => false, ]); + $attributes[] = new Document([ 'key' => '$updatedAt', 'type' => Database::VAR_DATETIME, From 42f72b4477ef951aa131429f4f513579b2a4cf46 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 13:33:36 +0200 Subject: [PATCH 218/275] Add internalId to orders --- .../Utopia/Database/Validator/Queries/Base.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index d1b143ec9b..85fbf43e66 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -61,12 +61,6 @@ class Base extends Queries 'array' => false, ]); - $attributes[] = new Document([ - 'key' => '$internalId', - 'type' => Database::VAR_STRING, - 'array' => false, - ]); - $attributes[] = new Document([ 'key' => '$createdAt', 'type' => Database::VAR_DATETIME, @@ -84,8 +78,15 @@ class Base extends Queries new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order($attributes), - ]; + new Order( + array_merge($attributes, [ + new Document([ + 'key' => '$internalId', + 'type' => Database::VAR_STRING, + 'array' => false, + ]) + ]) + )]; parent::__construct($validators); } From 0648de2046c3b7561d37877b3b06c8c811ddc06d Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 14:32:57 +0200 Subject: [PATCH 219/275] Revert Base --- .../Utopia/Database/Validator/Queries/Base.php | 13 ++----------- .../Databases/DatabasesCustomServerTest.php | 4 ++-- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index 85fbf43e66..b8cff64214 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -60,13 +60,11 @@ class Base extends Queries 'type' => Database::VAR_STRING, 'array' => false, ]); - $attributes[] = new Document([ 'key' => '$createdAt', 'type' => Database::VAR_DATETIME, 'array' => false, ]); - $attributes[] = new Document([ 'key' => '$updatedAt', 'type' => Database::VAR_DATETIME, @@ -78,15 +76,8 @@ class Base extends Queries new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order( - array_merge($attributes, [ - new Document([ - 'key' => '$internalId', - 'type' => Database::VAR_STRING, - 'array' => false, - ]) - ]) - )]; + new Order($attributes), + ]; parent::__construct($validators); } diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 57e0b93634..70f8bea4f2 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -120,7 +120,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'queries' => [ - Query::orderDesc()->toString(), + Query::orderDesc()->setAttribute('')->toString(), ], ]); @@ -453,7 +453,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'queries' => [ - Query::orderDesc()->toString(), + Query::orderDesc()->setAttribute('')->toString(), ], ]); From e11e7e6780cbf8a3dc2ac10c4302360b7d4963c5 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 17:33:02 +0200 Subject: [PATCH 220/275] Add internalId --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 8 +++++++- .../e2e/Services/Databases/DatabasesCustomServerTest.php | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index b8cff64214..a13314ab2f 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -71,12 +71,18 @@ class Base extends Queries 'array' => false, ]); + $internalId = new Document([ + 'key' => '$internalId', + 'type' => Database::VAR_STRING, + 'array' => false, + ]); + $validators = [ new Limit(), new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order($attributes), + new Order(array_merge($attributes, [$internalId])), ]; parent::__construct($validators); diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 70f8bea4f2..55cb17b9f8 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -123,7 +123,7 @@ class DatabasesCustomServerTest extends Scope Query::orderDesc()->setAttribute('')->toString(), ], ]); - + var_dump($databases); $this->assertEquals(2, $databases['body']['total']); $this->assertEquals($base[0]['$id'], $databases['body']['databases'][0]['$id']); $this->assertEquals($base[1]['$id'], $databases['body']['databases'][1]['$id']); From 5799ca1041b8f945c45285f557a35ef6b8633381 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 17:37:34 +0200 Subject: [PATCH 221/275] remove var_dump --- tests/e2e/Services/Databases/DatabasesCustomServerTest.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 55cb17b9f8..206567e789 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -120,10 +120,9 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'queries' => [ - Query::orderDesc()->setAttribute('')->toString(), + Query::orderDesc()->toString(), ], ]); - var_dump($databases); $this->assertEquals(2, $databases['body']['total']); $this->assertEquals($base[0]['$id'], $databases['body']['databases'][0]['$id']); $this->assertEquals($base[1]['$id'], $databases['body']['databases'][1]['$id']); @@ -453,7 +452,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'queries' => [ - Query::orderDesc()->setAttribute('')->toString(), + Query::orderDesc()->toString(), ], ]); From 2795505e8288074f8981edb90e5f3279bfaf9938 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 17:38:01 +0200 Subject: [PATCH 222/275] line --- tests/e2e/Services/Databases/DatabasesCustomServerTest.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 206567e789..57e0b93634 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -123,6 +123,7 @@ class DatabasesCustomServerTest extends Scope Query::orderDesc()->toString(), ], ]); + $this->assertEquals(2, $databases['body']['total']); $this->assertEquals($base[0]['$id'], $databases['body']['databases'][0]['$id']); $this->assertEquals($base[1]['$id'], $databases['body']['databases'][1]['$id']); From c0cb4b26a3ebf73822f0abea8c6aeae9dd4d8875 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 15:51:47 +1300 Subject: [PATCH 223/275] Apply suggestions from code review --- src/Appwrite/Platform/Workers/Deletes.php | 35 ++++------------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 8deb227c78..0de79cd67e 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -305,9 +305,6 @@ class Deletes extends Action */ private function deleteExpiredTargets(Document $project, callable $getProjectDB): void { - /** - * todo: No index found for `expired` attribute - */ $this->deleteByGroup( 'targets', [ @@ -323,9 +320,6 @@ class Deletes extends Action private function deleteSessionTargets(Document $project, callable $getProjectDB, Document $session): void { - /** - * todo: No index found for `sessionInternalId` attribute - */ $this->deleteByGroup( 'targets', [ @@ -368,10 +362,6 @@ class Deletes extends Action $queries[] = Query::select(['$internalId', '$id', '$updatedAt']); $queries[] = Query::orderAsc(); - /** - * todo: No index on `resource`, `resourceType`, But it is fine since resource index is a strong index! - */ - $this->deleteByGroup( 'cache', $queries, @@ -446,7 +436,7 @@ class Deletes extends Action Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC + Query::orderDesc(), ], $dbForProject); if ($project->getId() !== 'console') { @@ -459,7 +449,7 @@ class Deletes extends Action Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC + Query::orderDesc(), ], $dbForLogs); } } @@ -639,11 +629,7 @@ class Deletes extends Action Query::orderAsc() ], $dbForPlatform); - /** - * No projectInternalId in this collection - * todo: No index for `projectId` attribute - */ - // Delete Schedules () + // Delete Schedules $this->deleteByGroup('schedules', [ Query::equal('projectId', [$projectId]), Query::orderAsc() @@ -724,9 +710,6 @@ class Deletes extends Action ], $dbForProject); // Delete identities - /** - * todo: Remove Duplication index `_key_userInternalId` , But lets leave because of index length issue in unique index - */ $this->deleteByGroup('identities', [ Query::equal('userInternalId', [$userInternalId]), Query::orderAsc() @@ -799,7 +782,7 @@ class Deletes extends Action $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime), Query::orderDesc('timestamp'), - Query::orderAsc(), // KEY "_key_timestamp" ("timestamp" DESC), + Query::orderAsc(), ], $dbForPlatform); } @@ -820,7 +803,7 @@ class Deletes extends Action Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), - Query::orderAsc(), // KEY "index-time" ("time" DESC) + Query::orderAsc(), ], $dbForProject); } catch (DatabaseException $e) { Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); @@ -845,7 +828,6 @@ class Deletes extends Action /** * Delete rules - * todo: No index for this query, drop _key_projectInternalId and create _key_projectInternalId, resourceInternalId, resourceType */ Console::info("Deleting rules for function " . $functionId); $this->deleteByGroup('rules', [ @@ -859,7 +841,6 @@ class Deletes extends Action /** * Delete Variables - * todo: No index for this query , drop _key_resourceInternalId and create new one with {resourceInternalId, resourceType} */ Console::info("Deleting variables for function " . $functionId); $this->deleteByGroup('variables', [ @@ -885,7 +866,6 @@ class Deletes extends Action /** * Delete builds - * todo: No index for `deploymentInternalId`, perhaps use deploymentId until fixed */ Console::info("Deleting builds for function " . $functionId); @@ -900,7 +880,6 @@ class Deletes extends Action /** * Delete Executions - * todo: No index for `functionInternalId` , perhaps use functionId until fixed */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ @@ -923,9 +902,6 @@ class Deletes extends Action $providerRepositoryId = $document->getAttribute('providerRepositoryId', ''); $projectInternalId = $document->getAttribute('projectInternalId', ''); - /** - * todo: add index to this query - */ $this->deleteByGroup('vcsComments', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::equal('projectInternalId', [$projectInternalId]), @@ -1025,7 +1001,6 @@ class Deletes extends Action /** * Delete builds - * todo: no index for `deploymentInternalId` Same as above index , no need to handle again... */ Console::info("Deleting builds for deployment " . $deploymentId); From a20140eb36dfc7238324cf40989bab9842e89abf Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 15:52:54 +1300 Subject: [PATCH 224/275] Update src/Appwrite/Platform/Workers/Deletes.php --- src/Appwrite/Platform/Workers/Deletes.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 0de79cd67e..93dbaf501c 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -851,7 +851,6 @@ class Deletes extends Action /** * Delete Deployments - * todo: No index for `resourceInternalId`, perhaps use resourceId until fixed */ Console::info("Deleting deployments for function " . $functionId); From c9c9db564053523be084bb5a97832221eb6ee28c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 15:52:59 +1300 Subject: [PATCH 225/275] Update src/Appwrite/Platform/Workers/Deletes.php --- src/Appwrite/Platform/Workers/Deletes.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 93dbaf501c..6dff127784 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -889,7 +889,6 @@ class Deletes extends Action /** * Delete VCS Repositories and VCS Comments - * todo: no index for this query */ Console::info("Deleting VCS repositories and comments linked to function " . $functionId); $this->deleteByGroup('repositories', [ From 6bcdc8c769185b77dc9689ed691e53a4071b1453 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 16:01:27 +1300 Subject: [PATCH 226/275] Update src/Appwrite/Utopia/Database/Validator/Queries/Base.php --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index a13314ab2f..664476af36 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -82,7 +82,7 @@ class Base extends Queries new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order(array_merge($attributes, [$internalId])), + new Order([...$attributes, $internalId])), ]; parent::__construct($validators); From 74d9a3c181ebf09b9a7b6da710b6431dcaac4097 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 16:19:18 +1300 Subject: [PATCH 227/275] Fix syntax --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index 664476af36..e8eafba5a0 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -82,7 +82,7 @@ class Base extends Queries new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order([...$attributes, $internalId])), + new Order([...$attributes, $internalId]), ]; parent::__construct($validators); From 6195674611b22d8d65c83a038f2d28cdc662ba36 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 21:20:26 +1300 Subject: [PATCH 228/275] Update database --- composer.json | 2 +- composer.lock | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/composer.json b/composer.json index b1b6aed539..5b0f26b883 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.62.*", + "utopia-php/database": "dev-feat-multi-tenant-insert as 0.62.1", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 1ea887fdbc..09b8250af6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d2bec8137dcd84994121f89a29932d31", + "content-hash": "640d2a8be42ae521ea12d8a908873f2b", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.62.1", + "version": "dev-feat-multi-tenant-insert", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "65dc51466c12552add10395900cdbb4728da4068" + "reference": "2e01c999153a202a8bae9c21e4302e7a674442d0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/65dc51466c12552add10395900cdbb4728da4068", - "reference": "65dc51466c12552add10395900cdbb4728da4068", + "url": "https://api.github.com/repos/utopia-php/database/zipball/2e01c999153a202a8bae9c21e4302e7a674442d0", + "reference": "2e01c999153a202a8bae9c21e4302e7a674442d0", "shasum": "" }, "require": { @@ -3546,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.62.1" + "source": "https://github.com/utopia-php/database/tree/feat-multi-tenant-insert" }, - "time": "2025-03-24T08:27:18+00:00" + "time": "2025-03-25T04:45:25+00:00" }, { "name": "utopia-php/domains", @@ -8132,9 +8132,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-feat-multi-tenant-insert", + "alias": "0.62.1", + "alias_normalized": "0.62.1.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From d89cdbcf5a52717b7dc7e48b88020c2604502f0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 25 Mar 2025 21:34:09 +0100 Subject: [PATCH 229/275] Fix schedules region filtering --- src/Appwrite/Migration/Version/V19.php | 3 +-- src/Appwrite/Platform/Tasks/ScheduleBase.php | 9 ++++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 18234ebdc4..7adf5b85be 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -10,7 +10,6 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception; use Utopia\Database\Query; -use Utopia\System\System; class V19 extends Migration { @@ -731,7 +730,7 @@ class V19 extends Migration if (empty($document->getAttribute('scheduleId', null))) { $schedule = $this->consoleDB->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), // Todo replace with projects region + 'region' => $project->getAttribute('region'), // Todo replace with projects region 'resourceType' => 'function', 'resourceId' => $document->getId(), 'resourceInternalId' => $document->getInternalId(), diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index dad2db0d9a..324b88d07b 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -103,8 +103,15 @@ abstract class ScheduleBase extends Action $paginationQueries[] = Query::cursorAfter($latestDocument); } + // Temporarly accepting both 'fra' and 'default' + // When all migrated, only use _APP_REGION with 'default' as default value + $regions = [System::getEnv('_APP_REGION', 'default')]; + if (!in_array('default', $regions)) { + $regions[] = 'default'; + } + $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', $regions), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); From f25143b240177b141c487048d5e9de03a85e7029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 25 Mar 2025 21:52:02 +0100 Subject: [PATCH 230/275] Support fra-region on cloud; temporary fix --- src/Appwrite/Migration/Version/V19.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 9 ++++++++- src/Appwrite/Platform/Workers/Deletes.php | 9 ++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 7adf5b85be..4415003bfd 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -730,7 +730,7 @@ class V19 extends Migration if (empty($document->getAttribute('scheduleId', null))) { $schedule = $this->consoleDB->createDocument('schedules', new Document([ - 'region' => $project->getAttribute('region'), // Todo replace with projects region + 'region' => $project->getAttribute('region'), 'resourceType' => 'function', 'resourceId' => $document->getId(), 'resourceInternalId' => $document->getInternalId(), diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index 324b88d07b..a3c36cb96e 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -160,8 +160,15 @@ abstract class ScheduleBase extends Action $paginationQueries[] = Query::cursorAfter($latestDocument); } + // Temporarly accepting both 'fra' and 'default' + // When all migrated, only use _APP_REGION with 'default' as default value + $regions = [System::getEnv('_APP_REGION', 'default')]; + if (!in_array('default', $regions)) { + $regions[] = 'default'; + } + $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', $regions), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 6dff127784..8eaa08ce65 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -180,10 +180,17 @@ class Deletes extends Action */ private function deleteSchedules(Database $dbForPlatform, callable $getProjectDB, string $datetime): void { + // Temporarly accepting both 'fra' and 'default' + // When all migrated, only use _APP_REGION with 'default' as default value + $regions = [System::getEnv('_APP_REGION', 'default')]; + if (!in_array('default', $regions)) { + $regions[] = 'default'; + } + $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', $regions), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], From a461d5bf7281841ccbbe1c4a27d31e850b8dc2ad Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 26 Mar 2025 09:20:44 +0200 Subject: [PATCH 231/275] Deletes fix selects --- src/Appwrite/Platform/Workers/Deletes.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 8eaa08ce65..2cc1eaf9af 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -31,6 +31,8 @@ use Utopia\System\System; class Deletes extends Action { + protected array $selects = ['$internalId', '$id', '$collection', '$updatedAt']; + public static function getName(): string { return 'deletes'; @@ -366,7 +368,7 @@ class Deletes extends Action $queries[] = Query::equal('resourceType', [$resourceType]); } - $queries[] = Query::select(['$internalId', '$id', '$updatedAt']); + $queries[] = Query::select($this->selects); $queries[] = Query::orderAsc(); $this->deleteByGroup( @@ -403,7 +405,7 @@ class Deletes extends Action ); $queries = [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), Query::orderDesc(), @@ -439,7 +441,7 @@ class Deletes extends Action // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -452,7 +454,7 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -749,7 +751,7 @@ class Deletes extends Action // Delete Executions $this->deleteByGroup('executions', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -770,7 +772,7 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -807,7 +809,7 @@ class Deletes extends Action try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderAsc(), @@ -889,7 +891,7 @@ class Deletes extends Action */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::equal('functionInternalId', [$functionInternalId]), Query::orderAsc() ], $dbForProject); From 424a3575034c31b34b6c2c80a24fb953338642bd Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 26 Mar 2025 10:02:30 +0200 Subject: [PATCH 232/275] Add $permissions --- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 2cc1eaf9af..5033822241 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -31,7 +31,7 @@ use Utopia\System\System; class Deletes extends Action { - protected array $selects = ['$internalId', '$id', '$collection', '$updatedAt']; + protected array $selects = ['$internalId', '$id', '$collection', '$permissions', '$updatedAt']; public static function getName(): string { From d9d38197ddcd91c6429843ba5878b9a9a4e3bef7 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 26 Mar 2025 23:47:59 +1300 Subject: [PATCH 233/275] Update libs --- composer.json | 2 +- composer.lock | 65 ++++++++++++++++++++++----------------------------- 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/composer.json b/composer.json index 5b0f26b883..b8c5afa109 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-multi-tenant-insert as 0.62.1", + "utopia-php/database": "0.63.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 09b8250af6..d4956aeba0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "640d2a8be42ae521ea12d8a908873f2b", + "content-hash": "5b3c46863e4571c838c30090ad96436c", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-multi-tenant-insert", + "version": "0.63.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "2e01c999153a202a8bae9c21e4302e7a674442d0" + "reference": "51ff0a6514e6eda44a6da016d838e7e55282e01a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/2e01c999153a202a8bae9c21e4302e7a674442d0", - "reference": "2e01c999153a202a8bae9c21e4302e7a674442d0", + "url": "https://api.github.com/repos/utopia-php/database/zipball/51ff0a6514e6eda44a6da016d838e7e55282e01a", + "reference": "51ff0a6514e6eda44a6da016d838e7e55282e01a", "shasum": "" }, "require": { @@ -3546,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-multi-tenant-insert" + "source": "https://github.com/utopia-php/database/tree/0.63.0" }, - "time": "2025-03-25T04:45:25+00:00" + "time": "2025-03-26T10:35:37+00:00" }, { "name": "utopia-php/domains", @@ -3950,16 +3950,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.2", + "version": "0.8.3", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1" + "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1", - "reference": "aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/85c2e14647b240b75be6b6b762e5b30e48fb8d8a", + "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a", "shasum": "" }, "require": { @@ -3967,7 +3967,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.62.*", + "utopia-php/database": "0.63.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4000,9 +4000,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.2" + "source": "https://github.com/utopia-php/migration/tree/0.8.3" }, - "time": "2025-03-24T09:05:31+00:00" + "time": "2025-03-26T10:45:51+00:00" }, { "name": "utopia-php/orchestration", @@ -4385,16 +4385,16 @@ }, { "name": "utopia-php/swoole", - "version": "0.8.2", + "version": "0.8.3", "source": { "type": "git", "url": "https://github.com/utopia-php/swoole.git", - "reference": "5fa9d42c608ad46a4ce42a6d2b2eae00592fccd4" + "reference": "1af73dd3e73987cf729c7db399054e4a70befd99" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/swoole/zipball/5fa9d42c608ad46a4ce42a6d2b2eae00592fccd4", - "reference": "5fa9d42c608ad46a4ce42a6d2b2eae00592fccd4", + "url": "https://api.github.com/repos/utopia-php/swoole/zipball/1af73dd3e73987cf729c7db399054e4a70befd99", + "reference": "1af73dd3e73987cf729c7db399054e4a70befd99", "shasum": "" }, "require": { @@ -4430,9 +4430,9 @@ ], "support": { "issues": "https://github.com/utopia-php/swoole/issues", - "source": "https://github.com/utopia-php/swoole/tree/0.8.2" + "source": "https://github.com/utopia-php/swoole/tree/0.8.3" }, - "time": "2024-02-01T14:54:12+00:00" + "time": "2025-03-26T10:09:05+00:00" }, { "name": "utopia-php/system", @@ -4775,16 +4775,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.9", + "version": "0.40.10", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "dbb45a5db22cdc3368fe2573c07ba6088f188fa4" + "reference": "054ac96285caf4f77879087b2416a5ddb8263051" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/dbb45a5db22cdc3368fe2573c07ba6088f188fa4", - "reference": "dbb45a5db22cdc3368fe2573c07ba6088f188fa4", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/054ac96285caf4f77879087b2416a5ddb8263051", + "reference": "054ac96285caf4f77879087b2416a5ddb8263051", "shasum": "" }, "require": { @@ -4820,9 +4820,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.9" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.10" }, - "time": "2025-03-17T18:39:14+00:00" + "time": "2025-03-25T13:44:16+00:00" }, { "name": "doctrine/annotations", @@ -8132,18 +8132,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-feat-multi-tenant-insert", - "alias": "0.62.1", - "alias_normalized": "0.62.1.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From c79e8e5b954ce3e61c39bcd01413d05f7905b2dd Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 27 Mar 2025 18:04:17 +1300 Subject: [PATCH 234/275] Update database --- composer.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/composer.lock b/composer.lock index d4956aeba0..eb29c765a0 100644 --- a/composer.lock +++ b/composer.lock @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.30.1", + "version": "v4.30.2", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24" + "reference": "a4c4d8565b40b9f76debc9dfeb221412eacb8ced" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/f29ba8a30dfd940efb3a8a75dc44446539101f24", - "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/a4c4d8565b40b9f76debc9dfeb221412eacb8ced", + "reference": "a4c4d8565b40b9f76debc9dfeb221412eacb8ced", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.1" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.2" }, - "time": "2025-03-13T21:08:17+00:00" + "time": "2025-03-26T18:01:50+00:00" }, { "name": "league/csv", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.63.0", + "version": "0.63.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "51ff0a6514e6eda44a6da016d838e7e55282e01a" + "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/51ff0a6514e6eda44a6da016d838e7e55282e01a", - "reference": "51ff0a6514e6eda44a6da016d838e7e55282e01a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/ad191bf34151815f716f553796a363ff2b6ef7d3", + "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3", "shasum": "" }, "require": { @@ -3546,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.63.0" + "source": "https://github.com/utopia-php/database/tree/0.63.1" }, - "time": "2025-03-26T10:35:37+00:00" + "time": "2025-03-27T04:58:07+00:00" }, { "name": "utopia-php/domains", From 17dc8e3f8ad82afc911e492b0c4be9218f0305a3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 27 Mar 2025 21:37:10 +1300 Subject: [PATCH 235/275] Revert "chore(audits): return queue pre-fetch results" --- src/Appwrite/Platform/Workers/Audits.php | 49 +++++++++++------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index 962bc622d1..ed5ff8010a 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -12,12 +12,12 @@ use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Platform\Action; use Utopia\Queue\Message; -use Utopia\Queue\Result\Commit; -use Utopia\Queue\Result\NoCommit; use Utopia\System\System; class Audits extends Action { + protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development + protected const BATCH_SIZE_PRODUCTION = 5_000; protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds private int $lastTriggeredTime = 0; @@ -27,7 +27,9 @@ class Audits extends Action protected function getBatchSize(): int { - return intval(System::getEnv('_APP_QUEUE_PREFETCH_COUNT', 1)); + return System::getEnv('_APP_ENV', 'development') === 'development' + ? self::BATCH_SIZE_DEVELOPMENT + : self::BATCH_SIZE_PRODUCTION; } public static function getName(): string @@ -55,13 +57,13 @@ class Audits extends Action * @param Message $message * @param callable $getProjectDB * @param Document $project - * @return Commit|NoCommit + * @return void * @throws Throwable * @throws \Utopia\Database\Exception * @throws Authorization * @throws Structure */ - public function action(Message $message, callable $getProjectDB, Document $project): Commit|NoCommit + public function action(Message $message, callable $getProjectDB, Document $project): void { $payload = $message->getPayload() ?? []; @@ -117,34 +119,29 @@ class Audits extends Action // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); - $logCount = array_reduce($this->logs, fn (int $current, $logs) => $current + count($logs), 0); - $shouldProcessBatch = $logCount >= $batchSize; - if (!$shouldProcessBatch && $logCount > 0) { + $shouldProcessBatch = \count($this->logs) >= $batchSize; + if (!$shouldProcessBatch && \count($this->logs) > 0) { $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } - if (!$shouldProcessBatch) { - return new NoCommit(); - } + if ($shouldProcessBatch) { + try { + foreach ($this->logs as $internalId => $projectLogs) { + $dbForProject = $getProjectDB($projectLogs['project']); - try { - foreach ($this->logs as $internalId => $projectLogs) { - $dbForProject = $getProjectDB($projectLogs['project']); + Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); + $audit = new Audit($dbForProject); - Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); - $audit = new Audit($dbForProject); + $audit->logBatch($projectLogs['logs']); + Console::success('Audit logs processed successfully'); - $audit->logBatch($projectLogs['logs']); - Console::success('Audit logs processed successfully'); - - unset($this->logs[$internalId]); + unset($this->logs[$internalId]); + } + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + } finally { + $this->lastTriggeredTime = time(); } - return new Commit(); - } catch (Throwable $e) { - Console::error('Error processing audit logs: ' . $e->getMessage()); - return new NoCommit(); - } finally { - $this->lastTriggeredTime = time(); } } } From 85edfc6af8049fb0c88eebd70570cf3bdcfe827d Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 28 Mar 2025 05:47:18 +0000 Subject: [PATCH 236/275] chore: added test for users.listmemberships --- tests/e2e/Services/Users/UsersBase.php | 91 ++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index 04e0eb5bc3..00e999672f 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -801,6 +801,97 @@ trait UsersBase return $data; } + /** + * @depends testGetUser + */ + public function testListUserMemberships(array $data): array + { + /** + * Test for SUCCESS + */ + + // create a new team + $team = $this->client->call(Client::METHOD_POST, '/teams', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'teamId' => 'unique()', + 'name' => 'Test Team', + ]); + + // create a new membership + $membership = $this->client->call(Client::METHOD_POST, '/teams/' . $team['body']['$id'] . '/memberships', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'userId' => $data['userId'], + 'roles' => ['new-role'], + ]); + + // list the memberships + $response = $this->client->call(Client::METHOD_GET, '/users/' . $data['userId'] . '/memberships', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['memberships'][0]['$id'], $membership['body']['$id']); + $this->assertEquals($response['body']['memberships'][0]['roles'], ['new-role']); + $this->assertEquals($response['body']['total'], 1); + + // create another membership with a new role + $team = $this->client->call(Client::METHOD_POST, '/teams', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'teamId' => 'unique()', + 'name' => 'Test Team 2', + ]); + + $membership = $this->client->call(Client::METHOD_POST, '/teams/' . $team['body']['$id'] . '/memberships', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'userId' => $data['userId'], + 'roles' => ['new-role-2'], + ]); + + // list out memberships and query by role + $response = $this->client->call(Client::METHOD_GET, '/users/' . $data['userId'] . '/memberships', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + Query::contains('roles', ['new-role-2'])->toString() + ] + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['memberships'][0]['$id'], $membership['body']['$id']); + $this->assertEquals($response['body']['memberships'][0]['roles'], ['new-role-2']); + $this->assertEquals($response['body']['total'], 1); + + /** + * Test for FAILURE + */ + + // query using equal on array field + $response = $this->client->call(Client::METHOD_GET, '/users/' . $data['userId'] . '/memberships', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + Query::equal('roles', ['new-role-2'])->toString() + ] + ]); + + $this->assertEquals($response['body']['code'], 400); + $this->assertEquals($response['body']['message'], 'Invalid `queries` param: Invalid query: Cannot query equal on attribute "roles" because it is an array.'); + $this->assertEquals($response['body']['type'], 'general_argument_invalid'); + + return $data; + } + /** * @depends testGetUser */ From cba700402bb79598a526f0e6d026060cf0cd3e50 Mon Sep 17 00:00:00 2001 From: fogelito Date: Fri, 28 Mar 2025 09:04:50 +0300 Subject: [PATCH 237/275] Add order by for cursor --- src/Appwrite/Platform/Workers/Deletes.php | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 5033822241..5674448c18 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -405,7 +405,7 @@ class Deletes extends Action ); $queries = [ - Query::select($this->selects), + Query::select(array_merge($this->selects, ['accessedAt'])), Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), Query::orderDesc(), @@ -439,9 +439,11 @@ class Deletes extends Action /** @var Database $dbForProject*/ $dbForProject = $getProjectDB($project); + $select = array_merge($this->selects, ['time']); + // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ - Query::select($this->selects), + Query::select($select), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -454,7 +456,7 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ - Query::select($this->selects), + Query::select($select), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -751,7 +753,7 @@ class Deletes extends Action // Delete Executions $this->deleteByGroup('executions', [ - Query::select($this->selects), + Query::select(array_merge($this->selects, ['$createdAt'])), Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -772,7 +774,7 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ - Query::select($this->selects), + Query::select(array_merge($this->selects, ['$createdAt'])), Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -809,7 +811,7 @@ class Deletes extends Action try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::select($this->selects), + Query::select(array_merge($this->selects, ['time'])), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderAsc(), From a4c18c4b774eefa25af547b31929986d953a2552 Mon Sep 17 00:00:00 2001 From: fogelito Date: Fri, 28 Mar 2025 09:16:25 +0300 Subject: [PATCH 238/275] Remove array merge --- src/Appwrite/Platform/Workers/Deletes.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 5674448c18..a9b83976a4 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -405,7 +405,7 @@ class Deletes extends Action ); $queries = [ - Query::select(array_merge($this->selects, ['accessedAt'])), + Query::select([...$this->selects, 'accessedAt']), Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), Query::orderDesc(), @@ -439,11 +439,11 @@ class Deletes extends Action /** @var Database $dbForProject*/ $dbForProject = $getProjectDB($project); - $select = array_merge($this->selects, ['time']); + $selects = [...$this->selects, 'time']; // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ - Query::select($select), + Query::select($selects), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -456,7 +456,7 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ - Query::select($select), + Query::select($selects), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -753,7 +753,7 @@ class Deletes extends Action // Delete Executions $this->deleteByGroup('executions', [ - Query::select(array_merge($this->selects, ['$createdAt'])), + Query::select([...$this->selects, '$createdAt']), Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -774,7 +774,7 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ - Query::select(array_merge($this->selects, ['$createdAt'])), + Query::select([...$this->selects, '$createdAt']), Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -811,7 +811,7 @@ class Deletes extends Action try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::select(array_merge($this->selects, ['time'])), + Query::select([...$this->selects, 'time']), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderAsc(), From 22939831f2d759f26bc5912b2b3e7a9cdceadc62 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 28 Mar 2025 20:58:25 +0100 Subject: [PATCH 239/275] Updated fetch version --- composer.json | 2 +- composer.lock | 78 +++++++++++++++++++++++++-------------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/composer.json b/composer.json index b8c5afa109..176b5889fc 100644 --- a/composer.json +++ b/composer.json @@ -55,7 +55,7 @@ "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", - "utopia-php/fetch": "0.3.*", + "utopia-php/fetch": "0.4.*", "utopia-php/image": "0.8.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", diff --git a/composer.lock b/composer.lock index eb29c765a0..3772b25707 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5b3c46863e4571c838c30090ad96436c", + "content-hash": "c99b4733669c17013e211c7dc54a86f6", "packages": [ { "name": "adhocore/jwt", @@ -3659,16 +3659,16 @@ }, { "name": "utopia-php/fetch", - "version": "0.3.1", + "version": "0.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/fetch.git", - "reference": "524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0" + "reference": "46e791ff6a95864517750b9df6bbf4a17e3c9c4e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/fetch/zipball/524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0", - "reference": "524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/46e791ff6a95864517750b9df6bbf4a17e3c9c4e", + "reference": "46e791ff6a95864517750b9df6bbf4a17e3c9c4e", "shasum": "" }, "require": { @@ -3692,9 +3692,9 @@ "description": "A simple library that provides an interface for making HTTP Requests.", "support": { "issues": "https://github.com/utopia-php/fetch/issues", - "source": "https://github.com/utopia-php/fetch/tree/0.3.1" + "source": "https://github.com/utopia-php/fetch/tree/0.4.0" }, - "time": "2025-03-05T18:08:55+00:00" + "time": "2025-03-11T21:06:56+00:00" }, { "name": "utopia-php/framework", @@ -3950,16 +3950,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.3", + "version": "0.8.4", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a" + "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/85c2e14647b240b75be6b6b762e5b30e48fb8d8a", - "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/845fd04ccf5e0edb03c184b864e0596080a432b8", + "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8", "shasum": "" }, "require": { @@ -3967,7 +3967,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.63.*", + "utopia-php/database": "0.*.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4000,9 +4000,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.3" + "source": "https://github.com/utopia-php/migration/tree/0.8.4" }, - "time": "2025-03-26T10:45:51+00:00" + "time": "2025-03-28T02:08:22+00:00" }, { "name": "utopia-php/orchestration", @@ -4211,23 +4211,23 @@ }, { "name": "utopia-php/queue", - "version": "0.9.0", + "version": "0.9.1", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "077075f1d57afa430f76c35ed3bf4616e0eee8e7" + "reference": "32b6f84c55aae761db5a5ae76cc91ca8dbc8bc32" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/077075f1d57afa430f76c35ed3bf4616e0eee8e7", - "reference": "077075f1d57afa430f76c35ed3bf4616e0eee8e7", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/32b6f84c55aae761db5a5ae76cc91ca8dbc8bc32", + "reference": "32b6f84c55aae761db5a5ae76cc91ca8dbc8bc32", "shasum": "" }, "require": { "php": ">=8.3", "php-amqplib/php-amqplib": "^3.7", "utopia-php/cli": "0.15.*", - "utopia-php/fetch": "^0.3.0", + "utopia-php/fetch": "0.4.*", "utopia-php/framework": "0.33.*", "utopia-php/telemetry": "0.1.*" }, @@ -4270,9 +4270,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.9.0" + "source": "https://github.com/utopia-php/queue/tree/0.9.1" }, - "time": "2025-03-13T12:22:41+00:00" + "time": "2025-03-28T19:49:36+00:00" }, { "name": "utopia-php/registry", @@ -4775,16 +4775,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.10", + "version": "0.40.11", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "054ac96285caf4f77879087b2416a5ddb8263051" + "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/054ac96285caf4f77879087b2416a5ddb8263051", - "reference": "054ac96285caf4f77879087b2416a5ddb8263051", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", + "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", "shasum": "" }, "require": { @@ -4820,9 +4820,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.10" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.11" }, - "time": "2025-03-25T13:44:16+00:00" + "time": "2025-03-26T10:53:16+00:00" }, { "name": "doctrine/annotations", @@ -7154,16 +7154,16 @@ }, { "name": "symfony/console", - "version": "v7.2.1", + "version": "v7.2.5", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3" + "reference": "e51498ea18570c062e7df29d05a7003585b19b88" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/fefcc18c0f5d0efe3ab3152f15857298868dc2c3", - "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3", + "url": "https://api.github.com/repos/symfony/console/zipball/e51498ea18570c062e7df29d05a7003585b19b88", + "reference": "e51498ea18570c062e7df29d05a7003585b19b88", "shasum": "" }, "require": { @@ -7227,7 +7227,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.2.1" + "source": "https://github.com/symfony/console/tree/v7.2.5" }, "funding": [ { @@ -7243,7 +7243,7 @@ "type": "tidelift" } ], - "time": "2024-12-11T03:49:26+00:00" + "time": "2025-03-12T08:11:12+00:00" }, { "name": "symfony/filesystem", @@ -7758,16 +7758,16 @@ }, { "name": "symfony/process", - "version": "v7.2.4", + "version": "v7.2.5", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf" + "reference": "87b7c93e57df9d8e39a093d32587702380ff045d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", - "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", + "url": "https://api.github.com/repos/symfony/process/zipball/87b7c93e57df9d8e39a093d32587702380ff045d", + "reference": "87b7c93e57df9d8e39a093d32587702380ff045d", "shasum": "" }, "require": { @@ -7799,7 +7799,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.2.4" + "source": "https://github.com/symfony/process/tree/v7.2.5" }, "funding": [ { @@ -7815,7 +7815,7 @@ "type": "tidelift" } ], - "time": "2025-02-05T08:33:46+00:00" + "time": "2025-03-13T12:21:46+00:00" }, { "name": "symfony/string", From 612b064d2a9121949030fe7459ad43732a36e15c Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 31 Mar 2025 15:16:59 +0300 Subject: [PATCH 240/275] 0.63.2 --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 176b5889fc..2e25ca3748 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.63.*", + "utopia-php/database": "dev-0.63.2-fix as 0.63.2", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", From 4698323f38cfd0f49040eac13da1c63b8cca6003 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 31 Mar 2025 16:19:17 +0300 Subject: [PATCH 241/275] lock --- composer.lock | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/composer.lock b/composer.lock index 3772b25707..a25995668e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c99b4733669c17013e211c7dc54a86f6", + "content-hash": "796b9a9952aac4604cf2acfd0ce455c6", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.63.1", + "version": "dev-0.63.2-fix", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3" + "reference": "f32c48a8b94aa2e4b543947046daf59e7cbd49d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/ad191bf34151815f716f553796a363ff2b6ef7d3", - "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3", + "url": "https://api.github.com/repos/utopia-php/database/zipball/f32c48a8b94aa2e4b543947046daf59e7cbd49d4", + "reference": "f32c48a8b94aa2e4b543947046daf59e7cbd49d4", "shasum": "" }, "require": { @@ -3546,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.63.1" + "source": "https://github.com/utopia-php/database/tree/0.63.2-fix" }, - "time": "2025-03-27T04:58:07+00:00" + "time": "2025-03-31T11:53:46+00:00" }, { "name": "utopia-php/domains", @@ -8132,9 +8132,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-0.63.2-fix", + "alias": "0.63.2", + "alias_normalized": "0.63.2.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From d92413fd6ba46eb79f548aa957e4258a8d14f7f7 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 1 Apr 2025 08:13:56 +0300 Subject: [PATCH 242/275] Revert database lib --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 2e25ca3748..176b5889fc 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-0.63.2-fix as 0.63.2", + "utopia-php/database": "0.63.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", From b46de29735db1de51981aff820440eddad4bbbfe Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 1 Apr 2025 08:15:03 +0300 Subject: [PATCH 243/275] database 0.63.2 --- composer.lock | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/composer.lock b/composer.lock index a25995668e..06aea5ed69 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "796b9a9952aac4604cf2acfd0ce455c6", + "content-hash": "c99b4733669c17013e211c7dc54a86f6", "packages": [ { "name": "adhocore/jwt", @@ -3497,7 +3497,7 @@ }, { "name": "utopia-php/database", - "version": "dev-0.63.2-fix", + "version": "0.63.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", @@ -3546,7 +3546,7 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.63.2-fix" + "source": "https://github.com/utopia-php/database/tree/0.63.2" }, "time": "2025-03-31T11:53:46+00:00" }, @@ -8132,18 +8132,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-0.63.2-fix", - "alias": "0.63.2", - "alias_normalized": "0.63.2.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 938505da5121dc08c84db9ee39b36d332133fe62 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 1 Apr 2025 21:23:23 +0200 Subject: [PATCH 244/275] Updated websocket version --- composer.json | 2 +- composer.lock | 33 ++++++++++++--------------------- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/composer.json b/composer.json index 176b5889fc..39ac2f6574 100644 --- a/composer.json +++ b/composer.json @@ -72,7 +72,7 @@ "utopia-php/system": "0.9.*", "utopia-php/telemetry": "0.1.*", "utopia-php/vcs": "0.9.*", - "utopia-php/websocket": "0.1.*", + "utopia-php/websocket": "0.3.*", "matomo/device-detector": "6.1.*", "dragonmantank/cron-expression": "3.3.2", "phpmailer/phpmailer": "6.9.1", diff --git a/composer.lock b/composer.lock index 06aea5ed69..8ee7cd75f0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c99b4733669c17013e211c7dc54a86f6", + "content-hash": "d393dfb19f208bab8b2f165071915fc9", "packages": [ { "name": "adhocore/jwt", @@ -4592,27 +4592,28 @@ }, { "name": "utopia-php/websocket", - "version": "0.1.0", + "version": "0.3.0", "source": { "type": "git", "url": "https://github.com/utopia-php/websocket.git", - "reference": "51fcb86171400d8aa40d76c54593481fd273dab5" + "reference": "629e53640b108eab43c7cc9ab375efade8622d43" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/websocket/zipball/51fcb86171400d8aa40d76c54593481fd273dab5", - "reference": "51fcb86171400d8aa40d76c54593481fd273dab5", + "url": "https://api.github.com/repos/utopia-php/websocket/zipball/629e53640b108eab43c7cc9ab375efade8622d43", + "reference": "629e53640b108eab43c7cc9ab375efade8622d43", "shasum": "" }, "require": { "php": ">=8.0" }, "require-dev": { + "laravel/pint": "^1.15", + "phpstan/phpstan": "^1.12", "phpunit/phpunit": "^9.5.5", - "swoole/ide-helper": "4.6.6", + "swoole/ide-helper": "5.1.2", "textalk/websocket": "1.5.2", - "vimeo/psalm": "^4.8.1", - "workerman/workerman": "^4.0" + "workerman/workerman": "4.1.*" }, "type": "library", "autoload": { @@ -4624,16 +4625,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - }, - { - "name": "Torsten Dittmann", - "email": "torsten@appwrite.io" - } - ], "description": "A simple abstraction for WebSocket servers.", "keywords": [ "framework", @@ -4644,9 +4635,9 @@ ], "support": { "issues": "https://github.com/utopia-php/websocket/issues", - "source": "https://github.com/utopia-php/websocket/tree/0.1.0" + "source": "https://github.com/utopia-php/websocket/tree/0.3.0" }, - "time": "2021-12-20T10:50:09+00:00" + "time": "2025-03-28T01:11:13+00:00" }, { "name": "webmozart/assert", @@ -8134,7 +8125,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From c152d8e001a4b44e9c4069e6fdfb99f96a9a7077 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 2 Apr 2025 13:02:35 +1300 Subject: [PATCH 245/275] Update database --- composer.json | 4 +-- composer.lock | 92 ++++++++++++++++++++++++++++----------------------- 2 files changed, 53 insertions(+), 43 deletions(-) diff --git a/composer.json b/composer.json index b8c5afa109..2d13ef244f 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.63.*", + "utopia-php/database": "dev-fix-tenant as 0.64.0", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", @@ -63,7 +63,7 @@ "utopia-php/migration": "0.8.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "0.7.*", + "utopia-php/pools": "0.8.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.9.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index eb29c765a0..43fb23d14f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5b3c46863e4571c838c30090ad96436c", + "content-hash": "0f7e98ef962fcce12b0ea435188504aa", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.63.1", + "version": "dev-fix-tenant", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3" + "reference": "1b7d8b774ae28d98de82a86e4fef262514e8fc40" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/ad191bf34151815f716f553796a363ff2b6ef7d3", - "reference": "ad191bf34151815f716f553796a363ff2b6ef7d3", + "url": "https://api.github.com/repos/utopia-php/database/zipball/1b7d8b774ae28d98de82a86e4fef262514e8fc40", + "reference": "1b7d8b774ae28d98de82a86e4fef262514e8fc40", "shasum": "" }, "require": { @@ -3514,7 +3514,8 @@ "ext-pdo": "*", "php": ">=8.1", "utopia-php/cache": "0.12.*", - "utopia-php/framework": "0.33.*" + "utopia-php/framework": "0.33.*", + "utopia-php/pools": "0.8.*" }, "require-dev": { "fakerphp/faker": "1.23.*", @@ -3546,9 +3547,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.63.1" + "source": "https://github.com/utopia-php/database/tree/fix-tenant" }, - "time": "2025-03-27T04:58:07+00:00" + "time": "2025-04-02T00:00:03+00:00" }, { "name": "utopia-php/domains", @@ -3950,16 +3951,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.3", + "version": "0.8.4", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a" + "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/85c2e14647b240b75be6b6b762e5b30e48fb8d8a", - "reference": "85c2e14647b240b75be6b6b762e5b30e48fb8d8a", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/845fd04ccf5e0edb03c184b864e0596080a432b8", + "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8", "shasum": "" }, "require": { @@ -3967,7 +3968,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.63.*", + "utopia-php/database": "0.*.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4000,9 +4001,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.3" + "source": "https://github.com/utopia-php/migration/tree/0.8.4" }, - "time": "2025-03-26T10:45:51+00:00" + "time": "2025-03-28T02:08:22+00:00" }, { "name": "utopia-php/orchestration", @@ -4106,16 +4107,16 @@ }, { "name": "utopia-php/pools", - "version": "0.7.0", + "version": "0.8.0", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "ad64d45afda08ec8b29e2642a8d18075964d40bf" + "reference": "60733929dc328e7ea47e800579c8bbf0d49df5ba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/ad64d45afda08ec8b29e2642a8d18075964d40bf", - "reference": "ad64d45afda08ec8b29e2642a8d18075964d40bf", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/60733929dc328e7ea47e800579c8bbf0d49df5ba", + "reference": "60733929dc328e7ea47e800579c8bbf0d49df5ba", "shasum": "" }, "require": { @@ -4152,9 +4153,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.7.0" + "source": "https://github.com/utopia-php/pools/tree/0.8.0" }, - "time": "2025-03-18T03:55:33+00:00" + "time": "2025-03-19T10:22:03+00:00" }, { "name": "utopia-php/preloader", @@ -4775,16 +4776,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.10", + "version": "0.40.11", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "054ac96285caf4f77879087b2416a5ddb8263051" + "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/054ac96285caf4f77879087b2416a5ddb8263051", - "reference": "054ac96285caf4f77879087b2416a5ddb8263051", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", + "reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027", "shasum": "" }, "require": { @@ -4820,9 +4821,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.10" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.11" }, - "time": "2025-03-25T13:44:16+00:00" + "time": "2025-03-26T10:53:16+00:00" }, { "name": "doctrine/annotations", @@ -7154,16 +7155,16 @@ }, { "name": "symfony/console", - "version": "v7.2.1", + "version": "v7.2.5", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3" + "reference": "e51498ea18570c062e7df29d05a7003585b19b88" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/fefcc18c0f5d0efe3ab3152f15857298868dc2c3", - "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3", + "url": "https://api.github.com/repos/symfony/console/zipball/e51498ea18570c062e7df29d05a7003585b19b88", + "reference": "e51498ea18570c062e7df29d05a7003585b19b88", "shasum": "" }, "require": { @@ -7227,7 +7228,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.2.1" + "source": "https://github.com/symfony/console/tree/v7.2.5" }, "funding": [ { @@ -7243,7 +7244,7 @@ "type": "tidelift" } ], - "time": "2024-12-11T03:49:26+00:00" + "time": "2025-03-12T08:11:12+00:00" }, { "name": "symfony/filesystem", @@ -7758,16 +7759,16 @@ }, { "name": "symfony/process", - "version": "v7.2.4", + "version": "v7.2.5", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf" + "reference": "87b7c93e57df9d8e39a093d32587702380ff045d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", - "reference": "d8f411ff3c7ddc4ae9166fb388d1190a2df5b5cf", + "url": "https://api.github.com/repos/symfony/process/zipball/87b7c93e57df9d8e39a093d32587702380ff045d", + "reference": "87b7c93e57df9d8e39a093d32587702380ff045d", "shasum": "" }, "require": { @@ -7799,7 +7800,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.2.4" + "source": "https://github.com/symfony/process/tree/v7.2.5" }, "funding": [ { @@ -7815,7 +7816,7 @@ "type": "tidelift" } ], - "time": "2025-02-05T08:33:46+00:00" + "time": "2025-03-13T12:21:46+00:00" }, { "name": "symfony/string", @@ -8132,9 +8133,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-fix-tenant", + "alias": "0.64.0", + "alias_normalized": "0.64.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From 32a5ff4c8c5e42c9ad0652f20cb91764089e31e8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 2 Apr 2025 13:37:26 +1300 Subject: [PATCH 246/275] Update release --- composer.json | 2 +- composer.lock | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/composer.json b/composer.json index 0eb0f58f04..3920351c06 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-fix-tenant as 0.64.0", + "utopia-php/database": "0.64.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 80d9c144f5..270fbca6a4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d9bc106a753e57907134bb717d486310", + "content-hash": "6a54c8bc4f9f14cd3883f55880864630", "packages": [ { "name": "adhocore/jwt", @@ -3497,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "dev-fix-tenant", + "version": "0.64.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "1b7d8b774ae28d98de82a86e4fef262514e8fc40" + "reference": "6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/1b7d8b774ae28d98de82a86e4fef262514e8fc40", - "reference": "1b7d8b774ae28d98de82a86e4fef262514e8fc40", + "url": "https://api.github.com/repos/utopia-php/database/zipball/6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b", + "reference": "6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b", "shasum": "" }, "require": { @@ -3547,9 +3547,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/fix-tenant" + "source": "https://github.com/utopia-php/database/tree/0.64.1" }, - "time": "2025-04-02T00:00:03+00:00" + "time": "2025-04-02T00:35:29+00:00" }, { "name": "utopia-php/domains", @@ -8124,18 +8124,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-fix-tenant", - "alias": "0.64.0", - "alias_normalized": "0.64.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 1660c0b00c15c216d1f422719217893fc92349cd Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 3 Apr 2025 02:44:29 +0000 Subject: [PATCH 247/275] Fix: image transformation cache --- app/controllers/shared/api.php | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 1015400a12..87382bbfe2 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -390,7 +390,8 @@ App::init() ->inject('timelimit') ->inject('mode') ->inject('apiKey') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, string $mode, ?Key $apiKey) use ($usageDatabaseListener, $eventDatabaseListener) { + ->inject('plan') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, string $mode, ?Key $apiKey, array $plan) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -520,6 +521,10 @@ App::init() $useCache = $route->getLabel('cache', false); if ($useCache) { + $route = $utopia->match($request); + $isImageTransformation = $route->getPath() === '/v1/storage/buckets/:bucketId/files/:fileId/preview'; + $isDisabled = isset($plan['imageTransformations']) && $plan['imageTransformations'] === -1 && !Auth::isPrivilegedUser(Authorization::getRoles()); + $key = md5($request->getURI() . '*' . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER); $cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key)); $cache = new Cache( @@ -532,7 +537,7 @@ App::init() $parts = explode('/', $cacheLog->getAttribute('resourceType')); $type = $parts[0] ?? null; - if ($type === 'bucket') { + if ($type === 'bucket' && (!$isImageTransformation || !$isDisabled)) { $bucketId = $parts[1] ?? null; $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); @@ -573,8 +578,10 @@ App::init() $response ->addHeader('Cache-Control', sprintf('private, max-age=%d', $timestamp)) ->addHeader('X-Appwrite-Cache', 'hit') - ->setContentType($cacheLog->getAttribute('mimeType')) - ->send($data); + ->setContentType($cacheLog->getAttribute('mimeType')); + if (!$isImageTransformation || !$isDisabled) { + $response->send($data); + } } else { $response ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate') From bd87672607cbeb8c3152e8f3878d059bafb1daa9 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 3 Apr 2025 18:31:50 +0300 Subject: [PATCH 248/275] Remove audit payload --- src/Appwrite/Platform/Workers/Audits.php | 6 +++++- src/Appwrite/Platform/Workers/Databases.php | 3 ++- src/Appwrite/Platform/Workers/Deletes.php | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index ed5ff8010a..76309145b8 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -74,7 +74,11 @@ class Audits extends Action Console::info('Aggregating audit logs'); $event = $payload['event'] ?? ''; - $auditPayload = $payload['payload'] ?? ''; + + $auditPayload = ''; + if ($project->getId() === 'console') { + $auditPayload = $payload['payload'] ?? ''; + } $mode = $payload['mode'] ?? ''; $resource = $payload['resource'] ?? ''; $userAgent = $payload['userAgent'] ?? ''; diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 4abd035599..44738b557c 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -565,7 +565,8 @@ class Databases extends Action try { $documents = $database->deleteDocuments($collectionId, $queries); } catch (\Throwable $th) { - Console::error('Failed to delete documents for collection ' . $collectionId . ': ' . $th->getMessage()); + $tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : ''; + Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collectionId} {$tenant} :{$th->getMessage()}"); return; } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index a9b83976a4..09c60c7740 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -1050,7 +1050,8 @@ class Deletes extends Action try { $documents = $database->deleteDocuments($collection, $queries); } catch (Throwable $th) { - Console::error('Failed to delete documents for collection ' . $collection . ': ' . $th->getMessage()); + $tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : ''; + Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collection} {$tenant} :{$th->getMessage()}"); return; } From 2a672f5a633b98f1bcecc73d3addbfc78eecc4b5 Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 5 Apr 2025 19:20:24 +0530 Subject: [PATCH 249/275] fix: don't allow the only owner in an org to change their role to anything else. --- app/controllers/api/teams.php | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index dfdccf5e99..d4db6a4a52 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1034,7 +1034,6 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId') ->param('membershipId', '', new UID(), 'Membership ID.') ->param('roles', [], function (Document $project) { if ($project->getId() === 'console') { - ; $roles = array_keys(Config::getParam('roles', [])); array_filter($roles, function ($role) { return !in_array($role, [Auth::USER_ROLE_APPS, Auth::USER_ROLE_GUESTS, Auth::USER_ROLE_USERS]); @@ -1046,9 +1045,10 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId') ->inject('request') ->inject('response') ->inject('user') + ->inject('project') ->inject('dbForProject') ->inject('queueForEvents') - ->action(function (string $teamId, string $membershipId, array $roles, Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { + ->action(function (string $teamId, string $membershipId, array $roles, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) { $team = $dbForProject->getDocument('teams', $teamId); if ($team->isEmpty()) { @@ -1069,6 +1069,21 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId') $isAppUser = Auth::isAppUser(Authorization::getRoles()); $isOwner = Authorization::isRole('team:' . $team->getId() . '/owner'); + if ($project->getId() === 'console') { + // Quick check: fetch up to 2 owners to determine if only one exists + $ownersCount = $dbForProject->count( + collection: 'memberships', + queries: [Query::contains('roles', ['owner'])], + max: 2 + ); + + // If there's only one owner, + // and the requester is that owner, prevent role change + if ($ownersCount === 1 && $isOwner) { + throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'There must be at least one owner in the organization.'); + } + } + if (!$isOwner && !$isPrivilegedUser && !$isAppUser) { // Not owner, not admin, not app (server) throw new Exception(Exception::USER_UNAUTHORIZED, 'User is not allowed to modify roles'); } From 7297188c1bf0dd67229afc2ef30ad527cc76fecc Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 5 Apr 2025 19:42:20 +0530 Subject: [PATCH 250/275] update: roles check. --- app/controllers/api/teams.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index d4db6a4a52..3e0e366b6b 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1077,9 +1077,9 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId') max: 2 ); - // If there's only one owner, - // and the requester is that owner, prevent role change - if ($ownersCount === 1 && $isOwner) { + // Prevent role change if there's only one owner left, + // the requester is that owner, and the new `$roles` no longer include 'owner'! + if ($ownersCount === 1 && $isOwner && !\in_array('owner', $roles)) { throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'There must be at least one owner in the organization.'); } } From abd953974d4dec5af095ced36dec79ab0904df5f Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 5 Apr 2025 14:43:07 +0000 Subject: [PATCH 251/275] chore: give read write permission to vendor for gitpod config --- .gitpod.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index e9c0928229..c103d81900 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,13 +1,14 @@ tasks: - name: Run Appwrite Docker Stack init: | - git config --global --add safe.directory /app git submodule update --init docker compose build docker compose pull docker pull composer command: | + sudo chown -R $(id -u):$(id -g) vendor/ docker run --rm --interactive --tty \ + --user "$(id -u):$(id -g)" \ --volume $PWD:/app \ composer install \ --ignore-platform-reqs \ From e5ef825ac41fd63fad3983115730c48a3725fd14 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 5 Apr 2025 15:47:22 +0000 Subject: [PATCH 252/275] chore: update php image to 0.8.1 --- composer.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/composer.lock b/composer.lock index 270fbca6a4..8db4706bb5 100644 --- a/composer.lock +++ b/composer.lock @@ -2965,16 +2965,16 @@ }, { "name": "tbachert/spi", - "version": "v1.0.2", + "version": "v1.0.3", "source": { "type": "git", "url": "https://github.com/Nevay/spi.git", - "reference": "2ddfaf815dafb45791a61b08170de8d583c16062" + "reference": "506a79c98e1a51522e76ee921ccb6c62d52faf3a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Nevay/spi/zipball/2ddfaf815dafb45791a61b08170de8d583c16062", - "reference": "2ddfaf815dafb45791a61b08170de8d583c16062", + "url": "https://api.github.com/repos/Nevay/spi/zipball/506a79c98e1a51522e76ee921ccb6c62d52faf3a", + "reference": "506a79c98e1a51522e76ee921ccb6c62d52faf3a", "shasum": "" }, "require": { @@ -3011,9 +3011,9 @@ ], "support": { "issues": "https://github.com/Nevay/spi/issues", - "source": "https://github.com/Nevay/spi/tree/v1.0.2" + "source": "https://github.com/Nevay/spi/tree/v1.0.3" }, - "time": "2024-10-04T16:36:12+00:00" + "time": "2025-04-02T19:38:14+00:00" }, { "name": "thecodingmachine/safe", @@ -3746,16 +3746,16 @@ }, { "name": "utopia-php/image", - "version": "0.8.0", + "version": "0.8.1", "source": { "type": "git", "url": "https://github.com/utopia-php/image.git", - "reference": "dcae5b1c6deb3ff6865f4e68f012b3709c289bca" + "reference": "e8cc7dd14f423270a1b7570ec0dae88a66195b63" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/image/zipball/dcae5b1c6deb3ff6865f4e68f012b3709c289bca", - "reference": "dcae5b1c6deb3ff6865f4e68f012b3709c289bca", + "url": "https://api.github.com/repos/utopia-php/image/zipball/e8cc7dd14f423270a1b7570ec0dae88a66195b63", + "reference": "e8cc7dd14f423270a1b7570ec0dae88a66195b63", "shasum": "" }, "require": { @@ -3789,9 +3789,9 @@ ], "support": { "issues": "https://github.com/utopia-php/image/issues", - "source": "https://github.com/utopia-php/image/tree/0.8.0" + "source": "https://github.com/utopia-php/image/tree/0.8.1" }, - "time": "2025-02-20T11:49:03+00:00" + "time": "2025-04-04T18:55:20+00:00" }, { "name": "utopia-php/locale", From 403083b0cef7053588474b98587b036790fdc01c Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sun, 6 Apr 2025 08:31:46 +0530 Subject: [PATCH 253/275] Update .gitpod.yml --- .gitpod.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index c103d81900..78375bb1f0 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -6,7 +6,6 @@ tasks: docker compose pull docker pull composer command: | - sudo chown -R $(id -u):$(id -g) vendor/ docker run --rm --interactive --tty \ --user "$(id -u):$(id -g)" \ --volume $PWD:/app \ From c3c568653fd9f4a5693e5b06a8c4d6c342e30fc6 Mon Sep 17 00:00:00 2001 From: Darshan Date: Sun, 6 Apr 2025 10:21:55 +0530 Subject: [PATCH 254/275] add: roles check test. --- tests/e2e/Services/Teams/TeamsBase.php | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/e2e/Services/Teams/TeamsBase.php b/tests/e2e/Services/Teams/TeamsBase.php index 2328e4cdbf..87486696d2 100644 --- a/tests/e2e/Services/Teams/TeamsBase.php +++ b/tests/e2e/Services/Teams/TeamsBase.php @@ -37,6 +37,31 @@ trait TeamsBase $teamUid = $response1['body']['$id']; $teamName = $response1['body']['name']; + /** + * Test: Attempt to downgrade the only OWNER in a team (should fail) + */ + // Step 1: Fetch all team memberships — only one exists at this point + $response = $this->client->call(Client::METHOD_GET, '/teams/' . $teamUid . '/memberships', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + // Step 2: Extract the membership ID of the only member (also the only OWNER) + $membershipID = $response['body']['memberships'][0]['$id']; + + // Step 3: Attempt to downgrade the member's role to 'developer' + $response = $this->client->call(Client::METHOD_PATCH, '/teams/' . $teamUid . '/memberships/' . $membershipID, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'roles' => ['developer'] + ]); + + // Step 4: Assert failure — cannot remove the only OWNER from a team + $this->assertEquals(400, $response['headers']['status-code']); + $this->assertEquals('general_argument_invalid', $response['body']['type']); + $this->assertEquals('There must be at least one owner in the organization.', $response['body']['message']); + $teamId = ID::unique(); $response2 = $this->client->call(Client::METHOD_POST, '/teams', array_merge([ 'content-type' => 'application/json', From 2243f0c72e45e5a8abfe716bb2b8470375a07f6b Mon Sep 17 00:00:00 2001 From: Darshan Date: Sun, 6 Apr 2025 10:29:18 +0530 Subject: [PATCH 255/275] fix: test for only console db <> orgs. --- tests/e2e/Services/Teams/TeamsBase.php | 44 +++++++++++++++----------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/tests/e2e/Services/Teams/TeamsBase.php b/tests/e2e/Services/Teams/TeamsBase.php index 87486696d2..80ac1621ee 100644 --- a/tests/e2e/Services/Teams/TeamsBase.php +++ b/tests/e2e/Services/Teams/TeamsBase.php @@ -38,29 +38,35 @@ trait TeamsBase $teamName = $response1['body']['name']; /** - * Test: Attempt to downgrade the only OWNER in a team (should fail) + * Test: Attempt to downgrade the only OWNER in an organization (should fail) */ - // Step 1: Fetch all team memberships — only one exists at this point - $response = $this->client->call(Client::METHOD_GET, '/teams/' . $teamUid . '/memberships', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); + if ($this->getProject()['$id'] === 'console') { + // Step 1: Fetch all team memberships — only one exists at this point + $response = $this->client->call(Client::METHOD_GET, '/teams/' . $teamUid . '/memberships', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + Query::limit(1)->toString(), + ], + ]); - // Step 2: Extract the membership ID of the only member (also the only OWNER) - $membershipID = $response['body']['memberships'][0]['$id']; + // Step 2: Extract the membership ID of the only member (also the only OWNER) + $membershipID = $response['body']['memberships'][0]['$id']; - // Step 3: Attempt to downgrade the member's role to 'developer' - $response = $this->client->call(Client::METHOD_PATCH, '/teams/' . $teamUid . '/memberships/' . $membershipID, array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), [ - 'roles' => ['developer'] - ]); + // Step 3: Attempt to downgrade the member's role to 'developer' + $response = $this->client->call(Client::METHOD_PATCH, '/teams/' . $teamUid . '/memberships/' . $membershipID, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'roles' => ['developer'] + ]); - // Step 4: Assert failure — cannot remove the only OWNER from a team - $this->assertEquals(400, $response['headers']['status-code']); - $this->assertEquals('general_argument_invalid', $response['body']['type']); - $this->assertEquals('There must be at least one owner in the organization.', $response['body']['message']); + // Step 4: Assert failure — cannot remove the only OWNER from a team + $this->assertEquals(400, $response['headers']['status-code']); + $this->assertEquals('general_argument_invalid', $response['body']['type']); + $this->assertEquals('There must be at least one owner in the organization.', $response['body']['message']); + } $teamId = ID::unique(); $response2 = $this->client->call(Client::METHOD_POST, '/teams', array_merge([ From 179def8f4d84921a6cd5efe58deeeb81c910470e Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 7 Apr 2025 10:12:58 +0530 Subject: [PATCH 256/275] add: imports collection --- app/config/collections/projects.php | 112 ++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 4461fcada6..ee2c6c0e71 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1881,4 +1881,116 @@ return [ ] ], ], + + 'imports' => [ + '$collection' => ID::custom(Database::METADATA), + '$id' => ID::custom('imports'), + 'name' => 'CSV Imports', + 'attributes' => [ + [ + '$id' => ID::custom('migrationId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('migrationInternalId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('status'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('size'), + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('startedAt'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => ID::custom('resourceId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('resourceType'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('error'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_status', + 'type' => Database::INDEX_KEY, + 'attributes' => ['status'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_resourceId', + 'type' => Database::INDEX_KEY, + 'attributes' => ['resourceId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ], + ] ]; From 954ed45a36ff821125cbfa63c75f16aebc7fb8f1 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Fri, 28 Feb 2025 13:52:34 +0100 Subject: [PATCH 257/275] feat: update appwrite to use generic S3 adapter --- app/init/resources.php | 15 +++++++++++++-- docker-compose.yml | 6 +++++- src/Executor/Executor.php | 6 +++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/app/init/resources.php b/app/init/resources.php index 4e53b24c06..0011a8c677 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -38,6 +38,7 @@ use Utopia\Logger\Log; use Utopia\Pools\Group; use Utopia\Queue\Publisher; use Utopia\Storage\Device; +use Utopia\Storage\Device\AWS; use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Linode; @@ -540,7 +541,12 @@ function getDevice(string $root, string $connection = ''): Device switch ($device) { case Storage::DEVICE_S3: - return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl, $url); + if (!empty($url)) { + return new S3($root, $accessKey, $accessSecret, $url, $region, $acl); + } else { + return new AWS($root, $accessKey, $accessSecret, $bucket, $region, $acl); + } + // no break case STORAGE::DEVICE_DO_SPACES: $device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); $device->setHttpVersion(S3::HTTP_VERSION_1_1); @@ -567,7 +573,12 @@ function getDevice(string $root, string $connection = ''): Device $s3Bucket = System::getEnv('_APP_STORAGE_S3_BUCKET', ''); $s3Acl = 'private'; $s3EndpointUrl = System::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); - return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl, $s3EndpointUrl); + if (!empty($s3EndpointUrl)) { + return new S3($root, $s3AccessKey, $s3SecretKey, $s3EndpointUrl, $s3Region, $s3Acl); + } else { + return new AWS($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + } + // no break case Storage::DEVICE_DO_SPACES: $doSpacesAccessKey = System::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); $doSpacesSecretKey = System::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); diff --git a/docker-compose.yml b/docker-compose.yml index 05ddba967a..8c8a364f30 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -198,6 +198,8 @@ services: - _APP_DATABASE_SHARED_TABLES_V1 - _APP_DATABASE_SHARED_NAMESPACE - _APP_FUNCTIONS_CREATION_ABUSE_LIMIT + extra_hosts: + - "host.docker.internal:host-gateway" appwrite-console: <<: *x-logging @@ -489,6 +491,8 @@ services: - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES + extra_hosts: + - "host.docker.internal:host-gateway" appwrite-worker-certificates: entrypoint: worker-certificates @@ -1131,4 +1135,4 @@ volumes: appwrite-certificates: appwrite-functions: appwrite-builds: - appwrite-config: \ No newline at end of file + appwrite-config: diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index c230cfb664..936fba5b24 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -235,7 +235,11 @@ class Executor throw new \Exception($message, $status); } - $response['body']['headers'] = \json_decode($response['body']['headers'] ?? '{}', true); + $headers = $response['body']['headers'] ?? []; + if (is_string($headers)) { + $headers = \json_decode($headers, true); + } + $response['body']['headers'] = $headers; $response['body']['statusCode'] = \intval($response['body']['statusCode'] ?? 500); $response['body']['duration'] = \floatval($response['body']['duration'] ?? 0); $response['body']['startTime'] = \floatval($response['body']['startTime'] ?? \microtime(true)); From 0ae70f8321e1916eaad4f6afa1cd430e55f7b3b9 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 25 Mar 2025 12:33:03 +0100 Subject: [PATCH 258/275] feat: inject executor instead of creating a new instance --- app/cli.php | 3 ++ app/controllers/api/functions.php | 12 +++---- app/controllers/general.php | 23 +++++++------ app/init/resources.php | 3 ++ app/worker.php | 3 ++ src/Appwrite/Platform/Workers/Builds.php | 15 +++++---- src/Appwrite/Platform/Workers/Deletes.php | 26 ++++++++------- src/Appwrite/Platform/Workers/Functions.php | 11 ++++-- src/Executor/Executor.php | 37 +++++++++++++-------- 9 files changed, 82 insertions(+), 51 deletions(-) diff --git a/app/cli.php b/app/cli.php index f080217365..82f229673e 100644 --- a/app/cli.php +++ b/app/cli.php @@ -9,6 +9,7 @@ use Appwrite\Event\StatsResources; use Appwrite\Event\StatsUsage; use Appwrite\Platform\Appwrite; use Appwrite\Runtimes\Runtimes; +use Executor\Executor; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\CLI; @@ -255,6 +256,8 @@ CLI::setResource('logError', function (Registry $register) { }; }, ['register']); +CLI::setResource('executor', fn () => new Executor(fn (string $projectId, string $deploymentId) => System::getEnv('_APP_EXECUTOR_HOST'))); + $platform = new Appwrite(); $platform->init(Service::TYPE_TASK); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 37932e4165..c7f24f984a 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -869,7 +869,8 @@ App::put('/v1/functions/:functionId') ->inject('queueForBuilds') ->inject('dbForPlatform') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, ?string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { + ->inject('executor') + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, ?string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github, Executor $executor) use ($redeployVcs) { // TODO: If only branch changes, re-deploy $function = $dbForProject->getDocument('functions', $functionId); @@ -972,7 +973,6 @@ App::put('/v1/functions/:functionId') // Enforce Cold Start if spec limits change. if ($function->getAttribute('specification') !== $specification && !empty($function->getAttribute('deployment'))) { - $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); try { $executor->deleteRuntime($project->getId(), $function->getAttribute('deployment')); } catch (\Throwable $th) { @@ -1779,7 +1779,8 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId/build') ->inject('dbForProject') ->inject('project') ->inject('queueForEvents') - ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Document $project, Event $queueForEvents) { + ->inject('executor') + ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Executor $executor) { $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { @@ -1834,7 +1835,6 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId/build') $dbForProject->purgeCachedDocument('deployments', $deployment->getId()); try { - $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); $executor->deleteRuntime($project->getId(), $deploymentId . "-build"); } catch (\Throwable $th) { // Don't throw if the deployment doesn't exist @@ -1886,8 +1886,9 @@ App::post('/v1/functions/:functionId/executions') ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('queueForFunctions') + ->inject('executor') ->inject('geodb') - ->action(function (string $functionId, string $body, mixed $async, string $path, string $method, mixed $headers, ?string $scheduledAt, Response $response, Request $request, Document $project, Database $dbForProject, Database $dbForPlatform, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb) { + ->action(function (string $functionId, string $body, mixed $async, string $path, string $method, mixed $headers, ?string $scheduledAt, Response $response, Request $request, Document $project, Database $dbForProject, Database $dbForPlatform, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb) { $async = \strval($async) === 'true' || \strval($async) === '1'; if (!$async && !is_null($scheduledAt)) { @@ -2160,7 +2161,6 @@ App::post('/v1/functions/:functionId/executions') ]); /** Execute function */ - $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); try { $version = $function->getAttribute('version', 'v2'); $command = $runtime['startCommand']; diff --git a/app/controllers/general.php b/app/controllers/general.php index 1423763cec..6c1167cf76 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -50,7 +50,7 @@ Config::setParam('domainVerification', false); Config::setParam('cookieDomain', 'localhost'); Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) +function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml'); @@ -339,7 +339,6 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw ]); /** Execute function */ - $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); try { $version = $function->getAttribute('version', 'v2'); $command = $runtime['startCommand']; @@ -503,9 +502,10 @@ App::init() ->inject('queueForEvents') ->inject('queueForCertificates') ->inject('queueForFunctions') + ->inject('executor') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, StatsUsage $queueForStatsUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHostname) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, StatsUsage $queueForStatsUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, Executor $executor, callable $isResourceBlocked, string $previewHostname) { /* * Appwrite Router */ @@ -513,7 +513,7 @@ App::init() $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain if ($host !== $mainDomain || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $previewHostname)) { return; } } @@ -742,11 +742,12 @@ App::options() ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('queueForFunctions') + ->inject('executor') ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') ->inject('project') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname, Document $project) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, string $previewHostname, Document $project) { /* * Appwrite Router */ @@ -754,7 +755,7 @@ App::options() $mainDomain = System::getEnv('_APP_DOMAIN', ''); // Only run Router when external domain if ($host !== $mainDomain || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $previewHostname)) { return; } } @@ -1062,10 +1063,11 @@ App::get('/robots.txt') ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('queueForFunctions') + ->inject('executor') ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); @@ -1073,7 +1075,7 @@ App::get('/robots.txt') $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); + router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $previewHostname); } }); @@ -1090,10 +1092,11 @@ App::get('/humans.txt') ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('queueForFunctions') + ->inject('executor') ->inject('geodb') ->inject('isResourceBlocked') ->inject('previewHostname') - ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { + ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Executor $executor, Reader $geodb, callable $isResourceBlocked, string $previewHostname) { $host = $request->getHostname() ?? ''; $mainDomain = System::getEnv('_APP_DOMAIN', ''); @@ -1101,7 +1104,7 @@ App::get('/humans.txt') $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname); + router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $executor, $geodb, $isResourceBlocked, $previewHostname); } }); diff --git a/app/init/resources.php b/app/init/resources.php index 0011a8c677..2360179913 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -21,6 +21,7 @@ use Appwrite\Extend\Exception; use Appwrite\GraphQL\Schema; use Appwrite\Network\Validator\Origin; use Appwrite\Utopia\Request; +use Executor\Executor; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Sharding; @@ -833,3 +834,5 @@ App::setResource('apiKey', function (Request $request, Document $project): ?Key return Key::decode($project, $key); }, ['request', 'project']); + +App::setResource('executor', fn () => new Executor(fn (string $projectId, string $deploymentId) => System::getEnv('_APP_EXECUTOR_HOST'))); diff --git a/app/worker.php b/app/worker.php index 90496c0430..6a51ee55be 100644 --- a/app/worker.php +++ b/app/worker.php @@ -18,6 +18,7 @@ use Appwrite\Event\StatsUsage; use Appwrite\Event\StatsUsageDump; use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; +use Executor\Executor; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; @@ -413,6 +414,8 @@ Server::setResource('logError', function (Registry $register, Document $project) }; }, ['register', 'project']); +Server::setResource('executor', fn () => new Executor(fn (string $projectId, string $deploymentId) => System::getEnv('_APP_EXECUTOR_HOST'))); + $pools = $register->get('pools'); $platform = new Appwrite(); $args = $platform->getEnv('argv'); diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 6f26e9a80c..4057d4b190 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -59,8 +59,9 @@ class Builds extends Action ->inject('deviceForFunctions') ->inject('isResourceBlocked') ->inject('log') - ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) => - $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log)); + ->inject('executor') + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log, Executor $executor) => + $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log, $executor)); } /** @@ -76,10 +77,11 @@ class Builds extends Action * @param Database $dbForProject * @param Device $deviceForFunctions * @param Log $log + * @param Executor $executor * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log, Executor $executor): void { $payload = $message->getPayload() ?? []; @@ -100,7 +102,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($deviceForFunctions, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log); + $this->buildDeployment($deviceForFunctions, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log, $executor); break; default: @@ -123,14 +125,13 @@ class Builds extends Action * @param Document $deployment * @param Document $template * @param Log $log + * @param Executor $executor * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void + protected function buildDeployment(Device $deviceForFunctions, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log, Executor $executor): void { - $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); - $functionId = $function->getId(); $log->addTag('functionId', $function->getId()); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 09c60c7740..5092599d67 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -55,12 +55,13 @@ class Deletes extends Action ->inject('deviceForBuilds') ->inject('deviceForCache') ->inject('certificates') + ->inject('executor') ->inject('executionRetention') ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Executor $executor, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executor, $executionRetention, $auditRetention, $log) ); } @@ -68,7 +69,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Executor $executor, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -93,10 +94,10 @@ class Deletes extends Action $this->deleteProject($dbForPlatform, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project); + $this->deleteFunction($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project, $executor); break; case DELETE_TYPE_DEPLOYMENTS: - $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); + $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project, $executor); break; case DELETE_TYPE_USERS: $this->deleteUser($getProjectDB, $document, $project); @@ -827,10 +828,11 @@ class Deletes extends Action * @param Device $deviceForBuilds * @param Document $document function document * @param Document $project + * @param Executor $executor * @return void * @throws Exception */ - private function deleteFunction(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void + private function deleteFunction(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project, Executor $executor): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -922,7 +924,7 @@ class Deletes extends Action * Request executor to delete all deployment containers */ Console::info("Requesting executor to delete all deployment containers for function " . $functionId); - $this->deleteRuntimes($getProjectDB, $document, $project); + $this->deleteRuntimes($getProjectDB, $document, $project, $executor); } /** @@ -993,10 +995,11 @@ class Deletes extends Action * @param Device $deviceForBuilds * @param Document $document * @param Document $project + * @param Executor $executor * @return void * @throws Exception */ - private function deleteDeployment(callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void + private function deleteDeployment(callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project, Executor $executor): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); @@ -1024,7 +1027,7 @@ class Deletes extends Action * Request executor to delete all deployment containers */ Console::info("Requesting executor to delete deployment container for deployment " . $deploymentId); - $this->deleteRuntimes($getProjectDB, $document, $project); + $this->deleteRuntimes($getProjectDB, $document, $project, $executor); } /** @@ -1180,13 +1183,12 @@ class Deletes extends Action * @param callable $getProjectDB * @param ?Document $function * @param Document $project + * @param Executor $executor * @return void * @throws Exception */ - private function deleteRuntimes(callable $getProjectDB, ?Document $function, Document $project): void + private function deleteRuntimes(callable $getProjectDB, ?Document $function, Document $project, Executor $executor): void { - $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); - $deleteByFunction = function (Document $function) use ($getProjectDB, $project, $executor) { $this->listByGroup( 'deployments', diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index a7caa3207f..4e1794c085 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -51,11 +51,12 @@ class Functions extends Action ->inject('queueForEvents') ->inject('queueForStatsUsage') ->inject('log') + ->inject('executor') ->inject('isResourceBlocked') - ->callback(fn (Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked)); + ->callback(fn (Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, Executor $executor, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $executor, $isResourceBlocked)); } - public function action(Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void + public function action(Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, Executor $executor, callable $isResourceBlocked): void { $payload = $message->getPayload() ?? []; @@ -146,6 +147,7 @@ class Functions extends Action queueForEvents: $queueForEvents, project: $project, function: $function, + executor: $executor, trigger: 'event', path: '/', method: 'POST', @@ -188,6 +190,7 @@ class Functions extends Action queueForEvents: $queueForEvents, project: $project, function: $function, + executor: $executor, trigger: 'http', path: $path, method: $method, @@ -212,6 +215,7 @@ class Functions extends Action queueForEvents: $queueForEvents, project: $project, function: $function, + executor: $executor, trigger: 'schedule', path: $path, method: $method, @@ -298,6 +302,7 @@ class Functions extends Action * @param Event $queueForEvents * @param Document $project * @param Document $function + * @param Executor $executor * @param string $trigger * @param string $path * @param string $method @@ -324,6 +329,7 @@ class Functions extends Action Event $queueForEvents, Document $project, Document $function, + Executor $executor, string $trigger, string $path, string $method, @@ -514,7 +520,6 @@ class Functions extends Action try { $version = $function->getAttribute('version', 'v2'); $command = $runtime['startCommand']; - $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); $command = $version === 'v2' ? '' : 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "' . $command . '"'; $executionResponse = $executor->createExecution( projectId: $project->getId(), diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index 936fba5b24..6bc2fe7aab 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -21,17 +21,19 @@ class Executor private bool $selfSigned = false; - private string $endpoint; + /** + * @var callable(string, string): string $endpoint + */ + private $endpointSelector; protected array $headers; - public function __construct(string $endpoint) + /** + * @param callable(string, string): string $endpointSelector + */ + public function __construct(callable $endpointSelector) { - if (!filter_var($endpoint, FILTER_VALIDATE_URL)) { - throw new Exception('Unsupported endpoint'); - } - - $this->endpoint = $endpoint; + $this->endpointSelector = $endpointSelector; $this->headers = [ 'content-type' => 'application/json', 'authorization' => 'Bearer ' . System::getEnv('_APP_EXECUTOR_SECRET', ''), @@ -92,7 +94,8 @@ class Executor 'timeout' => $timeout, ]; - $response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout); + $endpoint = $this->selectEndpoint($projectId, $deploymentId); + $response = $this->call($endpoint, self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout); $status = $response['headers']['status-code']; if ($status >= 400) { @@ -123,7 +126,8 @@ class Executor 'timeout' => $timeout ]; - $this->call(self::METHOD_GET, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout, $callback); + $endpoint = $this->selectEndpoint($projectId, $deploymentId); + $this->call($endpoint, self::METHOD_GET, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout, $callback); } /** @@ -139,7 +143,8 @@ class Executor $runtimeId = "$projectId-$deploymentId"; $route = "/runtimes/$runtimeId"; - $response = $this->call(self::METHOD_DELETE, $route, [ + $endpoint = $this->selectEndpoint($projectId, $deploymentId); + $response = $this->call($endpoint, self::METHOD_DELETE, $route, [ 'x-opr-addressing-method' => 'broadcast' ], [], true, 30); @@ -227,7 +232,8 @@ class Executor $requestTimeout = $timeout + 15; } - $response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId, 'content-type' => 'multipart/form-data', 'accept' => 'multipart/form-data' ], $params, true, $requestTimeout); + $endpoint = $this->selectEndpoint($projectId, $deploymentId); + $response = $this->call($endpoint, self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId, 'content-type' => 'multipart/form-data', 'accept' => 'multipart/form-data' ], $params, true, $requestTimeout); $status = $response['headers']['status-code']; if ($status >= 400) { @@ -260,10 +266,10 @@ class Executor * @return array|string * @throws Exception */ - public function call(string $method, string $path = '', array $headers = [], array $params = [], bool $decode = true, int $timeout = 15, callable $callback = null) + private function call(string $endpoint, string $method, string $path = '', array $headers = [], array $params = [], bool $decode = true, int $timeout = 15, callable $callback = null) { $headers = array_merge($this->headers, $headers); - $ch = curl_init($this->endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : '')); + $ch = curl_init($endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : '')); $responseHeaders = []; $responseStatus = -1; $responseType = ''; @@ -426,4 +432,9 @@ class Executor return $output; } + + private function selectEndpoint(string $projectId, string $deploymentId): string + { + return call_user_func($this->endpointSelector, $projectId, $deploymentId); + } } From 8dfbc128acc173e741c92f7954bc2fdf717aec27 Mon Sep 17 00:00:00 2001 From: Darshan Date: Tue, 8 Apr 2025 11:19:39 +0530 Subject: [PATCH 259/275] feat: import csv. --- Dockerfile | 2 +- app/config/collections/projects.php | 24 +++++- app/controllers/api/migrations.php | 90 ++++++++++++++++++++- docs/references/migrations/migration-csv.md | 1 + 4 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 docs/references/migrations/migration-csv.md diff --git a/Dockerfile b/Dockerfile index 88d5ed030b..1fdaaf2f0e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ COPY --from=composer /usr/local/src/vendor /usr/src/code/vendor COPY ./app /usr/src/code/app COPY ./public /usr/src/code/public COPY ./bin /usr/local/bin -COPY ./docs /usr/src/code/docs +#COPY ./docs /usr/src/code/docs COPY ./src /usr/src/code/src COPY ./dev /usr/src/code/dev diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index ee2c6c0e71..851b467159 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1848,7 +1848,29 @@ return [ 'default' => null, 'array' => false, 'filters' => [], - ] + ], + [ + '$id' => ID::custom('resourceId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('resourceType'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 75afc7ed2c..10fc48bd33 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -1,5 +1,6 @@ dynamic($migration, Response::MODEL_MIGRATION); }); - App::post('/v1/migrations/firebase') ->groups(['api', 'migrations']) ->desc('Migrate Firebase data') @@ -290,6 +294,90 @@ App::post('/v1/migrations/nhost') ->dynamic($migration, Response::MODEL_MIGRATION); }); +App::post('/v1/migrations/csv') + ->groups(['api', 'migrations']) + ->desc('Import documents from a CSV') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.[migrationId].create') + ->label('audits.event', 'migration.create') + ->label('sdk', new Method( + namespace: 'migrations', + name: 'createCsvMigration', + description: '/docs/references/migrations/migration-csv.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_ACCEPTED, + model: Response::MODEL_MIGRATION, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + ->param('resourceId', null, new UID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('deviceForFiles') + ->inject('deviceForLocal') + ->inject('$queueForEvents') + ->inject('queueForMigrations') + ->action(function (string $bucketId, string $fileId, string $resourceId, Request $request, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Migration $queueForEvents, Migration $queueForMigrations) { + + // TODO: Check if there's already a migrations worker process running for CSV Import for the same collection. + // If so, short-circuit and cancel the task early on because console may not allow it but API will. + // if (inProgress(resourceId)) { + // throw some exception. + //} + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); + $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId)); + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $path = $file->getAttribute('path', ''); + + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); + } + + // TODO: send path migrations/csv worker + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => ID::unique(), + 'status' => 'pending', + 'stage' => 'init', + // TODO: add stuff to migration library + 'source' => SourcesCSV::getName(), + 'destination' => DestinationsCSV::getName(), + 'resources' => [Resource::TYPE_DOCUMENT], + 'resourceId' => $resourceId, + 'resourceType' => Resource::TYPE_DATABASE, + 'statusCounters' => [], + 'resourceData' => [], + 'errors' => [], + 'credentials' => [], + ])); + + // TODO: use migrationId or importId? + $queueForEvents->setParam('migrationId', $migration->getId()); + + // Trigger Import + $queueForMigrations + ->setMigration($migration) + ->setProject($project) + ->trigger(); + }); + App::get('/v1/migrations') ->groups(['api', 'migrations']) ->desc('List migrations') diff --git a/docs/references/migrations/migration-csv.md b/docs/references/migrations/migration-csv.md new file mode 100644 index 0000000000..7a32d5ff6e --- /dev/null +++ b/docs/references/migrations/migration-csv.md @@ -0,0 +1 @@ +Import documents from a CSV file into your Appwrite database. This endpoint allows you to import documents from a CSV file uploaded to Appwrite Storage bucket. \ No newline at end of file From 89815cba1c11193b2bc9e9ea77168df53541e24c Mon Sep 17 00:00:00 2001 From: Darshan Date: Tue, 8 Apr 2025 11:20:47 +0530 Subject: [PATCH 260/275] revert: local dockerfile change. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1fdaaf2f0e..88d5ed030b 100755 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ COPY --from=composer /usr/local/src/vendor /usr/src/code/vendor COPY ./app /usr/src/code/app COPY ./public /usr/src/code/public COPY ./bin /usr/local/bin -#COPY ./docs /usr/src/code/docs +COPY ./docs /usr/src/code/docs COPY ./src /usr/src/code/src COPY ./dev /usr/src/code/dev From 624538fcaf5d0c99f86f71e8cede48b3e95d402a Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 9 Apr 2025 16:35:27 +0530 Subject: [PATCH 261/275] revert: local dockerfile change. --- Dockerfile | 2 + app/config/collections/projects.php | 17 +++- app/controllers/api/migrations.php | 102 ++++++++++++++----- app/init/constants.php | 1 + app/init/resources.php | 4 + app/worker.php | 4 + composer.json | 8 +- composer.lock | 54 ++++++---- docker-compose.yml | 6 +- src/Appwrite/Platform/Workers/Migrations.php | 40 +++++++- 10 files changed, 182 insertions(+), 56 deletions(-) diff --git a/Dockerfile b/Dockerfile index 88d5ed030b..4b5ac3fc62 100755 --- a/Dockerfile +++ b/Dockerfile @@ -44,12 +44,14 @@ COPY ./dev /usr/src/code/dev # Set Volumes RUN mkdir -p /storage/uploads && \ + mkdir -p /storage/csv-imports && \ mkdir -p /storage/cache && \ mkdir -p /storage/config && \ mkdir -p /storage/certificates && \ mkdir -p /storage/functions && \ mkdir -p /storage/debug && \ chown -Rf www-data.www-data /storage/uploads && chmod -Rf 0755 /storage/uploads && \ + chown -Rf www-data.www-data /storage/csv-imports && chmod -Rf 0755 /storage/csv-imports && \ chown -Rf www-data.www-data /storage/cache && chmod -Rf 0755 /storage/cache && \ chown -Rf www-data.www-data /storage/config && chmod -Rf 0755 /storage/config && \ chown -Rf www-data.www-data /storage/certificates && chmod -Rf 0755 /storage/certificates && \ diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 851b467159..d28b2a7e38 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1894,6 +1894,13 @@ return [ 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_key_resource_id', + 'type' => Database::INDEX_KEY, + 'attributes' => ['resourceId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_DESC], + ], [ '$id' => ID::custom('_fulltext_search'), 'type' => Database::INDEX_FULLTEXT, @@ -1935,7 +1942,7 @@ return [ '$id' => ID::custom('status'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 16, + 'size' => Database::LENGTH_KEY, 'signed' => true, 'required' => false, 'default' => null, @@ -1987,14 +1994,14 @@ return [ 'filters' => [], ], [ - '$id' => ID::custom('error'), + '$id' => ID::custom('errors'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 2048, + 'size' => 65535, 'signed' => true, - 'required' => false, + 'required' => true, 'default' => null, - 'array' => false, + 'array' => true, 'filters' => [], ], ], diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 10fc48bd33..90f0930ff2 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -4,12 +4,12 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Event; use Appwrite\Event\Migration; use Appwrite\Extend\Exception; +use Appwrite\OpenSSL\OpenSSL; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Database\Validator\Queries\Migrations; -use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Utopia\App; use Utopia\Database\Database; @@ -22,10 +22,16 @@ use Utopia\Database\Validator\Query\Cursor; use Utopia\Database\Validator\UID; use Utopia\Migration\Resource; use Utopia\Migration\Sources\Appwrite; +use Utopia\Migration\Sources\Csv; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; +use Utopia\Migration\Transfer; +use Utopia\Storage\Compression\Algorithms\GZIP; +use Utopia\Storage\Compression\Algorithms\Zstd; +use Utopia\Storage\Compression\Compression; use Utopia\Storage\Device; +use Utopia\System\System; use Utopia\Validator\ArrayList; use Utopia\Validator\Integer; use Utopia\Validator\Text; @@ -314,28 +320,38 @@ App::post('/v1/migrations/csv') )) ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') ->param('fileId', '', new UID(), 'File ID.') - ->param('resourceId', null, new UID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') - ->inject('request') + ->param('resourceId', null, new Text(75), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') ->inject('response') ->inject('dbForProject') ->inject('project') ->inject('deviceForFiles') - ->inject('deviceForLocal') - ->inject('$queueForEvents') + ->inject('deviceForCsvImports') + ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $fileId, string $resourceId, Request $request, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Migration $queueForEvents, Migration $queueForMigrations) { - - // TODO: Check if there's already a migrations worker process running for CSV Import for the same collection. - // If so, short-circuit and cancel the task early on because console may not allow it but API will. - // if (inProgress(resourceId)) { - // throw some exception. - //} - - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - + ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForCsvImports, Event $queueForEvents, Migration $queueForMigrations) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); + // Check if migration/import is already in progress! + // if for some reason the worker crashes, the stage will always be `init`, what do we do? + $isInProgress = Authorization::skip(function () use ($dbForProject, $resourceId) { + $exists = $dbForProject->findOne( + 'migrations', + [ + Query::notEqual('stage', 'finished'), + Query::equal('resourceId', [$resourceId]), + ] + ); + + return !$exists->isEmpty(); + }); + + if ($isInProgress || (!$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::MIGRATION_IN_PROGRESS, 'An import is already in progress for this collection.'); + } + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } @@ -346,36 +362,72 @@ App::post('/v1/migrations/csv') } $path = $file->getAttribute('path', ''); - if (!$deviceForFiles->exists($path)) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); } - // TODO: send path migrations/csv worker + // read file content. + $source = $deviceForFiles->read($path); + + // decrypt + if (!empty($file->getAttribute('openSSLCipher'))) { + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + // decompress + switch ($file->getAttribute('algorithm', Compression::NONE)) { + case Compression::ZSTD: + $compressor = new Zstd(); + $source = $compressor->decompress($source); + break; + case Compression::GZIP: + $compressor = new GZIP(); + $source = $compressor->decompress($source); + break; + } + + // copy to temporary folder + $migrationId = ID::unique(); + $path = $deviceForCsvImports->getRoot() . '/' . $migrationId . '_' . $fileId . '.csv'; + $deviceForCsvImports->write($path, $source, 'text/csv'); + $fileSize = $deviceForCsvImports->getFileSize($path); + $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); + $migration = $dbForProject->createDocument('migrations', new Document([ - '$id' => ID::unique(), + '$id' => $migrationId, 'status' => 'pending', 'stage' => 'init', - // TODO: add stuff to migration library - 'source' => SourcesCSV::getName(), - 'destination' => DestinationsCSV::getName(), - 'resources' => [Resource::TYPE_DOCUMENT], + 'source' => Csv::getName(), + 'destination' => Appwrite::class::getName(), + 'resources' => $resources, 'resourceId' => $resourceId, 'resourceType' => Resource::TYPE_DATABASE, 'statusCounters' => [], 'resourceData' => [], 'errors' => [], - 'credentials' => [], + 'credentials' => [ + 'path' => $path, + 'size' => $fileSize, + ], ])); - // TODO: use migrationId or importId? $queueForEvents->setParam('migrationId', $migration->getId()); - // Trigger Import $queueForMigrations ->setMigration($migration) ->setProject($project) ->trigger(); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($migration, Response::MODEL_MIGRATION); }); App::get('/v1/migrations') diff --git a/app/init/constants.php b/app/init/constants.php index 5e4edfd97d..2deeff1c95 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -49,6 +49,7 @@ const APP_STORAGE_UPLOADS = '/storage/uploads'; const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_BUILDS = '/storage/builds'; const APP_STORAGE_CACHE = '/storage/cache'; +const APP_STORAGE_CSV_IMPORTS = '/storage/csv-imports'; // Temporary storage for csv imports const APP_STORAGE_CERTIFICATES = '/storage/certificates'; const APP_STORAGE_CONFIG = '/storage/config'; const APP_STORAGE_READ_BUFFER = 20 * (1000 * 1000); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT` diff --git a/app/init/resources.php b/app/init/resources.php index 2360179913..6eae238fee 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -508,6 +508,10 @@ App::setResource('deviceForFiles', function ($project) { return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); }, ['project']); +App::setResource('deviceForCsvImports', function (Document $project) { + return getDevice(APP_STORAGE_CSV_IMPORTS . '/app-' . $project->getId()); +}, ['project']); + App::setResource('deviceForFunctions', function ($project) { return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); }, ['project']); diff --git a/app/worker.php b/app/worker.php index 6a51ee55be..4e865858a0 100644 --- a/app/worker.php +++ b/app/worker.php @@ -339,6 +339,10 @@ Server::setResource('pools', function (Registry $register) { return $register->get('pools'); }, ['register']); +Server::setResource('deviceForCsvImports', function (Document $project) { + return getDevice(APP_STORAGE_CSV_IMPORTS . '/app-' . $project->getId()); +}, ['project']); + Server::setResource('deviceForFunctions', function (Document $project) { return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); }, ['project']); diff --git a/composer.json b/composer.json index 3920351c06..4c26b19d1e 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.16.*", - "utopia-php/migration": "0.8.*", + "utopia-php/migration": "dev-feat-csv", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.8.*", @@ -91,6 +91,12 @@ "laravel/pint": "1.*", "phpbench/phpbench": "1.*" }, + "repositories": [ + { + "type": "git", + "url": "https://github.com/utopia-php/migration" + } + ], "provide": { "ext-phpiredis": "*" }, diff --git a/composer.lock b/composer.lock index 8db4706bb5..63323ad236 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6a54c8bc4f9f14cd3883f55880864630", + "content-hash": "cfb5c437126bf194a6fe7225961c1582", "packages": [ { "name": "adhocore/jwt", @@ -1365,16 +1365,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.2.2", + "version": "1.2.3", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "37eec0fe47ddd627911f318f29b6cd48196be0c0" + "reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/37eec0fe47ddd627911f318f29b6cd48196be0c0", - "reference": "37eec0fe47ddd627911f318f29b6cd48196be0c0", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/0e7804c176c4b09d95b7985400aa38ce544cb7fc", + "reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc", "shasum": "" }, "require": { @@ -1451,7 +1451,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-29T21:40:28+00:00" + "time": "2025-04-08T09:55:41+00:00" }, { "name": "open-telemetry/sem-conv", @@ -3951,17 +3951,11 @@ }, { "name": "utopia-php/migration", - "version": "0.8.4", + "version": "dev-feat-csv", "source": { "type": "git", - "url": "https://github.com/utopia-php/migration.git", - "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/845fd04ccf5e0edb03c184b864e0596080a432b8", - "reference": "845fd04ccf5e0edb03c184b864e0596080a432b8", - "shasum": "" + "url": "https://github.com/utopia-php/migration", + "reference": "5c4e6c61c393d176348e88b65730a14b52f4ea2e" }, "require": { "appwrite/appwrite": "11.*", @@ -3987,7 +3981,25 @@ "Utopia\\Migration\\": "src/Migration" } }, - "notification-url": "https://packagist.org/downloads/", + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/Migration" + } + }, + "scripts": { + "test": [ + "./vendor/bin/phpunit" + ], + "lint": [ + "./vendor/bin/pint --test" + ], + "format": [ + "./vendor/bin/pint" + ], + "check": [ + "./vendor/bin/phpstan analyse --level 3 src tests --memory-limit 2G" + ] + }, "license": [ "MIT" ], @@ -3999,11 +4011,7 @@ "upf", "utopia" ], - "support": { - "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.4" - }, - "time": "2025-03-28T02:08:22+00:00" + "time": "2025-04-08T08:38:41+00:00" }, { "name": "utopia-php/orchestration", @@ -8126,7 +8134,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/migration": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index 8c8a364f30..4181cc6564 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,6 +72,7 @@ services: - traefik.http.routers.appwrite_api_https.tls=true volumes: - appwrite-uploads:/storage/uploads:rw + - appwrite-csv-imports:/storage/csv-imports:rw - appwrite-cache:/storage/cache:rw - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -204,7 +205,7 @@ services: appwrite-console: <<: *x-logging container_name: appwrite-console - image: appwrite/console:5.2.53 + image: appwrite/console:5.2.56 restart: unless-stopped networks: - appwrite @@ -672,6 +673,8 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests + # for csv import access + - appwrite-csv-imports:/storage/csv-imports:rw depends_on: - mariadb environment: @@ -1132,6 +1135,7 @@ volumes: appwrite-redis: appwrite-cache: appwrite-uploads: + appwrite-csv-imports: appwrite-certificates: appwrite-functions: appwrite-builds: diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 4939dc8143..b670c79822 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,10 +4,12 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Realtime; +use Appwrite\ID; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Conflict; @@ -18,12 +20,14 @@ use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite as SourceAppwrite; +use Utopia\Migration\Sources\Csv; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Storage\Device; use Utopia\System\System; class Migrations extends Action @@ -32,6 +36,8 @@ class Migrations extends Action protected Database $dbForPlatform; + protected Device $deviceForCsvImports; + protected Document $project; /** @@ -57,15 +63,17 @@ class Migrations extends Action ->inject('dbForPlatform') ->inject('logError') ->inject('queueForRealtime') - ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError, $queueForRealtime)); + ->inject('deviceForCsvImports') + ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForCsvImports) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError, $queueForRealtime, $deviceForCsvImports)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForCsvImports): void { $payload = $message->getPayload() ?? []; + $this->deviceForCsvImports = $deviceForCsvImports; if (empty($payload)) { throw new Exception('Missing payload'); @@ -99,6 +107,7 @@ class Migrations extends Action protected function processSource(Document $migration): Source { $source = $migration->getAttribute('source'); + $resourceId = $migration->getAttribute('resourceId'); $credentials = $migration->getAttribute('credentials'); return match ($source) { @@ -128,6 +137,11 @@ class Migrations extends Action $credentials['endpoint'] === 'http://localhost/v1' ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey'], ), + Csv::getName() => new Csv( + $resourceId, + $credentials['path'], + $this->deviceForCsvImports + ), default => throw new \Exception('Invalid source type'), }; } @@ -222,8 +236,23 @@ class Migrations extends Action $projectDocument = $this->dbForPlatform->getDocument('projects', $project->getId()); $tempAPIKey = $this->generateAPIKey($projectDocument); + $importDocument = null; $transfer = $source = $destination = null; + if ($migration->getAttribute('source') === Csv::getName()) { + $fileSize = $migration->getAttribute('credentials', [])['size'] ?? 0; + $importDocument = new Document([ + '$id' => ID::unique(), + 'size' => $fileSize, // uncompressed and decrypted file size + 'startedAt' => DateTime::now(), + 'migrationId' => $migration->getId(), + 'migrationInternalId' => $migration->getInternalId(), + 'resourceId' => $migration->getAttribute('resourceId', ''), + 'resourceType' => $migration->getAttribute('resourceType', ''), + 'errors' => [], + ]); + } + try { if ( $migration->getAttribute('source') === SourceAppwrite::getName() && @@ -337,6 +366,7 @@ class Migrations extends Action } $migration->setAttribute('errors', $errorMessages); + $importDocument?->setAttribute('errors', $errorMessages); } } finally { $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); @@ -379,6 +409,12 @@ class Migrations extends Action $destination?->success(); $source?->success(); } + + if ($migration->getAttribute('source') === Csv::getName()) { + // make and save the import document to database + $importDocument->setAttribute('status', $migration->getAttribute('status', '')); + $this->dbForProject->createDocument('imports', $importDocument); + } } } } From a53eb6c7500c305588f7dbb504e9b6698355edd9 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 9 Apr 2025 18:24:59 +0530 Subject: [PATCH 262/275] update: bump migrations and update params. --- composer.lock | 4 ++-- src/Appwrite/Platform/Workers/Migrations.php | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/composer.lock b/composer.lock index 63323ad236..9d9f0a78c7 100644 --- a/composer.lock +++ b/composer.lock @@ -3955,7 +3955,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/migration", - "reference": "5c4e6c61c393d176348e88b65730a14b52f4ea2e" + "reference": "b308d9183f1f8ab32e172f31744ad93db319cba9" }, "require": { "appwrite/appwrite": "11.*", @@ -4011,7 +4011,7 @@ "upf", "utopia" ], - "time": "2025-04-08T08:38:41+00:00" + "time": "2025-04-09T12:54:03+00:00" }, { "name": "utopia-php/orchestration", diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index b670c79822..ac13d54959 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -140,7 +140,8 @@ class Migrations extends Action Csv::getName() => new Csv( $resourceId, $credentials['path'], - $this->deviceForCsvImports + $this->deviceForCsvImports, + $this->dbForProject ), default => throw new \Exception('Invalid source type'), }; From a64b8e57af222e7178ffbd8b50e7b5715912b8af Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 12 Apr 2025 17:10:06 +0530 Subject: [PATCH 263/275] address comments. --- app/config/collections/projects.php | 4 ++-- app/controllers/api/migrations.php | 21 ++------------------- composer.lock | 4 ++-- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index d28b2a7e38..1fa474ff44 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1953,8 +1953,8 @@ return [ '$id' => ID::custom('size'), 'type' => Database::VAR_INTEGER, 'format' => '', - 'size' => 0, - 'signed' => true, + 'size' => 8, + 'signed' => false, 'required' => false, 'default' => null, 'array' => false, diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 90f0930ff2..fbd9b3eb13 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -9,6 +9,7 @@ use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Database\Validator\CompoundUID; use Appwrite\Utopia\Database\Validator\Queries\Migrations; use Appwrite\Utopia\Response; use Utopia\App; @@ -320,7 +321,7 @@ App::post('/v1/migrations/csv') )) ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') ->param('fileId', '', new UID(), 'File ID.') - ->param('resourceId', null, new Text(75), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') + ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') ->inject('response') ->inject('dbForProject') ->inject('project') @@ -332,24 +333,6 @@ App::post('/v1/migrations/csv') $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); - // Check if migration/import is already in progress! - // if for some reason the worker crashes, the stage will always be `init`, what do we do? - $isInProgress = Authorization::skip(function () use ($dbForProject, $resourceId) { - $exists = $dbForProject->findOne( - 'migrations', - [ - Query::notEqual('stage', 'finished'), - Query::equal('resourceId', [$resourceId]), - ] - ); - - return !$exists->isEmpty(); - }); - - if ($isInProgress || (!$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::MIGRATION_IN_PROGRESS, 'An import is already in progress for this collection.'); - } - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { diff --git a/composer.lock b/composer.lock index 9d9f0a78c7..270ceb6f8b 100644 --- a/composer.lock +++ b/composer.lock @@ -3955,7 +3955,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/migration", - "reference": "b308d9183f1f8ab32e172f31744ad93db319cba9" + "reference": "9847a1387136574c7539872232765c5c4d8064f7" }, "require": { "appwrite/appwrite": "11.*", @@ -4011,7 +4011,7 @@ "upf", "utopia" ], - "time": "2025-04-09T12:54:03+00:00" + "time": "2025-04-12T11:12:09+00:00" }, { "name": "utopia-php/orchestration", From 2ad967043c7b17fd2cbba0e1e94585dba0761220 Mon Sep 17 00:00:00 2001 From: Darshan Date: Sat, 12 Apr 2025 17:10:29 +0530 Subject: [PATCH 264/275] update: deps. --- composer.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.lock b/composer.lock index 270ceb6f8b..5369b6da5a 100644 --- a/composer.lock +++ b/composer.lock @@ -3955,7 +3955,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/migration", - "reference": "9847a1387136574c7539872232765c5c4d8064f7" + "reference": "52fb030dfb988233dee96845b9c481542fe1a360" }, "require": { "appwrite/appwrite": "11.*", @@ -4011,7 +4011,7 @@ "upf", "utopia" ], - "time": "2025-04-12T11:12:09+00:00" + "time": "2025-04-12T11:28:18+00:00" }, { "name": "utopia-php/orchestration", From 91199e746bba1fa24e7558a4fa8c5f26759355f3 Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 14 Apr 2025 15:03:30 +0530 Subject: [PATCH 265/275] update: use plain csv, use `options`. --- app/config/collections/projects.php | 123 ++----------------- app/controllers/api/migrations.php | 42 ++----- src/Appwrite/Platform/Workers/Migrations.php | 27 +--- 3 files changed, 22 insertions(+), 170 deletions(-) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 1fa474ff44..688153dc29 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1794,6 +1794,17 @@ return [ 'array' => false, 'filters' => ['json', 'encrypt'], ], + [ + '$id' => ID::custom('options'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 65536, + 'signed' => true, + 'required' => false, + 'default' => [], + 'array' => false, + 'filters' => ['json'], + ], [ '$id' => ID::custom('resources'), 'type' => Database::VAR_STRING, @@ -1910,116 +1921,4 @@ return [ ] ], ], - - 'imports' => [ - '$collection' => ID::custom(Database::METADATA), - '$id' => ID::custom('imports'), - 'name' => 'CSV Imports', - 'attributes' => [ - [ - '$id' => ID::custom('migrationId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('migrationInternalId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('status'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('size'), - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 8, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('startedAt'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => ID::custom('resourceId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('resourceType'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('errors'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 65535, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => true, - 'filters' => [], - ], - ], - 'indexes' => [ - [ - '$id' => '_key_status', - 'type' => Database::INDEX_KEY, - 'attributes' => ['status'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => '_key_resourceId', - 'type' => Database::INDEX_KEY, - 'attributes' => ['resourceId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - ], - ] ]; diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index fbd9b3eb13..01c7a260f2 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -4,7 +4,6 @@ use Appwrite\Auth\Auth; use Appwrite\Event\Event; use Appwrite\Event\Migration; use Appwrite\Extend\Exception; -use Appwrite\OpenSSL\OpenSSL; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; @@ -28,11 +27,8 @@ use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; -use Utopia\Storage\Compression\Algorithms\GZIP; -use Utopia\Storage\Compression\Algorithms\Zstd; use Utopia\Storage\Compression\Compression; use Utopia\Storage\Device; -use Utopia\System\System; use Utopia\Validator\ArrayList; use Utopia\Validator\Integer; use Utopia\Validator\Text; @@ -349,37 +345,17 @@ App::post('/v1/migrations/csv') throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); } - // read file content. - $source = $deviceForFiles->read($path); - - // decrypt - if (!empty($file->getAttribute('openSSLCipher'))) { - $source = OpenSSL::decrypt( - $source, - $file->getAttribute('openSSLCipher'), - System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), - 0, - \hex2bin($file->getAttribute('openSSLIV')), - \hex2bin($file->getAttribute('openSSLTag')) - ); - } - - // decompress - switch ($file->getAttribute('algorithm', Compression::NONE)) { - case Compression::ZSTD: - $compressor = new Zstd(); - $source = $compressor->decompress($source); - break; - case Compression::GZIP: - $compressor = new GZIP(); - $source = $compressor->decompress($source); - break; + if (!empty($file->getAttribute('openSSLCipher')) || $file->getAttribute('algorithm', Compression::NONE) !== Compression::NONE) { + throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, "Only uncompressed, unencrypted CSV files can be used for document import."); } // copy to temporary folder $migrationId = ID::unique(); - $path = $deviceForCsvImports->getRoot() . '/' . $migrationId . '_' . $fileId . '.csv'; - $deviceForCsvImports->write($path, $source, 'text/csv'); + $newPath = $deviceForCsvImports->getPath('/' . $migrationId . '_' . $fileId . '.csv'); + if (!$deviceForFiles->transfer($path, $newPath, $deviceForCsvImports)) { + throw new \Exception("Unable to copy file"); + } + $fileSize = $deviceForCsvImports->getFileSize($path); $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); @@ -395,8 +371,8 @@ App::post('/v1/migrations/csv') 'statusCounters' => [], 'resourceData' => [], 'errors' => [], - 'credentials' => [ - 'path' => $path, + 'options' => [ + 'path' => $newPath, 'size' => $fileSize, ], ])); diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index ac13d54959..417a42fa9c 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,12 +4,10 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Realtime; -use Appwrite\ID; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; -use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Conflict; @@ -109,6 +107,7 @@ class Migrations extends Action $source = $migration->getAttribute('source'); $resourceId = $migration->getAttribute('resourceId'); $credentials = $migration->getAttribute('credentials'); + $migrationOptions = $migration->getAttribute('options'); return match ($source) { Firebase::getName() => new Firebase( @@ -139,7 +138,7 @@ class Migrations extends Action ), Csv::getName() => new Csv( $resourceId, - $credentials['path'], + $migrationOptions['path'], $this->deviceForCsvImports, $this->dbForProject ), @@ -237,23 +236,8 @@ class Migrations extends Action $projectDocument = $this->dbForPlatform->getDocument('projects', $project->getId()); $tempAPIKey = $this->generateAPIKey($projectDocument); - $importDocument = null; $transfer = $source = $destination = null; - if ($migration->getAttribute('source') === Csv::getName()) { - $fileSize = $migration->getAttribute('credentials', [])['size'] ?? 0; - $importDocument = new Document([ - '$id' => ID::unique(), - 'size' => $fileSize, // uncompressed and decrypted file size - 'startedAt' => DateTime::now(), - 'migrationId' => $migration->getId(), - 'migrationInternalId' => $migration->getInternalId(), - 'resourceId' => $migration->getAttribute('resourceId', ''), - 'resourceType' => $migration->getAttribute('resourceType', ''), - 'errors' => [], - ]); - } - try { if ( $migration->getAttribute('source') === SourceAppwrite::getName() && @@ -367,7 +351,6 @@ class Migrations extends Action } $migration->setAttribute('errors', $errorMessages); - $importDocument?->setAttribute('errors', $errorMessages); } } finally { $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); @@ -410,12 +393,6 @@ class Migrations extends Action $destination?->success(); $source?->success(); } - - if ($migration->getAttribute('source') === Csv::getName()) { - // make and save the import document to database - $importDocument->setAttribute('status', $migration->getAttribute('status', '')); - $this->dbForProject->createDocument('imports', $importDocument); - } } } } From ab73a70c83584cacec739e446a004aa3f4e94b6d Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 14 Apr 2025 16:30:05 +0530 Subject: [PATCH 266/275] address comments. --- Dockerfile | 4 ++-- app/init/constants.php | 2 +- docker-compose.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4b5ac3fc62..3f85078152 100755 --- a/Dockerfile +++ b/Dockerfile @@ -44,14 +44,14 @@ COPY ./dev /usr/src/code/dev # Set Volumes RUN mkdir -p /storage/uploads && \ - mkdir -p /storage/csv-imports && \ + mkdir -p /storage/imports && \ mkdir -p /storage/cache && \ mkdir -p /storage/config && \ mkdir -p /storage/certificates && \ mkdir -p /storage/functions && \ mkdir -p /storage/debug && \ chown -Rf www-data.www-data /storage/uploads && chmod -Rf 0755 /storage/uploads && \ - chown -Rf www-data.www-data /storage/csv-imports && chmod -Rf 0755 /storage/csv-imports && \ + chown -Rf www-data.www-data /storage/imports && chmod -Rf 0755 /storage/imports && \ chown -Rf www-data.www-data /storage/cache && chmod -Rf 0755 /storage/cache && \ chown -Rf www-data.www-data /storage/config && chmod -Rf 0755 /storage/config && \ chown -Rf www-data.www-data /storage/certificates && chmod -Rf 0755 /storage/certificates && \ diff --git a/app/init/constants.php b/app/init/constants.php index 2deeff1c95..cdfb0b4d03 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -49,7 +49,7 @@ const APP_STORAGE_UPLOADS = '/storage/uploads'; const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_BUILDS = '/storage/builds'; const APP_STORAGE_CACHE = '/storage/cache'; -const APP_STORAGE_CSV_IMPORTS = '/storage/csv-imports'; // Temporary storage for csv imports +const APP_STORAGE_CSV_IMPORTS = '/storage/imports'; // Temporary storage for csv imports const APP_STORAGE_CERTIFICATES = '/storage/certificates'; const APP_STORAGE_CONFIG = '/storage/config'; const APP_STORAGE_READ_BUFFER = 20 * (1000 * 1000); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT` diff --git a/docker-compose.yml b/docker-compose.yml index 4181cc6564..13a4136ca6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,7 +72,7 @@ services: - traefik.http.routers.appwrite_api_https.tls=true volumes: - appwrite-uploads:/storage/uploads:rw - - appwrite-csv-imports:/storage/csv-imports:rw + - appwrite-csv-imports:/storage/imports:rw - appwrite-cache:/storage/cache:rw - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -674,7 +674,7 @@ services: - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests # for csv import access - - appwrite-csv-imports:/storage/csv-imports:rw + - appwrite-csv-imports:/storage/imports:rw depends_on: - mariadb environment: From 73b2a14ea31f4d503d8f045dc1f435907dc510b5 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 16 Apr 2025 11:55:04 +0530 Subject: [PATCH 267/275] update: changes as per migrations lib. --- app/controllers/api/migrations.php | 4 ++-- composer.lock | 4 ++-- src/Appwrite/Platform/Workers/Migrations.php | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 01c7a260f2..7dd8fe42f5 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -22,7 +22,7 @@ use Utopia\Database\Validator\Query\Cursor; use Utopia\Database\Validator\UID; use Utopia\Migration\Resource; use Utopia\Migration\Sources\Appwrite; -use Utopia\Migration\Sources\Csv; +use Utopia\Migration\Sources\CSV; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; @@ -363,7 +363,7 @@ App::post('/v1/migrations/csv') '$id' => $migrationId, 'status' => 'pending', 'stage' => 'init', - 'source' => Csv::getName(), + 'source' => CSV::getName(), 'destination' => Appwrite::class::getName(), 'resources' => $resources, 'resourceId' => $resourceId, diff --git a/composer.lock b/composer.lock index 5369b6da5a..c25ec33591 100644 --- a/composer.lock +++ b/composer.lock @@ -3955,7 +3955,7 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/migration", - "reference": "52fb030dfb988233dee96845b9c481542fe1a360" + "reference": "9088ef1079da3fb13b8abc3821feee618475a0c3" }, "require": { "appwrite/appwrite": "11.*", @@ -4011,7 +4011,7 @@ "upf", "utopia" ], - "time": "2025-04-12T11:28:18+00:00" + "time": "2025-04-16T06:21:15+00:00" }, { "name": "utopia-php/orchestration", diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 417a42fa9c..c4182f8deb 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -18,7 +18,7 @@ use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite as SourceAppwrite; -use Utopia\Migration\Sources\Csv; +use Utopia\Migration\Sources\CSV; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; @@ -136,7 +136,7 @@ class Migrations extends Action $credentials['endpoint'] === 'http://localhost/v1' ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey'], ), - Csv::getName() => new Csv( + CSV::getName() => new CSV( $resourceId, $migrationOptions['path'], $this->deviceForCsvImports, From 9d6cd8249ac5e3b542ab3cfd402c351e70f57441 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 16 Apr 2025 11:56:40 +0530 Subject: [PATCH 268/275] change: constant name. --- app/init/constants.php | 2 +- app/init/resources.php | 2 +- app/worker.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/init/constants.php b/app/init/constants.php index cdfb0b4d03..d22d4b3667 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -49,7 +49,7 @@ const APP_STORAGE_UPLOADS = '/storage/uploads'; const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_BUILDS = '/storage/builds'; const APP_STORAGE_CACHE = '/storage/cache'; -const APP_STORAGE_CSV_IMPORTS = '/storage/imports'; // Temporary storage for csv imports +const APP_STORAGE_IMPORTS = '/storage/imports'; // Temporary storage for csv imports const APP_STORAGE_CERTIFICATES = '/storage/certificates'; const APP_STORAGE_CONFIG = '/storage/config'; const APP_STORAGE_READ_BUFFER = 20 * (1000 * 1000); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT` diff --git a/app/init/resources.php b/app/init/resources.php index 6eae238fee..6aceaf6fb4 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -509,7 +509,7 @@ App::setResource('deviceForFiles', function ($project) { }, ['project']); App::setResource('deviceForCsvImports', function (Document $project) { - return getDevice(APP_STORAGE_CSV_IMPORTS . '/app-' . $project->getId()); + return getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId()); }, ['project']); App::setResource('deviceForFunctions', function ($project) { diff --git a/app/worker.php b/app/worker.php index 4e865858a0..cd3b433d44 100644 --- a/app/worker.php +++ b/app/worker.php @@ -340,7 +340,7 @@ Server::setResource('pools', function (Registry $register) { }, ['register']); Server::setResource('deviceForCsvImports', function (Document $project) { - return getDevice(APP_STORAGE_CSV_IMPORTS . '/app-' . $project->getId()); + return getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId()); }, ['project']); Server::setResource('deviceForFunctions', function (Document $project) { From bc8683ab757cf0dc311c809d90b115f1a313628b Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 16 Apr 2025 16:52:23 +0530 Subject: [PATCH 269/275] add: csv import tests! --- composer.json | 8 +- composer.lock | 44 ++-- .../Services/Migrations/MigrationsBase.php | 226 ++++++++++++++++++ tests/resources/documents.csv | 101 ++++++++ 4 files changed, 345 insertions(+), 34 deletions(-) create mode 100644 tests/resources/documents.csv diff --git a/composer.json b/composer.json index 4c26b19d1e..d0f3803210 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.16.*", - "utopia-php/migration": "dev-feat-csv", + "utopia-php/migration": "0.9.0", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.8.*", @@ -91,12 +91,6 @@ "laravel/pint": "1.*", "phpbench/phpbench": "1.*" }, - "repositories": [ - { - "type": "git", - "url": "https://github.com/utopia-php/migration" - } - ], "provide": { "ext-phpiredis": "*" }, diff --git a/composer.lock b/composer.lock index c25ec33591..46353a4a11 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cfb5c437126bf194a6fe7225961c1582", + "content-hash": "85afadfc660334537aaba2c355f98b9c", "packages": [ { "name": "adhocore/jwt", @@ -3951,11 +3951,17 @@ }, { "name": "utopia-php/migration", - "version": "dev-feat-csv", + "version": "0.9.0", "source": { "type": "git", - "url": "https://github.com/utopia-php/migration", - "reference": "9088ef1079da3fb13b8abc3821feee618475a0c3" + "url": "https://github.com/utopia-php/migration.git", + "reference": "545705e251b766940d2833893f267975d73abe32" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/545705e251b766940d2833893f267975d73abe32", + "reference": "545705e251b766940d2833893f267975d73abe32", + "shasum": "" }, "require": { "appwrite/appwrite": "11.*", @@ -3981,25 +3987,7 @@ "Utopia\\Migration\\": "src/Migration" } }, - "autoload-dev": { - "psr-4": { - "Utopia\\Tests\\": "tests/Migration" - } - }, - "scripts": { - "test": [ - "./vendor/bin/phpunit" - ], - "lint": [ - "./vendor/bin/pint --test" - ], - "format": [ - "./vendor/bin/pint" - ], - "check": [ - "./vendor/bin/phpstan analyse --level 3 src tests --memory-limit 2G" - ] - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -4011,7 +3999,11 @@ "upf", "utopia" ], - "time": "2025-04-16T06:21:15+00:00" + "support": { + "issues": "https://github.com/utopia-php/migration/issues", + "source": "https://github.com/utopia-php/migration/tree/0.9.0" + }, + "time": "2025-04-16T07:52:53+00:00" }, { "name": "utopia-php/orchestration", @@ -8134,9 +8126,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/migration": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 6c468ee730..f7e7d20e99 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -10,8 +10,10 @@ use Tests\E2E\Services\Functions\FunctionsBase; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; +use Utopia\Database\Query; use Utopia\Migration\Resource; use Utopia\Migration\Sources\Appwrite; +use Utopia\Migration\Sources\CSV; trait MigrationsBase { @@ -896,4 +898,228 @@ trait MigrationsBase 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); } + + /** + * Import documents from a CSV file. + */ + public function testCreateCsvMigration(): array + { + // make a database + $response = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database' + ]); + + $this->assertNotEmpty($response['body']['$id']); + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals('Test Database', $response['body']['name']); + + $databaseId = $response['body']['$id']; + + // make a collection + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'name' => 'Test collection', + 'collectionId' => ID::unique(), + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals($response['body']['name'], 'Test collection'); + + $collectionId = $response['body']['$id']; + + // make attributes + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEquals(202, $response['headers']['status-code']); + $this->assertEquals($response['body']['key'], 'name'); + $this->assertEquals($response['body']['type'], 'string'); + $this->assertEquals($response['body']['size'], 256); + $this->assertEquals($response['body']['required'], true); + + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'age', + 'min' => 18, + 'max' => 65, + 'required' => true, + ]); + + $this->assertEquals(202, $response['headers']['status-code']); + $this->assertEquals($response['body']['key'], 'age'); + $this->assertEquals($response['body']['type'], 'integer'); + $this->assertEquals($response['body']['min'], 18); + $this->assertEquals($response['body']['max'], 65); + $this->assertEquals($response['body']['required'], true); + + // make a bucket, upload a file to it! + // 1. enable compression, encryption + $bucketOne = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket', + 'maximumFileSize' => 2000000, //2MB + 'allowedFileExtensions' => ['csv'], + 'compression' => 'gzip', + 'encryption' => true + ]); + $this->assertEquals(201, $bucketOne['headers']['status-code']); + $this->assertNotEmpty($bucketOne['body']['$id']); + + $bucketOneId = $bucketOne['body']['$id']; + + // 2. no compression and encryption + $bucketTwo = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket 2', + 'maximumFileSize' => 2000000, //2MB + 'allowedFileExtensions' => ['csv'], + 'compression' => 'none', + 'encryption' => false + ]); + + $this->assertNotEmpty($bucketTwo['body']['$id']); + $this->assertEquals(201, $bucketTwo['headers']['status-code']); + + $bucketTwoId = $bucketTwo['body']['$id']; + + $bucketIds = [ + 'compressed' => $bucketOneId, + 'uncompressed' => $bucketTwoId, + ]; + + $fileIds = []; + + foreach ($bucketIds as $label => $bucketId) { + $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'fileId' => ID::unique(), + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/documents.csv'), 'text/csv', 'documents.csv'), + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']['$id']); + $this->assertEquals('documents.csv', $response['body']['name']); + $this->assertEquals('text/csv', $response['body']['mimeType']); + + $fileIds[$label] = $response['body']['$id']; + } + + // compressed, fail. + $compressed = $this->performCsvMigration( + [ + 'fileId' => $fileIds['compressed'], + 'bucketId' => $bucketIds['compressed'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + // fail on compressed, encrypted buckets! + $this->assertEquals(400, $compressed['body']['code']); + $this->assertEquals('storage_file_type_unsupported', $compressed['body']['type']); + $this->assertEquals('Only uncompressed, unencrypted CSV files can be used for document import.', $compressed['body']['message']); + + // no compression, no encryption, pass. + $migration = $this->performCsvMigration( + [ + 'endpoint' => 'http://localhost/v1', + 'fileId' => $fileIds['uncompressed'], + 'bucketId' => $bucketIds['uncompressed'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('pending', $migration['body']['status']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'migrationId' => $migration['body']['$id'], + ]; + } + + /** + * @depends testCreateCsvMigration + */ + public function testImportSuccessful(array $response): void + { + $databaseId = $response['databaseId']; + $collectionId = $response['collectionId']; + $migrationId = $response['migrationId']; + + $documentsCountInCSV = 100; + + // get migration stats + $this->assertEventually(function () use ($migrationId, $databaseId, $collectionId, $documentsCountInCSV) { + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('completed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertArrayHasKey(Resource::TYPE_DOCUMENT, $migration['body']['statusCounters']); + $this->assertEquals($documentsCountInCSV, $migration['body']['statusCounters'][Resource::TYPE_DOCUMENT]['success']); + }, 60000, 500); + + // get documents count + $documents = $this->client->call(Client::METHOD_GET, '/databases/'.$databaseId.'/collections/'.$collectionId.'/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + // there should be only 100! + Query::limit(150)->toString() + ] + ]); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertIsArray($documents['body']['documents']); + $this->assertIsNumeric($documents['body']['total']); + $this->assertEquals($documentsCountInCSV, $documents['body']['total']); + } + + private function performCsvMigration(array $body): array + { + return $this->client->call(Client::METHOD_POST, '/migrations/csv', [ + 'content-type' => 'application/json', + 'x-appwrite-key' => $this->getProject()['apiKey'], + 'x-appwrite-project' => $this->getProject()['$id'], + ], $body); + } } diff --git a/tests/resources/documents.csv b/tests/resources/documents.csv new file mode 100644 index 0000000000..ea1e33b5bd --- /dev/null +++ b/tests/resources/documents.csv @@ -0,0 +1,101 @@ +$id,name,age +hxfcwpcas5xokpwe,Diamond Mendez,56 +gw8nxwf6esn3tfwf,Michael Huff,20 +xb6bxg56lral1qy9,Alyssa Rodriguez,37 +imerjq5j36y3agh2,Barbara Smith,26 +07yq9qdlhmbzmr35,Evelyn Edwards,54 +ksqo631sbhwj5ltg,Tina Richardson,41 +j7zlndgu0gbshp15,Joel Hernandez,49 +mfntvnljrcmf7h6v,Zachary Cooper,59 +5f9b01nziqu2h8ed,Brittany Spears,20 +4vxzbnzraqznk5u8,Holly White,47 +d4ywy3mtphaatbpf,Kimberly Barnes,27 +88odnk6nthyyvbal,Stephen Miller,53 +08oekee3fn7mzaa5,Yvonne Newman,41 +quw55kn9895i5e4v,Carol Kane,38 +nge6bm8ykripei6f,Doris Foster,44 +4k16i33s0xl2ypx9,Joseph Stokes,28 +q0j5rxbgid66snyf,Steve Williams,31 +n1oxun7mqq3p103y,James Carey,29 +0dbvs840jkf8i0ye,Kathryn Henry,38 +5sfaidgs1h87v15v,Christopher Landry,23 +vg3punvfu5khmf41,Jennifer Mcgee,62 +f933qydr9u5b2r11,Cathy Church,35 +wjv87y1inf8yk32s,Jose Lopez,41 +uljysdvdlcyrbrwk,William Rose,30 +ot8xtzh77j55wq0s,Sarah Ford,26 +9t76vnsv2u36s43t,Alisha Jones,61 +66y4tnty62hw8c02,Kristin Kelly,61 +2punfblazi5v16ar,Brendan Stout,40 +sxhr4nf5w2gx4wbg,Kelly Cruz,18 +68dvrqfwqnkq5el9,Samantha Martin,50 +20192l6dbeinhkh0,David Santos,46 +si0l4dgay09ebfmf,Elizabeth Carroll,22 +lhse40vbldqb6ap1,Corey Owens,46 +h5t3pslykyx3kxfm,Shelby Mueller,65 +ldc0luydrw6jub0f,Dr. Sylvia Myers,29 +voc9628xg4dsgw2y,Scott Freeman,48 +o4y0gk3gqv1ax2fz,Christopher Atkinson,21 +u1n3x4e4u7e0vzj6,Sean Diaz,31 +s36eskwtm0w7lwr7,Bobby Dyer,57 +4hjnag1p5iwvtixd,Daniel Hall,62 +m91d80oxsa216zbh,Jennifer Ramirez,65 +5hj6858zo2g85n6v,Angela Jackson,57 +8m8oihv9a1e7nn92,Kelly Lewis,36 +7azy39la0no0mxi7,Jessica Munoz,55 +47pmjkhnnqhyit8c,Kelly George,65 +6j6cpy4kgneg1mmh,Anthony Johnson,65 +tnlmtvap1zz89km9,Regina Fields,61 +6cyuvnwwqdmrpfzh,Sharon Schaefer,30 +p1v4pyu2pqodc0ey,Jacob French,62 +6npynnhjt2jd05xo,Jessica Costa,23 +wcxedf13n2e9qi4l,George Hardy,53 +yf2xlcmszk2tqeig,Andrea Allison,20 +3bf2zzv7poststwa,Kevin Ferguson,32 +c2iataz0hhv39q63,Joseph Johnson,58 +3e8npxhov4a39pvq,Ashley Martinez,18 +t7dp41tysipytywq,Charles Nixon,23 +z8cztq7c47phyfhk,Carol Dudley,40 +2636f9d8r4ipm3h6,David Weber,51 +eh3f6wxtvkjq6ykq,Scott Robinson,32 +raskbwpsje69a59h,Anthony Hardy,38 +90hn1p0b4cs9e2og,Mackenzie Owens,52 +am3swwfbo076x0v1,Brian Foster,27 +5uw7utb9lq5cfncw,Hannah Forbes,56 +cs6mbfzkzifefx6r,Lauren Reed,26 +ftw3uvztziiz9x00,Morgan Smith,28 +uhrqseeo43mozpaq,Samantha Alexander,65 +pvvmzyfc1lxor11e,Tiffany Roberts,20 +jia7bdag4abz123s,Emily Hayes,34 +h6oozcngbz8o5x4y,Rebecca Villegas,52 +9v6z1pn2f9twcy12,Donald Shah,61 +wzz3jduioso77o7f,Denise Cain,59 +u51plhgvjodkswnr,Kristine Ramirez,53 +t1uhkmiytfyc13vc,Stacey Adkins,61 +iqaqnf0ybg2ct507,Daniel Hunt,20 +idwrwv2uu4hcpv2i,Roberta Johnson,48 +2yd2hd6auetjacyo,Jason Williamson,39 +egrmdbibnjhi914x,Sandra Robinson,50 +15m1pz2bb0ercgyk,Steve Rice,25 +0i21bhkxdagjurb7,Kimberly Fritz,53 +726ofi7h5snreq67,Brianna Reynolds,33 +csqxse3wym56eim6,Alexander Williams,50 +qeaoylnrsf8p3byg,Andrew Thomas,25 +edsswobumzyzbvhf,Austin Williams,57 +hdzhzpt0ahy5hkib,Nicholas Williams,24 +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros,48 +3z3o73x7adyuo6w0,Stacey Smith,39 +sse2u5zlgoqrgmcf,Laura Beck,20 +rvovijmvch58r4yx,Molly Clark,51 +doe06nrx8sg5mcuv,Carmen Morris,41 +jbjdwuvj5s4kw04y,Amanda Munoz,20 +6k2ewkla7js0yw23,Rachel Collins,44 +fcxuyr4kkhrnigu1,John Alexander,18 +d25fuwlos5mk07o0,Stacy Hunter,22 +1vdai2rxmwd57oet,Eric Massey,40 +pq4jnt9izu1wlrzd,Scott Garcia,20 +lz9kfc0lty5xcz14,Cassandra Nelson,35 +pu7w6tyab5jd4we9,Aaron Johnson,50 +8dupswd2kqwdyn8v,Shannon Sherman,45 +ye466l71jthiz2p6,April Garcia,60 +xogsmfwb73l16qdt,Evan Lynn,20 From cc7d038c3efae8b213649a0c109f3ff29856819b Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 17 Apr 2025 08:54:50 +0530 Subject: [PATCH 270/275] address comments: change `deviceForCsvImports` to `deviceForImports`. --- app/controllers/api/migrations.php | 10 +++++----- app/init/resources.php | 2 +- app/worker.php | 2 +- src/Appwrite/Platform/Workers/Migrations.php | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 7dd8fe42f5..2c4090bed1 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -322,10 +322,10 @@ App::post('/v1/migrations/csv') ->inject('dbForProject') ->inject('project') ->inject('deviceForFiles') - ->inject('deviceForCsvImports') + ->inject('deviceForImports') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForCsvImports, Event $queueForEvents, Migration $queueForMigrations) { + ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForImports, Event $queueForEvents, Migration $queueForMigrations) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -351,12 +351,12 @@ App::post('/v1/migrations/csv') // copy to temporary folder $migrationId = ID::unique(); - $newPath = $deviceForCsvImports->getPath('/' . $migrationId . '_' . $fileId . '.csv'); - if (!$deviceForFiles->transfer($path, $newPath, $deviceForCsvImports)) { + $newPath = $deviceForImports->getPath('/' . $migrationId . '_' . $fileId . '.csv'); + if (!$deviceForFiles->transfer($path, $newPath, $deviceForImports)) { throw new \Exception("Unable to copy file"); } - $fileSize = $deviceForCsvImports->getFileSize($path); + $fileSize = $deviceForImports->getFileSize($path); $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); $migration = $dbForProject->createDocument('migrations', new Document([ diff --git a/app/init/resources.php b/app/init/resources.php index 6aceaf6fb4..ed124be1ac 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -508,7 +508,7 @@ App::setResource('deviceForFiles', function ($project) { return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); }, ['project']); -App::setResource('deviceForCsvImports', function (Document $project) { +App::setResource('deviceForImports', function (Document $project) { return getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId()); }, ['project']); diff --git a/app/worker.php b/app/worker.php index cd3b433d44..6a62fb7e7d 100644 --- a/app/worker.php +++ b/app/worker.php @@ -339,7 +339,7 @@ Server::setResource('pools', function (Registry $register) { return $register->get('pools'); }, ['register']); -Server::setResource('deviceForCsvImports', function (Document $project) { +Server::setResource('deviceForImports', function (Document $project) { return getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId()); }, ['project']); diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index c4182f8deb..323924bc86 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -34,7 +34,7 @@ class Migrations extends Action protected Database $dbForPlatform; - protected Device $deviceForCsvImports; + protected Device $deviceForImports; protected Document $project; @@ -61,17 +61,17 @@ class Migrations extends Action ->inject('dbForPlatform') ->inject('logError') ->inject('queueForRealtime') - ->inject('deviceForCsvImports') - ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForCsvImports) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError, $queueForRealtime, $deviceForCsvImports)); + ->inject('deviceForImports') + ->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError, $queueForRealtime, $deviceForImports)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForCsvImports): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports): void { $payload = $message->getPayload() ?? []; - $this->deviceForCsvImports = $deviceForCsvImports; + $this->deviceForImports = $deviceForImports; if (empty($payload)) { throw new Exception('Missing payload'); @@ -139,7 +139,7 @@ class Migrations extends Action CSV::getName() => new CSV( $resourceId, $migrationOptions['path'], - $this->deviceForCsvImports, + $this->deviceForImports, $this->dbForProject ), default => throw new \Exception('Invalid source type'), From 680b214950e804c188fc49f550fe7a48e13fc97f Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 17 Apr 2025 08:59:30 +0530 Subject: [PATCH 271/275] update: storage volume name. --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 13a4136ca6..6de5ad4cd0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,7 +72,7 @@ services: - traefik.http.routers.appwrite_api_https.tls=true volumes: - appwrite-uploads:/storage/uploads:rw - - appwrite-csv-imports:/storage/imports:rw + - appwrite-imports:/storage/imports:rw - appwrite-cache:/storage/cache:rw - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -674,7 +674,7 @@ services: - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests # for csv import access - - appwrite-csv-imports:/storage/imports:rw + - appwrite-imports:/storage/imports:rw depends_on: - mariadb environment: @@ -1135,7 +1135,7 @@ volumes: appwrite-redis: appwrite-cache: appwrite-uploads: - appwrite-csv-imports: + appwrite-imports: appwrite-certificates: appwrite-functions: appwrite-builds: From 36ab6ce236cb5ad0f5d4f470816591adfcf4114b Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 17 Apr 2025 10:36:05 +0530 Subject: [PATCH 272/275] address comments: add more tests. --- .../Services/Migrations/MigrationsBase.php | 110 +++++++++++++++++- tests/resources/{ => csv}/documents.csv | 0 tests/resources/csv/irrelevant-column.csv | 101 ++++++++++++++++ tests/resources/csv/missing-column.csv | 101 ++++++++++++++++ tests/resources/csv/missing-row.csv | 101 ++++++++++++++++ 5 files changed, 410 insertions(+), 3 deletions(-) rename tests/resources/{ => csv}/documents.csv (100%) create mode 100644 tests/resources/csv/irrelevant-column.csv create mode 100644 tests/resources/csv/missing-column.csv create mode 100644 tests/resources/csv/missing-row.csv diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index f7e7d20e99..45b57d6b0c 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1011,23 +1011,40 @@ trait MigrationsBase $bucketIds = [ 'compressed' => $bucketOneId, 'uncompressed' => $bucketTwoId, + + // in uncompressed buckets! + 'missing-row' => $bucketTwoId, + 'missing-column' => $bucketTwoId, + 'irrelevant-column' => $bucketTwoId, ]; $fileIds = []; foreach ($bucketIds as $label => $bucketId) { + $csvFileName = match ($label) { + 'missing-row', + 'missing-column', + 'irrelevant-column' => "{$label}.csv", + default => 'documents.csv', + }; + + $mimeType = match ($csvFileName) { + default => 'text/csv', + 'missing-row.csv' => 'text/plain', // invalid csv structure, falls back to plain text! + }; + $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ 'content-type' => 'multipart/form-data', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'fileId' => ID::unique(), - 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/documents.csv'), 'text/csv', 'documents.csv'), + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/csv/'.$csvFileName), $mimeType, $csvFileName), ]); $this->assertEquals(201, $response['headers']['status-code']); $this->assertNotEmpty($response['body']['$id']); - $this->assertEquals('documents.csv', $response['body']['name']); - $this->assertEquals('text/csv', $response['body']['mimeType']); + $this->assertEquals($csvFileName, $response['body']['name']); + $this->assertEquals($mimeType, $response['body']['mimeType']); $fileIds[$label] = $response['body']['$id']; } @@ -1046,6 +1063,93 @@ trait MigrationsBase $this->assertEquals('storage_file_type_unsupported', $compressed['body']['type']); $this->assertEquals('Only uncompressed, unencrypted CSV files can be used for document import.', $compressed['body']['message']); + // missing attribute, fail in worker. + $missingColumn = $this->performCsvMigration( + [ + 'fileId' => $fileIds['missing-column'], + 'bucketId' => $bucketIds['missing-column'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEventually(function () use ($missingColumn, $databaseId, $collectionId) { + $migrationId = $missingColumn['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('failed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertThat( + implode("\n", $migration['body']['errors']), + $this->stringContains("CSV header mismatch. Missing attribute: 'age'") + ); + }, 60000, 500); + + // missing row data, fail in worker. + $missingColumn = $this->performCsvMigration( + [ + 'fileId' => $fileIds['missing-row'], + 'bucketId' => $bucketIds['missing-row'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEventually(function () use ($missingColumn, $databaseId, $collectionId) { + $migrationId = $missingColumn['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('failed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertThat( + implode("\n", $migration['body']['errors']), + $this->stringContains('CSV row does not match the number of header columns') + ); + }, 60000, 500); + + // irrelevant column - email, fail in worker. + $irrelevantColumn = $this->performCsvMigration( + [ + 'fileId' => $fileIds['irrelevant-column'], + 'bucketId' => $bucketIds['irrelevant-column'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEventually(function () use ($irrelevantColumn, $databaseId, $collectionId) { + $migrationId = $irrelevantColumn['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('failed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertThat( + implode("\n", $migration['body']['errors']), + $this->stringContains("CSV header mismatch. Unexpected attribute: 'email'") + ); + }, 60000, 500); + // no compression, no encryption, pass. $migration = $this->performCsvMigration( [ diff --git a/tests/resources/documents.csv b/tests/resources/csv/documents.csv similarity index 100% rename from tests/resources/documents.csv rename to tests/resources/csv/documents.csv diff --git a/tests/resources/csv/irrelevant-column.csv b/tests/resources/csv/irrelevant-column.csv new file mode 100644 index 0000000000..92105ceaa2 --- /dev/null +++ b/tests/resources/csv/irrelevant-column.csv @@ -0,0 +1,101 @@ +$id,name,age,email +hxfcwpcas5xokpwe,Diamond Mendez,56,diamond.mendez@example.com +gw8nxwf6esn3tfwf,Michael Huff,20,michael.huff@example.com +xb6bxg56lral1qy9,Alyssa Rodriguez,37,alyssa.rodriguez@example.com +imerjq5j36y3agh2,Barbara Smith,26,barbara.smith@example.com +07yq9qdlhmbzmr35,Evelyn Edwards,54,evelyn.edwards@example.com +ksqo631sbhwj5ltg,Tina Richardson,41,tina.richardson@example.com +j7zlndgu0gbshp15,Joel Hernandez,49,joel.hernandez@example.com +mfntvnljrcmf7h6v,Zachary Cooper,59,zachary.cooper@example.com +5f9b01nziqu2h8ed,Brittany Spears,20,brittany.spears@example.com +4vxzbnzraqznk5u8,Holly White,47,holly.white@example.com +d4ywy3mtphaatbpf,Kimberly Barnes,27,kimberly.barnes@example.com +88odnk6nthyyvbal,Stephen Miller,53,stephen.miller@example.com +08oekee3fn7mzaa5,Yvonne Newman,41,yvonne.newman@example.com +quw55kn9895i5e4v,Carol Kane,38,carol.kane@example.com +nge6bm8ykripei6f,Doris Foster,44,doris.foster@example.com +4k16i33s0xl2ypx9,Joseph Stokes,28,joseph.stokes@example.com +q0j5rxbgid66snyf,Steve Williams,31,steve.williams@example.com +n1oxun7mqq3p103y,James Carey,29,james.carey@example.com +0dbvs840jkf8i0ye,Kathryn Henry,38,kathryn.henry@example.com +5sfaidgs1h87v15v,Christopher Landry,23,christopher.landry@example.com +vg3punvfu5khmf41,Jennifer Mcgee,62,jennifer.mcgee@example.com +f933qydr9u5b2r11,Cathy Church,35,cathy.church@example.com +wjv87y1inf8yk32s,Jose Lopez,41,jose.lopez@example.com +uljysdvdlcyrbrwk,William Rose,30,william.rose@example.com +ot8xtzh77j55wq0s,Sarah Ford,26,sarah.ford@example.com +9t76vnsv2u36s43t,Alisha Jones,61,alisha.jones@example.com +66y4tnty62hw8c02,Kristin Kelly,61,kristin.kelly@example.com +2punfblazi5v16ar,Brendan Stout,40,brendan.stout@example.com +sxhr4nf5w2gx4wbg,Kelly Cruz,18,kelly.cruz@example.com +68dvrqfwqnkq5el9,Samantha Martin,50,samantha.martin@example.com +20192l6dbeinhkh0,David Santos,46,david.santos@example.com +si0l4dgay09ebfmf,Elizabeth Carroll,22,elizabeth.carroll@example.com +lhse40vbldqb6ap1,Corey Owens,46,corey.owens@example.com +h5t3pslykyx3kxfm,Shelby Mueller,65,shelby.mueller@example.com +ldc0luydrw6jub0f,Dr. Sylvia Myers,29,sylvia.myers@example.com +voc9628xg4dsgw2y,Scott Freeman,48,scott.freeman@example.com +o4y0gk3gqv1ax2fz,Christopher Atkinson,21,christopher.atkinson@example.com +u1n3x4e4u7e0vzj6,Sean Diaz,31,sean.diaz@example.com +s36eskwtm0w7lwr7,Bobby Dyer,57,bobby.dyer@example.com +4hjnag1p5iwvtixd,Daniel Hall,62,daniel.hall@example.com +m91d80oxsa216zbh,Jennifer Ramirez,65,jennifer.ramirez@example.com +5hj6858zo2g85n6v,Angela Jackson,57,angela.jackson@example.com +8m8oihv9a1e7nn92,Kelly Lewis,36,kelly.lewis@example.com +7azy39la0no0mxi7,Jessica Munoz,55,jessica.munoz@example.com +47pmjkhnnqhyit8c,Kelly George,65,kelly.george@example.com +6j6cpy4kgneg1mmh,Anthony Johnson,65,anthony.johnson@example.com +tnlmtvap1zz89km9,Regina Fields,61,regina.fields@example.com +6cyuvnwwqdmrpfzh,Sharon Schaefer,30,sharon.schaefer@example.com +p1v4pyu2pqodc0ey,Jacob French,62,jacob.french@example.com +6npynnhjt2jd05xo,Jessica Costa,23,jessica.costa@example.com +wcxedf13n2e9qi4l,George Hardy,53,george.hardy@example.com +yf2xlcmszk2tqeig,Andrea Allison,20,andrea.allison@example.com +3bf2zzv7poststwa,Kevin Ferguson,32,kevin.ferguson@example.com +c2iataz0hhv39q63,Joseph Johnson,58,joseph.johnson@example.com +3e8npxhov4a39pvq,Ashley Martinez,18,ashley.martinez@example.com +t7dp41tysipytywq,Charles Nixon,23,charles.nixon@example.com +z8cztq7c47phyfhk,Carol Dudley,40,carol.dudley@example.com +2636f9d8r4ipm3h6,David Weber,51,david.weber@example.com +eh3f6wxtvkjq6ykq,Scott Robinson,32,scott.robinson@example.com +raskbwpsje69a59h,Anthony Hardy,38,anthony.hardy@example.com +90hn1p0b4cs9e2og,Mackenzie Owens,52,mackenzie.owens@example.com +am3swwfbo076x0v1,Brian Foster,27,brian.foster@example.com +5uw7utb9lq5cfncw,Hannah Forbes,56,hannah.forbes@example.com +cs6mbfzkzifefx6r,Lauren Reed,26,lauren.reed@example.com +ftw3uvztziiz9x00,Morgan Smith,28,morgan.smith@example.com +uhrqseeo43mozpaq,Samantha Alexander,65,samantha.alexander@example.com +pvvmzyfc1lxor11e,Tiffany Roberts,20,tiffany.roberts@example.com +jia7bdag4abz123s,Emily Hayes,34,emily.hayes@example.com +h6oozcngbz8o5x4y,Rebecca Villegas,52,rebecca.villegas@example.com +9v6z1pn2f9twcy12,Donald Shah,61,donald.shah@example.com +wzz3jduioso77o7f,Denise Cain,59,denise.cain@example.com +u51plhgvjodkswnr,Kristine Ramirez,53,kristine.ramirez@example.com +t1uhkmiytfyc13vc,Stacey Adkins,61,stacey.adkins@example.com +iqaqnf0ybg2ct507,Daniel Hunt,20,daniel.hunt@example.com +idwrwv2uu4hcpv2i,Roberta Johnson,48,roberta.johnson@example.com +2yd2hd6auetjacyo,Jason Williamson,39,jason.williamson@example.com +egrmdbibnjhi914x,Sandra Robinson,50,sandra.robinson@example.com +15m1pz2bb0ercgyk,Steve Rice,25,steve.rice@example.com +0i21bhkxdagjurb7,Kimberly Fritz,53,kimberly.fritz@example.com +726ofi7h5snreq67,Brianna Reynolds,33,brianna.reynolds@example.com +csqxse3wym56eim6,Alexander Williams,50,alexander.williams@example.com +qeaoylnrsf8p3byg,Andrew Thomas,25,andrew.thomas@example.com +edsswobumzyzbvhf,Austin Williams,57,austin.williams@example.com +hdzhzpt0ahy5hkib,Nicholas Williams,24,nicholas.williams@example.com +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros,48,michelle.cisneros@example.com +3z3o73x7adyuo6w0,Stacey Smith,39,stacey.smith@example.com +sse2u5zlgoqrgmcf,Laura Beck,20,laura.beck@example.com +rvovijmvch58r4yx,Molly Clark,51,molly.clark@example.com +doe06nrx8sg5mcuv,Carmen Morris,41,carmen.morris@example.com +jbjdwuvj5s4kw04y,Amanda Munoz,20,amanda.munoz@example.com +6k2ewkla7js0yw23,Rachel Collins,44,rachel.collins@example.com +fcxuyr4kkhrnigu1,John Alexander,18,john.alexander@example.com +d25fuwlos5mk07o0,Stacy Hunter,22,stacy.hunter@example.com +1vdai2rxmwd57oet,Eric Massey,40,eric.massey@example.com +pq4jnt9izu1wlrzd,Scott Garcia,20,scott.garcia@example.com +lz9kfc0lty5xcz14,Cassandra Nelson,35,cassandra.nelson@example.com +pu7w6tyab5jd4we9,Aaron Johnson,50,aaron.johnson@example.com +8dupswd2kqwdyn8v,Shannon Sherman,45,shannon.sherman@example.com +ye466l71jthiz2p6,April Garcia,60,april.garcia@example.com +xogsmfwb73l16qdt,Evan Lynn,20,evan.lynn@example.com diff --git a/tests/resources/csv/missing-column.csv b/tests/resources/csv/missing-column.csv new file mode 100644 index 0000000000..e57b5ccb2e --- /dev/null +++ b/tests/resources/csv/missing-column.csv @@ -0,0 +1,101 @@ +$id,name +hxfcwpcas5xokpwe,Diamond Mendez +gw8nxwf6esn3tfwf,Michael Huff +xb6bxg56lral1qy9,Alyssa Rodriguez +imerjq5j36y3agh2,Barbara Smith +07yq9qdlhmbzmr35,Evelyn Edwards +ksqo631sbhwj5ltg,Tina Richardson +j7zlndgu0gbshp15,Joel Hernandez +mfntvnljrcmf7h6v,Zachary Cooper +5f9b01nziqu2h8ed,Brittany Spears +4vxzbnzraqznk5u8,Holly White +d4ywy3mtphaatbpf,Kimberly Barnes +88odnk6nthyyvbal,Stephen Miller +08oekee3fn7mzaa5,Yvonne Newman +quw55kn9895i5e4v,Carol Kane +nge6bm8ykripei6f,Doris Foster +4k16i33s0xl2ypx9,Joseph Stokes +q0j5rxbgid66snyf,Steve Williams +n1oxun7mqq3p103y,James Carey +0dbvs840jkf8i0ye,Kathryn Henry +5sfaidgs1h87v15v,Christopher Landry +vg3punvfu5khmf41,Jennifer Mcgee +f933qydr9u5b2r11,Cathy Church +wjv87y1inf8yk32s,Jose Lopez +uljysdvdlcyrbrwk,William Rose +ot8xtzh77j55wq0s,Sarah Ford +9t76vnsv2u36s43t,Alisha Jones +66y4tnty62hw8c02,Kristin Kelly +2punfblazi5v16ar,Brendan Stout +sxhr4nf5w2gx4wbg,Kelly Cruz +68dvrqfwqnkq5el9,Samantha Martin +20192l6dbeinhkh0,David Santos +si0l4dgay09ebfmf,Elizabeth Carroll +lhse40vbldqb6ap1,Corey Owens +h5t3pslykyx3kxfm,Shelby Mueller +ldc0luydrw6jub0f,Dr. Sylvia Myers +voc9628xg4dsgw2y,Scott Freeman +o4y0gk3gqv1ax2fz,Christopher Atkinson +u1n3x4e4u7e0vzj6,Sean Diaz +s36eskwtm0w7lwr7,Bobby Dyer +4hjnag1p5iwvtixd,Daniel Hall +m91d80oxsa216zbh,Jennifer Ramirez +5hj6858zo2g85n6v,Angela Jackson +8m8oihv9a1e7nn92,Kelly Lewis +7azy39la0no0mxi7,Jessica Munoz +47pmjkhnnqhyit8c,Kelly George +6j6cpy4kgneg1mmh,Anthony Johnson +tnlmtvap1zz89km9,Regina Fields +6cyuvnwwqdmrpfzh,Sharon Schaefer +p1v4pyu2pqodc0ey,Jacob French +6npynnhjt2jd05xo,Jessica Costa +wcxedf13n2e9qi4l,George Hardy +yf2xlcmszk2tqeig,Andrea Allison +3bf2zzv7poststwa,Kevin Ferguson +c2iataz0hhv39q63,Joseph Johnson +3e8npxhov4a39pvq,Ashley Martinez +t7dp41tysipytywq,Charles Nixon +z8cztq7c47phyfhk,Carol Dudley +2636f9d8r4ipm3h6,David Weber +eh3f6wxtvkjq6ykq,Scott Robinson +raskbwpsje69a59h,Anthony Hardy +90hn1p0b4cs9e2og,Mackenzie Owens +am3swwfbo076x0v1,Brian Foster +5uw7utb9lq5cfncw,Hannah Forbes +cs6mbfzkzifefx6r,Lauren Reed +ftw3uvztziiz9x00,Morgan Smith +uhrqseeo43mozpaq,Samantha Alexander +pvvmzyfc1lxor11e,Tiffany Roberts +jia7bdag4abz123s,Emily Hayes +h6oozcngbz8o5x4y,Rebecca Villegas +9v6z1pn2f9twcy12,Donald Shah +wzz3jduioso77o7f,Denise Cain +u51plhgvjodkswnr,Kristine Ramirez +t1uhkmiytfyc13vc,Stacey Adkins +iqaqnf0ybg2ct507,Daniel Hunt +idwrwv2uu4hcpv2i,Roberta Johnson +2yd2hd6auetjacyo,Jason Williamson +egrmdbibnjhi914x,Sandra Robinson +15m1pz2bb0ercgyk,Steve Rice +0i21bhkxdagjurb7,Kimberly Fritz +726ofi7h5snreq67,Brianna Reynolds +csqxse3wym56eim6,Alexander Williams +qeaoylnrsf8p3byg,Andrew Thomas +edsswobumzyzbvhf,Austin Williams +hdzhzpt0ahy5hkib,Nicholas Williams +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros +3z3o73x7adyuo6w0,Stacey Smith +sse2u5zlgoqrgmcf,Laura Beck +rvovijmvch58r4yx,Molly Clark +doe06nrx8sg5mcuv,Carmen Morris +jbjdwuvj5s4kw04y,Amanda Munoz +6k2ewkla7js0yw23,Rachel Collins +fcxuyr4kkhrnigu1,John Alexander +d25fuwlos5mk07o0,Stacy Hunter +1vdai2rxmwd57oet,Eric Massey +pq4jnt9izu1wlrzd,Scott Garcia +lz9kfc0lty5xcz14,Cassandra Nelson +pu7w6tyab5jd4we9,Aaron Johnson +8dupswd2kqwdyn8v,Shannon Sherman +ye466l71jthiz2p6,April Garcia +xogsmfwb73l16qdt,Evan Lynn diff --git a/tests/resources/csv/missing-row.csv b/tests/resources/csv/missing-row.csv new file mode 100644 index 0000000000..7399fa9f51 --- /dev/null +++ b/tests/resources/csv/missing-row.csv @@ -0,0 +1,101 @@ +$id,name,age +hxfcwpcas5xokpwe,Diamond Mendez +gw8nxwf6esn3tfwf,Michael Huff +xb6bxg56lral1qy9,Alyssa Rodriguez +imerjq5j36y3agh2,Barbara Smith +07yq9qdlhmbzmr35,Evelyn Edwards +ksqo631sbhwj5ltg,Tina Richardson +j7zlndgu0gbshp15,Joel Hernandez +mfntvnljrcmf7h6v,Zachary Cooper +5f9b01nziqu2h8ed,Brittany Spears +4vxzbnzraqznk5u8,Holly White +d4ywy3mtphaatbpf,Kimberly Barnes +88odnk6nthyyvbal,Stephen Miller +08oekee3fn7mzaa5,Yvonne Newman +quw55kn9895i5e4v,Carol Kane +nge6bm8ykripei6f,Doris Foster +4k16i33s0xl2ypx9,Joseph Stokes +q0j5rxbgid66snyf,Steve Williams +n1oxun7mqq3p103y,James Carey +0dbvs840jkf8i0ye,Kathryn Henry +5sfaidgs1h87v15v,Christopher Landry +vg3punvfu5khmf41,Jennifer Mcgee +f933qydr9u5b2r11,Cathy Church +wjv87y1inf8yk32s,Jose Lopez +uljysdvdlcyrbrwk,William Rose +ot8xtzh77j55wq0s,Sarah Ford +9t76vnsv2u36s43t,Alisha Jones +66y4tnty62hw8c02,Kristin Kelly +2punfblazi5v16ar,Brendan Stout +sxhr4nf5w2gx4wbg,Kelly Cruz +68dvrqfwqnkq5el9,Samantha Martin +20192l6dbeinhkh0,David Santos +si0l4dgay09ebfmf,Elizabeth Carroll +lhse40vbldqb6ap1,Corey Owens +h5t3pslykyx3kxfm,Shelby Mueller +ldc0luydrw6jub0f,Dr. Sylvia Myers +voc9628xg4dsgw2y,Scott Freeman +o4y0gk3gqv1ax2fz,Christopher Atkinson +u1n3x4e4u7e0vzj6,Sean Diaz +s36eskwtm0w7lwr7,Bobby Dyer +4hjnag1p5iwvtixd,Daniel Hall +m91d80oxsa216zbh,Jennifer Ramirez +5hj6858zo2g85n6v,Angela Jackson +8m8oihv9a1e7nn92,Kelly Lewis +7azy39la0no0mxi7,Jessica Munoz +47pmjkhnnqhyit8c,Kelly George +6j6cpy4kgneg1mmh,Anthony Johnson +tnlmtvap1zz89km9,Regina Fields +6cyuvnwwqdmrpfzh,Sharon Schaefer +p1v4pyu2pqodc0ey,Jacob French +6npynnhjt2jd05xo,Jessica Costa +wcxedf13n2e9qi4l,George Hardy +yf2xlcmszk2tqeig,Andrea Allison +3bf2zzv7poststwa,Kevin Ferguson +c2iataz0hhv39q63,Joseph Johnson +3e8npxhov4a39pvq,Ashley Martinez +t7dp41tysipytywq,Charles Nixon +z8cztq7c47phyfhk,Carol Dudley +2636f9d8r4ipm3h6,David Weber +eh3f6wxtvkjq6ykq,Scott Robinson +raskbwpsje69a59h,Anthony Hardy +90hn1p0b4cs9e2og,Mackenzie Owens +am3swwfbo076x0v1,Brian Foster +5uw7utb9lq5cfncw,Hannah Forbes +cs6mbfzkzifefx6r,Lauren Reed +ftw3uvztziiz9x00,Morgan Smith +uhrqseeo43mozpaq,Samantha Alexander +pvvmzyfc1lxor11e,Tiffany Roberts +jia7bdag4abz123s,Emily Hayes +h6oozcngbz8o5x4y,Rebecca Villegas +9v6z1pn2f9twcy12,Donald Shah +wzz3jduioso77o7f,Denise Cain +u51plhgvjodkswnr,Kristine Ramirez +t1uhkmiytfyc13vc,Stacey Adkins +iqaqnf0ybg2ct507,Daniel Hunt +idwrwv2uu4hcpv2i,Roberta Johnson +2yd2hd6auetjacyo,Jason Williamson +egrmdbibnjhi914x,Sandra Robinson +15m1pz2bb0ercgyk,Steve Rice +0i21bhkxdagjurb7,Kimberly Fritz +726ofi7h5snreq67,Brianna Reynolds +csqxse3wym56eim6,Alexander Williams +qeaoylnrsf8p3byg,Andrew Thomas +edsswobumzyzbvhf,Austin Williams +hdzhzpt0ahy5hkib,Nicholas Williams +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros +3z3o73x7adyuo6w0,Stacey Smith +sse2u5zlgoqrgmcf,Laura Beck +rvovijmvch58r4yx,Molly Clark +doe06nrx8sg5mcuv,Carmen Morris +jbjdwuvj5s4kw04y,Amanda Munoz +6k2ewkla7js0yw23,Rachel Collins +fcxuyr4kkhrnigu1,John Alexander +d25fuwlos5mk07o0,Stacy Hunter +1vdai2rxmwd57oet,Eric Massey +pq4jnt9izu1wlrzd,Scott Garcia +lz9kfc0lty5xcz14,Cassandra Nelson +pu7w6tyab5jd4we9,Aaron Johnson +8dupswd2kqwdyn8v,Shannon Sherman +ye466l71jthiz2p6,April Garcia +xogsmfwb73l16qdt,Evan Lynn From fda0af6694a0c62b1ba4bc02434867f91c5ca9ef Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 17 Apr 2025 10:51:30 +0530 Subject: [PATCH 273/275] bump: dependency. --- composer.json | 2 +- composer.lock | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index d0f3803210..39477c83a0 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.16.*", - "utopia-php/migration": "0.9.0", + "utopia-php/migration": "0.9.1", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.8.*", diff --git a/composer.lock b/composer.lock index 46353a4a11..7fa497a8f5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "85afadfc660334537aaba2c355f98b9c", + "content-hash": "e22cfd0e495f55633218f029d8c7ec9d", "packages": [ { "name": "adhocore/jwt", @@ -3951,16 +3951,16 @@ }, { "name": "utopia-php/migration", - "version": "0.9.0", + "version": "0.9.1", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "545705e251b766940d2833893f267975d73abe32" + "reference": "f8b54727c7b0abe416a74a2a4c9fa4350c7a59a3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/545705e251b766940d2833893f267975d73abe32", - "reference": "545705e251b766940d2833893f267975d73abe32", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/f8b54727c7b0abe416a74a2a4c9fa4350c7a59a3", + "reference": "f8b54727c7b0abe416a74a2a4c9fa4350c7a59a3", "shasum": "" }, "require": { @@ -4001,9 +4001,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.9.0" + "source": "https://github.com/utopia-php/migration/tree/0.9.1" }, - "time": "2025-04-16T07:52:53+00:00" + "time": "2025-04-17T05:18:58+00:00" }, { "name": "utopia-php/orchestration", From 767e202b342a2fdca91e08da22029d30e160117a Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 17 Apr 2025 10:54:23 +0530 Subject: [PATCH 274/275] update: name. --- app/controllers/api/migrations.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 2c4090bed1..4a1e5de227 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -364,7 +364,7 @@ App::post('/v1/migrations/csv') 'status' => 'pending', 'stage' => 'init', 'source' => CSV::getName(), - 'destination' => Appwrite::class::getName(), + 'destination' => Appwrite::getName(), 'resources' => $resources, 'resourceId' => $resourceId, 'resourceType' => Resource::TYPE_DATABASE, From 6d3e40d73b1c378598aac4a58ce77afbb54d3e15 Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 17 Apr 2025 11:03:03 +0530 Subject: [PATCH 275/275] address comments: change order, remove comment. --- docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6de5ad4cd0..9bd55becd1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -670,11 +670,10 @@ services: networks: - appwrite volumes: + - appwrite-imports:/storage/imports:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests - # for csv import access - - appwrite-imports:/storage/imports:rw depends_on: - mariadb environment: