From 74ff9a955b4cdd67cb30bfcb70ab1c0c930107fb Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 11 Nov 2025 20:20:08 +1300 Subject: [PATCH 01/16] Fix transactions + operators --- composer.lock | 124 +++--- .../Databases/Http/Databases/Action.php | 71 +++- .../Collections/Documents/Action.php | 54 +-- .../Http/Databases/Transactions/Action.php | 6 +- .../Http/Databases/Transactions/Update.php | 12 + .../Transactions/TransactionsBase.php | 376 ++++++++++++++++++ 6 files changed, 526 insertions(+), 117 deletions(-) diff --git a/composer.lock b/composer.lock index 8faa5a477c..cf92abde7c 100644 --- a/composer.lock +++ b/composer.lock @@ -2673,16 +2673,16 @@ }, { "name": "symfony/http-client", - "version": "v7.3.4", + "version": "v7.3.6", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62" + "reference": "3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/4b62871a01c49457cf2a8e560af7ee8a94b87a62", - "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62", + "url": "https://api.github.com/repos/symfony/http-client/zipball/3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de", + "reference": "3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de", "shasum": "" }, "require": { @@ -2749,7 +2749,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.3.4" + "source": "https://github.com/symfony/http-client/tree/v7.3.6" }, "funding": [ { @@ -2769,7 +2769,7 @@ "type": "tidelift" } ], - "time": "2025-09-11T10:12:26+00:00" + "time": "2025-11-05T17:41:46+00:00" }, { "name": "symfony/http-client-contracts", @@ -3176,16 +3176,16 @@ }, { "name": "symfony/service-contracts", - "version": "v3.6.0", + "version": "v3.6.1", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4" + "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/f021b05a130d35510bd6b25fe9053c2a8a15d5d4", - "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/45112560a3ba2d715666a509a0bc9521d10b6c43", + "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43", "shasum": "" }, "require": { @@ -3239,7 +3239,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.6.0" + "source": "https://github.com/symfony/service-contracts/tree/v3.6.1" }, "funding": [ { @@ -3250,12 +3250,16 @@ "url": "https://github.com/fabpot", "type": "github" }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" } ], - "time": "2025-04-25T09:37:31+00:00" + "time": "2025-07-15T11:30:57+00:00" }, { "name": "tbachert/spi", @@ -3840,16 +3844,16 @@ }, { "name": "utopia-php/database", - "version": "3.1.2", + "version": "3.2.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "b6541a9cd9b21786a5020327f582838afdb159aa" + "reference": "f2d01b6b38057891184f62107bf70a55bc2ea068" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/b6541a9cd9b21786a5020327f582838afdb159aa", - "reference": "b6541a9cd9b21786a5020327f582838afdb159aa", + "url": "https://api.github.com/repos/utopia-php/database/zipball/f2d01b6b38057891184f62107bf70a55bc2ea068", + "reference": "f2d01b6b38057891184f62107bf70a55bc2ea068", "shasum": "" }, "require": { @@ -3922,10 +3926,10 @@ "utopia" ], "support": { - "source": "https://github.com/utopia-php/database/tree/3.1.2", + "source": "https://github.com/utopia-php/database/tree/3.2.0", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2025-10-30T13:10:13+00:00" + "time": "2025-11-06T05:41:54+00:00" }, { "name": "utopia-php/detector", @@ -5468,16 +5472,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.5.0", + "version": "1.5.3", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "42df22195d6457e52e4c819678168470b114a816" + "reference": "1a7a3b89147aa8c1bde5247f8eeb7e4832c6016d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/42df22195d6457e52e4c819678168470b114a816", - "reference": "42df22195d6457e52e4c819678168470b114a816", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/1a7a3b89147aa8c1bde5247f8eeb7e4832c6016d", + "reference": "1a7a3b89147aa8c1bde5247f8eeb7e4832c6016d", "shasum": "" }, "require": { @@ -5513,9 +5517,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.5.0" + "source": "https://github.com/appwrite/sdk-generator/tree/1.5.3" }, - "time": "2025-10-31T10:10:25+00:00" + "time": "2025-11-10T09:50:41+00:00" }, { "name": "doctrine/annotations", @@ -6168,24 +6172,24 @@ }, { "name": "phpbench/container", - "version": "2.2.2", + "version": "2.2.3", "source": { "type": "git", "url": "https://github.com/phpbench/container.git", - "reference": "a59b929e00b87b532ca6d0edd8eca0967655af33" + "reference": "0c7b2d36c1ea53fe27302fb8873ded7172047196" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpbench/container/zipball/a59b929e00b87b532ca6d0edd8eca0967655af33", - "reference": "a59b929e00b87b532ca6d0edd8eca0967655af33", + "url": "https://api.github.com/repos/phpbench/container/zipball/0c7b2d36c1ea53fe27302fb8873ded7172047196", + "reference": "0c7b2d36c1ea53fe27302fb8873ded7172047196", "shasum": "" }, "require": { "psr/container": "^1.0|^2.0", - "symfony/options-resolver": "^4.2 || ^5.0 || ^6.0 || ^7.0" + "symfony/options-resolver": "^4.2 || ^5.0 || ^6.0 || ^7.0 || ^8.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^2.16", + "php-cs-fixer/shim": "^3.89", "phpstan/phpstan": "^0.12.52", "phpunit/phpunit": "^8" }, @@ -6213,22 +6217,22 @@ "description": "Simple, configurable, service container.", "support": { "issues": "https://github.com/phpbench/container/issues", - "source": "https://github.com/phpbench/container/tree/2.2.2" + "source": "https://github.com/phpbench/container/tree/2.2.3" }, - "time": "2023-10-30T13:38:26+00:00" + "time": "2025-11-06T09:05:13+00:00" }, { "name": "phpbench/phpbench", - "version": "1.4.2", + "version": "1.4.3", "source": { "type": "git", "url": "https://github.com/phpbench/phpbench.git", - "reference": "bb61ae6c54b3d58642be154eb09f4e73c3511018" + "reference": "b641dde59d969ea42eed70a39f9b51950bc96878" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpbench/phpbench/zipball/bb61ae6c54b3d58642be154eb09f4e73c3511018", - "reference": "bb61ae6c54b3d58642be154eb09f4e73c3511018", + "url": "https://api.github.com/repos/phpbench/phpbench/zipball/b641dde59d969ea42eed70a39f9b51950bc96878", + "reference": "b641dde59d969ea42eed70a39f9b51950bc96878", "shasum": "" }, "require": { @@ -6243,26 +6247,26 @@ "phpbench/container": "^2.2", "psr/log": "^1.1 || ^2.0 || ^3.0", "seld/jsonlint": "^1.1", - "symfony/console": "^6.1 || ^7.0", - "symfony/filesystem": "^6.1 || ^7.0", - "symfony/finder": "^6.1 || ^7.0", - "symfony/options-resolver": "^6.1 || ^7.0", - "symfony/process": "^6.1 || ^7.0", + "symfony/console": "^6.1 || ^7.0 || ^8.0", + "symfony/filesystem": "^6.1 || ^7.0 || ^8.0", + "symfony/finder": "^6.1 || ^7.0 || ^8.0", + "symfony/options-resolver": "^6.1 || ^7.0 || ^8.0", + "symfony/process": "^6.1 || ^7.0 || ^8.0", "webmozart/glob": "^4.6" }, "require-dev": { "dantleech/invoke": "^2.0", "ergebnis/composer-normalize": "^2.39", - "friendsofphp/php-cs-fixer": "^3.0", "jangregor/phpstan-prophecy": "^1.0", + "php-cs-fixer/shim": "^3.9", "phpspec/prophecy": "^1.22", "phpstan/extension-installer": "^1.1", "phpstan/phpstan": "^1.0", "phpstan/phpstan-phpunit": "^1.0", "phpunit/phpunit": "^10.4 || ^11.0", "rector/rector": "^1.2", - "symfony/error-handler": "^6.1 || ^7.0", - "symfony/var-dumper": "^6.1 || ^7.0" + "symfony/error-handler": "^6.1 || ^7.0 || ^8.0", + "symfony/var-dumper": "^6.1 || ^7.0 || ^8.0" }, "suggest": { "ext-xdebug": "For Xdebug profiling extension." @@ -6305,7 +6309,7 @@ ], "support": { "issues": "https://github.com/phpbench/phpbench/issues", - "source": "https://github.com/phpbench/phpbench/tree/1.4.2" + "source": "https://github.com/phpbench/phpbench/tree/1.4.3" }, "funding": [ { @@ -6313,7 +6317,7 @@ "type": "github" } ], - "time": "2025-10-26T14:21:59+00:00" + "time": "2025-11-06T19:07:31+00:00" }, { "name": "phpstan/phpstan", @@ -7962,16 +7966,16 @@ }, { "name": "symfony/console", - "version": "v7.3.5", + "version": "v7.3.6", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "cdb80fa5869653c83cfe1a9084a673b6daf57ea7" + "reference": "c28ad91448f86c5f6d9d2c70f0cf68bf135f252a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/cdb80fa5869653c83cfe1a9084a673b6daf57ea7", - "reference": "cdb80fa5869653c83cfe1a9084a673b6daf57ea7", + "url": "https://api.github.com/repos/symfony/console/zipball/c28ad91448f86c5f6d9d2c70f0cf68bf135f252a", + "reference": "c28ad91448f86c5f6d9d2c70f0cf68bf135f252a", "shasum": "" }, "require": { @@ -8036,7 +8040,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.3.5" + "source": "https://github.com/symfony/console/tree/v7.3.6" }, "funding": [ { @@ -8056,20 +8060,20 @@ "type": "tidelift" } ], - "time": "2025-10-14T15:46:26+00:00" + "time": "2025-11-04T01:21:42+00:00" }, { "name": "symfony/filesystem", - "version": "v7.3.2", + "version": "v7.3.6", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "edcbb768a186b5c3f25d0643159a787d3e63b7fd" + "reference": "e9bcfd7837928ab656276fe00464092cc9e1826a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/edcbb768a186b5c3f25d0643159a787d3e63b7fd", - "reference": "edcbb768a186b5c3f25d0643159a787d3e63b7fd", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/e9bcfd7837928ab656276fe00464092cc9e1826a", + "reference": "e9bcfd7837928ab656276fe00464092cc9e1826a", "shasum": "" }, "require": { @@ -8106,7 +8110,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v7.3.2" + "source": "https://github.com/symfony/filesystem/tree/v7.3.6" }, "funding": [ { @@ -8126,7 +8130,7 @@ "type": "tidelift" } ], - "time": "2025-07-07T08:17:47+00:00" + "time": "2025-11-05T09:52:27+00:00" }, { "name": "symfony/finder", @@ -8982,7 +8986,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -9006,5 +9010,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php index 884b9c5589..63a7fe1559 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php @@ -2,9 +2,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases; -use Utopia\Platform\Action as UtopiaAction; +use Appwrite\Extend\Exception; +use Appwrite\Platform\Action as AppwriteAction; +use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Database\Operator; -class Action extends UtopiaAction +class Action extends AppwriteAction { private string $context = 'legacy'; @@ -13,11 +17,72 @@ class Action extends UtopiaAction return $this->context; } - public function setHttpPath(string $path): UtopiaAction + public function setHttpPath(string $path): AppwriteAction { if (\str_contains($path, '/tablesdb')) { $this->context = 'tablesdb'; } return parent::setHttpPath($path); } + + /** + * Parse operator strings in data array and convert them to Operator objects. + * + * @param array $data The data array that may contain operator JSON strings or arrays + * @param Document $collection The collection document to check for relationship attributes + * @return array The data array with operators converted to Operator objects + * @throws Exception If an operator string is invalid + */ + protected function parseOperators(array $data, Document $collection): array + { + $relationshipKeys = []; + foreach ($collection->getAttribute('attributes', []) as $attribute) { + if ($attribute->getAttribute('type') === Database::VAR_RELATIONSHIP) { + $relationshipKeys[$attribute->getAttribute('key')] = true; + } + } + + foreach ($data as $key => $value) { + if (\str_starts_with($key, '$')) { + continue; + } + + if (isset($relationshipKeys[$key])) { + continue; + } + + // Handle operator as JSON string (from API requests) + if (\is_string($value)) { + $decoded = \json_decode($value, true); + + if ( + \is_array($decoded) && + isset($decoded['method']) && + \is_string($decoded['method']) && + Operator::isMethod($decoded['method']) + ) { + try { + $data[$key] = Operator::parse($value); + } catch (\Exception $e) { + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Invalid operator for attribute "' . $key . '": ' . $e->getMessage()); + } + } + } + // Handle operator as array (from transaction logs after serialization) + elseif ( + \is_array($value) && + isset($value['method']) && + \is_string($value['method']) && + Operator::isMethod($value['method']) + ) { + try { + $data[$key] = Operator::parseOperator($value); + } catch (\Exception $e) { + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Invalid operator for attribute "' . $key . '": ' . $e->getMessage()); + } + } + } + + return $data; + } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php index c4c169b77c..08eea88e19 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php @@ -4,13 +4,12 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documen use Appwrite\Event\Event; use Appwrite\Extend\Exception; -use Appwrite\Platform\Action as AppwriteAction; +use Appwrite\Platform\Modules\Databases\Http\Databases\Action as DatabasesAction; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Operator; use Utopia\Database\Validator\Authorization; -abstract class Action extends AppwriteAction +abstract class Action extends DatabasesAction { /** * @var string|null The current context (either 'row' or 'document') @@ -22,7 +21,7 @@ abstract class Action extends AppwriteAction */ abstract protected function getResponseModel(): string; - public function setHttpPath(string $path): AppwriteAction + public function setHttpPath(string $path): DatabasesAction { if (str_contains($path, '/tablesdb/')) { $this->context = ROWS; @@ -339,53 +338,6 @@ abstract class Action extends AppwriteAction return true; } - /** - * Parse operator strings in data array and convert them to Operator objects. - * - * @param array $data The data array that may contain operator JSON strings - * @param Document $collection The collection document to check for relationship attributes - * @return array The data array with operators converted to Operator objects - * @throws Exception If an operator string is invalid - */ - protected function parseOperators(array $data, Document $collection): array - { - $relationshipKeys = []; - foreach ($collection->getAttribute('attributes', []) as $attribute) { - if ($attribute->getAttribute('type') === Database::VAR_RELATIONSHIP) { - $relationshipKeys[$attribute->getAttribute('key')] = true; - } - } - - foreach ($data as $key => $value) { - if (\str_starts_with($key, '$')) { - continue; - } - - if (isset($relationshipKeys[$key])) { - continue; - } - - if (\is_string($value)) { - $decoded = \json_decode($value, true); - - if ( - \is_array($decoded) && - isset($decoded['method']) && - \is_string($decoded['method']) && - Operator::isMethod($decoded['method']) - ) { - try { - $data[$key] = Operator::parse($value); - } catch (\Exception $e) { - throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Invalid operator for attribute "' . $key . '": ' . $e->getMessage()); - } - } - } - } - - return $data; - } - /** * For triggering different queues for each document for a bulk documents * @param string $event diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php index 8915ae6141..e2a4491736 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php @@ -2,16 +2,16 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Transactions; -use Utopia\Platform\Action as UtopiaAction; +use Appwrite\Platform\Modules\Databases\Http\Databases\Action as DatabasesAction; -abstract class Action extends UtopiaAction +abstract class Action extends DatabasesAction { /** * The current API context (either 'table' or 'collection'). */ private ?string $context = COLLECTIONS; - public function setHttpPath(string $path): UtopiaAction + public function setHttpPath(string $path): DatabasesAction { if (\str_contains($path, '/tablesdb')) { $this->context = TABLES; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php index 6e2bd63827..07c2f17426 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php @@ -149,6 +149,7 @@ class Update extends Action ])); $state = []; + $collections = []; foreach ($operations as $operation) { $databaseInternalId = $operation['databaseInternalId']; @@ -159,6 +160,17 @@ class Update extends Action $action = $operation['action']; $data = $operation['data']; + if (!isset($collections[$collectionId])) { + $collections[$collectionId] = Authorization::skip( + fn () => $dbForProject->getCollection($collectionId) + ); + } + $collection = $collections[$collectionId]; + + if (\is_array($data) && !empty($data)) { + $data = $this->parseOperators($data, $collection); + } + if ($action === 'delete' && $documentId && empty($data)) { $doc = $dbForProject->getDocument($collectionId, $documentId); if (!$doc->isEmpty()) { diff --git a/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php b/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php index efa3b52cef..8ebc7582b3 100644 --- a/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php +++ b/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php @@ -5561,4 +5561,380 @@ trait TransactionsBase $this->assertEquals('Updated after upsert', $response['body']['name']); $this->assertEquals(20, $response['body']['counter']); } + + /** + * Test array operators in transactions using updateRow with transactionId + * This tests the fix for operators not being parsed when stored in transaction logs + */ + public function testArrayOperatorsWithUpdateRow(): void + { + // Create database + $database = $this->client->call(Client::METHOD_POST, '/tablesdb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'ArrayOperatorsTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create table with array column + $table = $this->client->call(Client::METHOD_POST, "/tablesdb/{$databaseId}/tables", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'tableId' => ID::unique(), + 'name' => 'Items', + ]); + + $this->assertEquals(201, $table['headers']['status-code']); + $tableId = $table['body']['$id']; + + // Create array column + $column = $this->client->call(Client::METHOD_POST, "/tablesdb/{$databaseId}/tables/{$tableId}/columns/string", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'columnId' => 'items', + 'name' => 'Items', + 'size' => 255, + 'required' => false, + 'array' => true, + ]); + + $this->assertEquals(202, $column['headers']['status-code']); + sleep(2); // Wait for column to be created + + // Create initial row with some items + $row = $this->client->call(Client::METHOD_POST, "/tablesdb/{$databaseId}/tables/{$tableId}/rows", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'rowId' => 'test-row', + 'data' => [ + 'items' => ['item1', 'item2', 'item3', 'item4'] + ] + ]); + + $this->assertEquals(201, $row['headers']['status-code']); + $this->assertEquals(['item1', 'item2', 'item3', 'item4'], $row['body']['items']); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/tablesdb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Test arrayRemove operator + $updateResponse = $this->client->call(Client::METHOD_PATCH, "/tablesdb/{$databaseId}/tables/{$tableId}/rows/test-row", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'transactionId' => $transactionId, + 'data' => [ + 'items' => '{"method":"arrayRemove","attribute":"","values":["item2"]}' + ] + ]); + + $this->assertEquals(200, $updateResponse['headers']['status-code']); + + // Test arrayInsert operator + $updateResponse = $this->client->call(Client::METHOD_PATCH, "/tablesdb/{$databaseId}/tables/{$tableId}/rows/test-row", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'transactionId' => $transactionId, + 'data' => [ + 'items' => '{"method":"arrayInsert","attribute":"","values":[2,"newItem"]}' + ] + ]); + + $this->assertEquals(200, $updateResponse['headers']['status-code']); + + // Commit transaction + $commitResponse = $this->client->call(Client::METHOD_PATCH, "/tablesdb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'commit' => true + ]); + + $this->assertEquals(200, $commitResponse['headers']['status-code']); + + // Verify the operations were applied correctly + $row = $this->client->call(Client::METHOD_GET, "/tablesdb/{$databaseId}/tables/{$tableId}/rows/test-row", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $row['headers']['status-code']); + // After removing item2: ['item1', 'item3', 'item4'] + // After inserting 'newItem' at index 2: ['item1', 'item3', 'newItem', 'item4'] + $this->assertEquals(['item1', 'item3', 'newItem', 'item4'], $row['body']['items']); + } + + /** + * Test array operators in transactions using createOperations + * This tests the fix for operators not being parsed in bulk operation creation + */ + public function testArrayOperatorsWithCreateOperations(): void + { + // Create database + $database = $this->client->call(Client::METHOD_POST, '/tablesdb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'ArrayOperatorsBulkTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create table with array column + $table = $this->client->call(Client::METHOD_POST, "/tablesdb/{$databaseId}/tables", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'tableId' => ID::unique(), + 'name' => 'Tags', + ]); + + $this->assertEquals(201, $table['headers']['status-code']); + $tableId = $table['body']['$id']; + + // Create array column + $column = $this->client->call(Client::METHOD_POST, "/tablesdb/{$databaseId}/tables/{$tableId}/columns/string", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'columnId' => 'tags', + 'name' => 'Tags', + 'size' => 255, + 'required' => false, + 'array' => true, + ]); + + $this->assertEquals(202, $column['headers']['status-code']); + sleep(2); // Wait for column to be created + + // Create initial row + $row = $this->client->call(Client::METHOD_POST, "/tablesdb/{$databaseId}/tables/{$tableId}/rows", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'rowId' => 'doc1', + 'data' => [ + 'tags' => ['php', 'javascript', 'python', 'ruby'] + ] + ]); + + $this->assertEquals(201, $row['headers']['status-code']); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/tablesdb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Create operations using bulk createOperations endpoint with array operators + $operations = $this->client->call(Client::METHOD_POST, "/tablesdb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'operations' => [ + [ + 'action' => 'update', + 'databaseId' => $databaseId, + 'tableId' => $tableId, + 'rowId' => 'doc1', + 'data' => [ + 'tags' => '{"method":"arrayRemove","attribute":"","values":["javascript"]}' + ] + ], + [ + 'action' => 'update', + 'databaseId' => $databaseId, + 'tableId' => $tableId, + 'rowId' => 'doc1', + 'data' => [ + 'tags' => '{"method":"arrayAppend","attribute":"","values":["go","rust"]}' + ] + ] + ] + ]); + + $this->assertEquals(201, $operations['headers']['status-code']); + $this->assertEquals(2, $operations['body']['operations']); + + // Commit transaction + $commitResponse = $this->client->call(Client::METHOD_PATCH, "/tablesdb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'commit' => true + ]); + + $this->assertEquals(200, $commitResponse['headers']['status-code']); + + // Verify the operations were applied correctly + $row = $this->client->call(Client::METHOD_GET, "/tablesdb/{$databaseId}/tables/{$tableId}/rows/doc1", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $row['headers']['status-code']); + // After removing 'javascript': ['php', 'python', 'ruby'] + // After appending ['go', 'rust']: ['php', 'python', 'ruby', 'go', 'rust'] + $this->assertEquals(['php', 'python', 'ruby', 'go', 'rust'], $row['body']['tags']); + } + + /** + * Test multiple array operators in a single transaction + * This tests all common array operators to ensure comprehensive coverage + */ + public function testMultipleArrayOperators(): void + { + // Create database + $database = $this->client->call(Client::METHOD_POST, '/tablesdb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'MultipleOperatorsTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create table + $table = $this->client->call(Client::METHOD_POST, "/tablesdb/{$databaseId}/tables", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'tableId' => ID::unique(), + 'name' => 'Arrays', + ]); + + $this->assertEquals(201, $table['headers']['status-code']); + $tableId = $table['body']['$id']; + + // Create multiple array columns + $columns = [ + ['columnId' => 'list1', 'name' => 'List1'], + ['columnId' => 'list2', 'name' => 'List2'], + ['columnId' => 'list3', 'name' => 'List3'], + ]; + + foreach ($columns as $col) { + $column = $this->client->call(Client::METHOD_POST, "/tablesdb/{$databaseId}/tables/{$tableId}/columns/string", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'columnId' => $col['columnId'], + 'name' => $col['name'], + 'size' => 255, + 'required' => false, + 'array' => true, + ]); + $this->assertEquals(202, $column['headers']['status-code']); + } + + sleep(2); // Wait for columns to be created + + // Create initial row + $row = $this->client->call(Client::METHOD_POST, "/tablesdb/{$databaseId}/tables/{$tableId}/rows", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'rowId' => 'multi-ops', + 'data' => [ + 'list1' => ['a', 'b', 'c'], + 'list2' => ['x', 'y', 'z'], + 'list3' => ['1', '2', '3', '4', '5'] + ] + ]); + + $this->assertEquals(201, $row['headers']['status-code']); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/tablesdb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Test arrayPrepend + $this->client->call(Client::METHOD_PATCH, "/tablesdb/{$databaseId}/tables/{$tableId}/rows/multi-ops", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'transactionId' => $transactionId, + 'data' => [ + 'list1' => '{"method":"arrayPrepend","attribute":"","values":["z"]}' + ] + ]); + + // Test arrayAppend + $this->client->call(Client::METHOD_PATCH, "/tablesdb/{$databaseId}/tables/{$tableId}/rows/multi-ops", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'transactionId' => $transactionId, + 'data' => [ + 'list2' => '{"method":"arrayAppend","attribute":"","values":["w"]}' + ] + ]); + + // Test arrayRemove + $this->client->call(Client::METHOD_PATCH, "/tablesdb/{$databaseId}/tables/{$tableId}/rows/multi-ops", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'transactionId' => $transactionId, + 'data' => [ + 'list3' => '{"method":"arrayRemove","attribute":"","values":["3"]}' + ] + ]); + + // Commit transaction + $commitResponse = $this->client->call(Client::METHOD_PATCH, "/tablesdb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'commit' => true + ]); + + $this->assertEquals(200, $commitResponse['headers']['status-code']); + + // Verify all operations were applied correctly + $row = $this->client->call(Client::METHOD_GET, "/tablesdb/{$databaseId}/tables/{$tableId}/rows/multi-ops", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $row['headers']['status-code']); + $this->assertEquals(['z', 'a', 'b', 'c'], $row['body']['list1'], 'arrayPrepend should add element at the beginning'); + $this->assertEquals(['x', 'y', 'z', 'w'], $row['body']['list2'], 'arrayAppend should add element at the end'); + $this->assertEquals(['1', '2', '4', '5'], $row['body']['list3'], 'arrayRemove should remove the element'); + } } From 94614d32d0c1a4dca3e109bdb015dfa84480ae96 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 11 Nov 2025 21:09:12 +1300 Subject: [PATCH 02/16] Handle deep nested ops --- .../Modules/Databases/Http/Databases/Action.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php index 63a7fe1559..f751fbab33 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php @@ -43,6 +43,13 @@ class Action extends AppwriteAction } foreach ($data as $key => $value) { + if (!\is_string($key)) { + if (\is_array($value)) { + $data[$key] = $this->parseOperators($value, $collection); + } + continue; + } + if (\str_starts_with($key, '$')) { continue; } @@ -81,6 +88,9 @@ class Action extends AppwriteAction throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Invalid operator for attribute "' . $key . '": ' . $e->getMessage()); } } + elseif (\is_array($value)) { + $data[$key] = $this->parseOperators($value, $collection); + } } return $data; From 353e84bb7422a4761a2fcc4353af47aa96f84d14 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 11 Nov 2025 21:10:11 +1300 Subject: [PATCH 03/16] Fix keys --- .../Databases/TablesDB/Transactions/TransactionsBase.php | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php b/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php index 8ebc7582b3..cbdc7826a0 100644 --- a/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php +++ b/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php @@ -5600,8 +5600,7 @@ trait TransactionsBase 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ - 'columnId' => 'items', - 'name' => 'Items', + 'key' => 'items', 'size' => 255, 'required' => false, 'array' => true, @@ -5719,8 +5718,7 @@ trait TransactionsBase 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ - 'columnId' => 'tags', - 'name' => 'Tags', + 'key' => 'tags', 'size' => 255, 'required' => false, 'array' => true, @@ -5848,8 +5846,7 @@ trait TransactionsBase 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ - 'columnId' => $col['columnId'], - 'name' => $col['name'], + 'key' => $col['columnId'], 'size' => 255, 'required' => false, 'array' => true, From ab5aac00c40a0fb2d41fa7821c4dbc669953eee5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 11 Nov 2025 21:25:21 +1300 Subject: [PATCH 04/16] Format --- .../Platform/Modules/Databases/Http/Databases/Action.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php index f751fbab33..728e732cc5 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php @@ -87,8 +87,7 @@ class Action extends AppwriteAction } catch (\Exception $e) { throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Invalid operator for attribute "' . $key . '": ' . $e->getMessage()); } - } - elseif (\is_array($value)) { + } elseif (\is_array($value)) { $data[$key] = $this->parseOperators($value, $collection); } } From ae21fadf0d5cd484dec9261f3c703d41aa98d731 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Nov 2025 16:48:26 +1300 Subject: [PATCH 05/16] Fix event firing on txn ops --- .../Databases/Collections/Documents/Attribute/Decrement.php | 2 ++ .../Databases/Collections/Documents/Attribute/Increment.php | 2 ++ .../Http/Databases/Collections/Documents/Bulk/Delete.php | 2 ++ .../Http/Databases/Collections/Documents/Bulk/Update.php | 2 ++ .../Http/Databases/Collections/Documents/Bulk/Upsert.php | 2 ++ .../Databases/Http/Databases/Collections/Documents/Create.php | 2 ++ .../Databases/Http/Databases/Collections/Documents/Delete.php | 2 ++ .../Databases/Http/Databases/Collections/Documents/Update.php | 4 +++- .../Databases/Http/Databases/Collections/Documents/Upsert.php | 2 ++ 9 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php index 0b3d0e9fb4..4c6e1a38e3 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php @@ -152,6 +152,8 @@ class Decrement extends Action ); }); + $queueForEvents->reset(); + // Return successful response without actually decrementing $groupId = $this->getGroupId(); $mockDocument = new Document([ diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php index ef64099fa8..56c76137db 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php @@ -152,6 +152,8 @@ class Increment extends Action ); }); + $queueForEvents->reset(); + // Return successful response without actually incrementing $groupId = $this->getGroupId(); $mockDocument = new Document([ diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php index 4b1251e016..d4c70d82bc 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php @@ -150,6 +150,8 @@ class Delete extends Action ); }); + $queueForEvents->reset(); + // Return successful response without actually deleting documents $response->dynamic(new Document([ $this->getSDKGroup() => [], diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php index 2152534efe..7b7e40567a 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php @@ -174,6 +174,8 @@ class Update extends Action ); }); + $queueForEvents->reset(); + // Return successful response without actually updating documents $response->dynamic(new Document([ $this->getSDKGroup() => [], diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php index 6c869723a7..6ceec3b8f7 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php @@ -149,6 +149,8 @@ class Upsert extends Action ); }); + $queueForEvents->reset(); + // Return successful response without actually upserting documents $response->dynamic(new Document([ $this->getSDKGroup() => [], diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php index 521190d3dc..b80c170791 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php @@ -412,6 +412,8 @@ class Create extends Action ); }); + $queueForEvents->reset(); + // Return successful response without actually creating documents if ($isBulk) { $response->dynamic(new Document([ diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php index a4cef59e5f..b9f3f7eb86 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php @@ -176,6 +176,8 @@ class Delete extends Action ); }); + $queueForEvents->reset(); + // Return successful response without actually deleting document $response->noContent(); return; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php index ed83f3fdd3..060a60d863 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php @@ -243,7 +243,6 @@ class Update extends Action ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); - // Handle transaction staging if ($transactionId !== null) { $transaction = ($isAPIKey || $isPrivilegedUser) @@ -302,6 +301,9 @@ class Update extends Action ...$document->getArrayCopy(), ...$data ]); + + $queueForEvents->reset(); + $response ->setStatusCode(SwooleResponse::STATUS_CODE_OK) ->dynamic($mockDocument, $this->getResponseModel()); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php index 8d47410649..96a4b32584 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php @@ -302,6 +302,8 @@ class Upsert extends Action ); }); + $queueForEvents->reset(); + // Return successful response without actually upserting document $groupId = $this->getGroupId(); $mockDocument = new Document([ From c9582978ed209e52e1d075827f8b749fb3384a53 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Nov 2025 16:48:42 +1300 Subject: [PATCH 06/16] Fix operator parsing on txn ops --- .../Http/Databases/Collections/Documents/Bulk/Update.php | 4 +++- .../Http/Databases/Collections/Documents/Bulk/Upsert.php | 4 +++- .../Databases/Http/Databases/Collections/Documents/Update.php | 4 +++- .../Databases/Http/Databases/Collections/Documents/Upsert.php | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php index 7b7e40567a..5143dabc83 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php @@ -107,7 +107,9 @@ class Update extends Action throw new Exception($this->getParentNotFoundException()); } - $data = $this->parseOperators($data, $collection); + if ($transactionId === null) { + $data = $this->parseOperators($data, $collection); + } $hasRelationships = \array_filter( $collection->getAttribute('attributes', []), diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php index 6ceec3b8f7..00699bdfab 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php @@ -107,7 +107,9 @@ class Upsert extends Action } foreach ($documents as $key => $document) { - $document = $this->parseOperators($document, $collection); + if ($transactionId === null) { + $document = $this->parseOperators($document, $collection); + } $document = $this->removeReadonlyAttributes($document, privileged: true); $documents[$key] = new Document($document); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php index 060a60d863..f920c73281 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php @@ -112,7 +112,9 @@ class Update extends Action throw new Exception($this->getParentNotFoundException()); } - $data = $this->parseOperators($data, $collection); + if ($transactionId === null) { + $data = $this->parseOperators($data, $collection); + } // Read permission should not be required for update /** @var Document $document */ diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php index 96a4b32584..1d1db9bece 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php @@ -118,7 +118,9 @@ class Upsert extends Action throw new Exception($this->getParentNotFoundException()); } - $data = $this->parseOperators($data, $collection); + if ($transactionId === null) { + $data = $this->parseOperators($data, $collection); + } $allowedPermissions = [ Database::PERMISSION_READ, From 46d785bbb8acdbb37b6953897cb92142f5660c24 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Nov 2025 21:04:17 +1300 Subject: [PATCH 07/16] Fix tests --- .../TablesDB/Transactions/TransactionsBase.php | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php b/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php index cbdc7826a0..09f62d165d 100644 --- a/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php +++ b/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php @@ -5589,6 +5589,12 @@ trait TransactionsBase ]), [ 'tableId' => ID::unique(), 'name' => 'Items', + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], ]); $this->assertEquals(201, $table['headers']['status-code']); @@ -5707,6 +5713,12 @@ trait TransactionsBase ]), [ 'tableId' => ID::unique(), 'name' => 'Tags', + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], ]); $this->assertEquals(201, $table['headers']['status-code']); @@ -5828,6 +5840,12 @@ trait TransactionsBase ]), [ 'tableId' => ID::unique(), 'name' => 'Arrays', + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], ]); $this->assertEquals(201, $table['headers']['status-code']); From 37f16ef0bb6500ab5c69f617329b19c58d8df319 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Nov 2025 21:54:37 +1300 Subject: [PATCH 08/16] Fix array conversion --- .../Databases/Http/Databases/Transactions/Update.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php index 07c2f17426..899c2b3eaf 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php @@ -160,6 +160,10 @@ class Update extends Action $action = $operation['action']; $data = $operation['data']; + if ($data instanceof Document) { + $data = $data->getArrayCopy(); + } + if (!isset($collections[$collectionId])) { $collections[$collectionId] = Authorization::skip( fn () => $dbForProject->getCollection($collectionId) @@ -184,10 +188,6 @@ class Update extends Action $databaseOperations[$databaseInternalId] = ($databaseOperations[$databaseInternalId] ?? 0) + 1; } - if ($data instanceof Document) { - $data = $data->getArrayCopy(); - } - switch ($action) { case 'create': $this->handleCreateOperation($dbForProject, $collectionId, $documentId, $data, $createdAt, $state); From e8f2e78ac44a79536156b8c9ea08a8a53347d8b0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 12 Nov 2025 22:18:25 +1300 Subject: [PATCH 09/16] Use internal bucket --- app/controllers/api/migrations.php | 8 +- .../examples/migrations/create-csv-export.md | 1 - .../examples/migrations/create-csv-export.md | 1 - .../migrations/migration-csv-export.md | 2 +- src/Appwrite/Platform/Workers/Migrations.php | 32 +++---- .../Services/Migrations/MigrationsBase.php | 83 +------------------ 6 files changed, 26 insertions(+), 101 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 1d1e6e999c..41b98ab333 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -468,7 +468,6 @@ App::post('/v1/migrations/csv/exports') ] )) ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') - ->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.') ->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .csv extension.') ->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true) ->param('queries', [], new ArrayList(new Text(0)), 'Array of query strings generated using the Query class provided by the SDK to filter documents to export. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) @@ -480,12 +479,12 @@ App::post('/v1/migrations/csv/exports') ->inject('user') ->inject('response') ->inject('dbForProject') + ->inject('dbForPlatform') ->inject('project') ->inject('queueForEvents') ->inject('queueForMigrations') ->action(function ( string $resourceId, - string $bucketId, string $filename, array $columns, array $queries, @@ -497,6 +496,7 @@ App::post('/v1/migrations/csv/exports') Document $user, Response $response, Database $dbForProject, + Database $dbForPlatform, Document $project, Event $queueForEvents, Migration $queueForMigrations @@ -507,7 +507,7 @@ App::post('/v1/migrations/csv/exports') throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + $bucket = Authorization::skip(fn () => $dbForPlatform->getDocument('buckets', 'default')); if ($bucket->isEmpty()) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } @@ -553,7 +553,7 @@ App::post('/v1/migrations/csv/exports') 'resourceData' => '{}', 'errors' => [], 'options' => [ - 'bucketId' => $bucketId, + 'bucketId' => 'default', // Always use internal bucket 'filename' => $filename, 'columns' => $columns, 'queries' => $queries, diff --git a/docs/examples/1.8.x/console-cli/examples/migrations/create-csv-export.md b/docs/examples/1.8.x/console-cli/examples/migrations/create-csv-export.md index e56afae786..61eceabcd8 100644 --- a/docs/examples/1.8.x/console-cli/examples/migrations/create-csv-export.md +++ b/docs/examples/1.8.x/console-cli/examples/migrations/create-csv-export.md @@ -1,4 +1,3 @@ appwrite migrations create-csv-export \ --resource-id \ - --bucket-id \ --filename diff --git a/docs/examples/1.8.x/console-web/examples/migrations/create-csv-export.md b/docs/examples/1.8.x/console-web/examples/migrations/create-csv-export.md index e1b909a852..89f779fc4c 100644 --- a/docs/examples/1.8.x/console-web/examples/migrations/create-csv-export.md +++ b/docs/examples/1.8.x/console-web/examples/migrations/create-csv-export.md @@ -8,7 +8,6 @@ const migrations = new Migrations(client); const result = await migrations.createCSVExport({ resourceId: '', - bucketId: '', filename: '', columns: [], // optional queries: [], // optional diff --git a/docs/references/migrations/migration-csv-export.md b/docs/references/migrations/migration-csv-export.md index 866faed2d2..069dda895e 100644 --- a/docs/references/migrations/migration-csv-export.md +++ b/docs/references/migrations/migration-csv-export.md @@ -1 +1 @@ -Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in an Appwrite Storage bucket. \ No newline at end of file +Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in a secure internal bucket. You'll receive an email with a download link when the export is complete. \ No newline at end of file diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index fc7949e783..42d81d8999 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -16,6 +16,8 @@ use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; +use Utopia\Database\Helpers\Permission; +use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Locale\Locale; use Utopia\Migration\Destination; @@ -432,13 +434,15 @@ class Migrations extends Action Mail $queueForMails ): void { $options = $migration->getAttribute('options', []); - $bucketId = $options['bucketId'] ?? null; + $bucketId = 'default'; // Always use platform default bucket $filename = $options['filename'] ?? 'export_' . \time(); $userInternalId = $options['userInternalId'] ?? ''; + $user = $this->dbForPlatform->findOne('users', [ + Query::equal('$sequence', [$userInternalId]) + ]); - $bucket = $this->dbForProject->getDocument('buckets', $bucketId); - if ($bucket->isEmpty()) { - throw new \Exception("Bucket not found: $bucketId"); + if ($user->isEmpty()) { + throw new \Exception('User ' . $userInternalId . ' not found'); } $path = $this->deviceForFiles->getPath($bucketId . '/' . $this->sanitizeFilename($filename) . '.csv'); @@ -469,7 +473,7 @@ class Migrations extends Action $this->sendCSVEmail( success: false, project: $project, - userInternalId: $userInternalId, + user: $user, options: $options, queueForMails: $queueForMails, sizeMB: $sizeMB @@ -479,9 +483,11 @@ class Migrations extends Action } } - $this->dbForProject->createDocument('bucket_' . $bucket->getSequence(), new Document([ + $this->dbForPlatform->createDocument('bucket_' . $bucket->getSequence(), new Document([ '$id' => $fileId, - '$permissions' => [], + '$permissions' => [ + Permission::read(Role::user($user->getId())), + ], 'bucketId' => $bucket->getId(), 'bucketInternalId' => $bucket->getSequence(), 'name' => $filename, @@ -521,7 +527,7 @@ class Migrations extends Action $this->sendCSVEmail( success: true, project: $project, - userInternalId: $userInternalId, + user: $user, options: $options, queueForMails: $queueForMails, downloadUrl: $downloadUrl @@ -533,7 +539,7 @@ class Migrations extends Action * * @param bool $success Whether the export was successful * @param Document $project - * @param string $userInternalId Internal ID of the user + * @param string $user The user who triggered the operation * @param array $options Migration options * @param Mail $queueForMails * @param string $downloadUrl Download URL for successful exports @@ -544,7 +550,7 @@ class Migrations extends Action protected function sendCSVEmail( bool $success, Document $project, - string $userInternalId, + Document $user, array $options, Mail $queueForMails, string $downloadUrl = '', @@ -554,12 +560,8 @@ class Migrations extends Action return; } - $user = $this->dbForPlatform->findOne('users', [ - Query::equal('$sequence', [$userInternalId]) - ]); - if ($user->isEmpty()) { - Console::warning("User not found for CSV export notification: $userInternalId"); + Console::warning("User not found for CSV export notification: {$user->getInternalId()}"); return; } diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 16e58c9c2c..a6ee9c15d5 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1282,39 +1282,11 @@ trait MigrationsBase $this->assertEquals(200, $docs['headers']['status-code']); $this->assertEquals(10, $docs['body']['total'], 'Expected 10 documents but got ' . $docs['body']['total']); - // Create a storage bucket for the export - $bucketIdUnique = ID::unique(); - $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ], [ - 'bucketId' => $bucketIdUnique, - 'name' => 'Test Export Bucket', - 'permissions' => [ - Permission::read(Role::any()), - Permission::create(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'fileSecurity' => false, - 'enabled' => true, - 'maximumFileSize' => 10485760, // 10MB - 'allowedFileExtensions' => ['csv'], - 'compression' => 'none', - 'encryption' => false, - 'antivirus' => false - ]); - - $this->assertEquals(201, $bucket['headers']['status-code']); - $bucketId = $bucket['body']['$id']; - - // Perform CSV export with notification enabled + // Perform CSV export with notification enabled (uses internal bucket) $migration = $this->client->call(Client::METHOD_POST, '/migrations/csv/exports', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] ], $this->getHeaders()), [ - 'bucketId' => $bucketId, 'resourceId' => $databaseId . ':' . $collectionId, 'filename' => 'test-export', 'columns' => [], @@ -1329,7 +1301,7 @@ trait MigrationsBase $this->assertNotEmpty($migration['body']['$id']); $migrationId = $migration['body']['$id']; - $this->assertEventually(function () use ($bucketId, $migrationId) { + $this->assertEventually(function () use ($migrationId) { $response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -1341,55 +1313,10 @@ trait MigrationsBase $this->assertEquals('completed', $response['body']['status']); $this->assertEquals('Appwrite', $response['body']['source']); $this->assertEquals('CSV', $response['body']['destination']); - $this->assertEquals($bucketId, $response['body']['options']['bucketId']); return true; }, 30000, 500); - // Check that the file was created in the bucket - // Query files by filename - $files = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ], [ - 'queries' => [ - Query::equal('name', ['test-export'])->toString() - ] - ]); - - $this->assertEquals(200, $files['headers']['status-code']); - $this->assertEquals(1, $files['body']['total'], 'Expected exactly one file with name "test-export"'); - - // Get the exported file - $file = $files['body']['files'][0]; - $fileId = $file['$id']; - - $this->assertEquals($bucketId, $file['bucketId']); - $this->assertEquals('test-export', $file['name']); - $this->assertEquals('text/csv', $file['mimeType']); - $this->assertGreaterThan(0, $file['sizeOriginal']); - - // Download and verify CSV content - $download = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download', \array_merge([ - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); - - $this->assertEquals(200, $download['headers']['status-code']); - - $csvContent = $download['body']; - $lines = explode("\n", trim($csvContent)); - $this->assertCount(11, $lines); - $this->assertStringContainsString('$id', $lines[0]); - $this->assertStringContainsString('$permissions', $lines[0]); - $this->assertStringContainsString('$createdAt', $lines[0]); - $this->assertStringContainsString('$updatedAt', $lines[0]); - $this->assertStringContainsString('name', $lines[0]); - $this->assertStringContainsString('email', $lines[0]); - - $this->assertStringContainsString('Test User 1', $lines[1]); - $this->assertStringContainsString('user1@appwrite.io', $lines[1]); - // Check that email was sent with download link $lastEmail = $this->getLastEmail(); $this->assertNotEmpty($lastEmail); @@ -1407,6 +1334,8 @@ trait MigrationsBase \parse_str($components['query'] ?? '', $queryParams); $this->assertArrayHasKey('jwt', $queryParams, 'JWT not found in download URL'); $this->assertNotEmpty($queryParams['jwt']); + $this->assertArrayHasKey('project', $queryParams, 'Project not found in download URL'); + $this->assertStringContainsString('/storage/buckets/default/files/', $downloadUrl); // Test download with JWT $path = \str_replace('/v1', '', $components['path']); @@ -1426,9 +1355,5 @@ trait MigrationsBase 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]); - $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [ - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]); } } From b2d228e102bdebd00f67854d8cfe9d6e8631a159 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 13 Nov 2025 23:42:13 +1300 Subject: [PATCH 10/16] Update specs --- app/config/specs/open-api3-1.8.x-console.json | 8 +------- app/config/specs/open-api3-latest-console.json | 8 +------- app/config/specs/swagger2-1.8.x-console.json | 9 +-------- app/config/specs/swagger2-latest-console.json | 9 +-------- 4 files changed, 4 insertions(+), 30 deletions(-) diff --git a/app/config/specs/open-api3-1.8.x-console.json b/app/config/specs/open-api3-1.8.x-console.json index 287c33e5ec..ccf430cb28 100644 --- a/app/config/specs/open-api3-1.8.x-console.json +++ b/app/config/specs/open-api3-1.8.x-console.json @@ -22902,7 +22902,7 @@ "tags": [ "migrations" ], - "description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in an Appwrite Storage bucket.", + "description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in a secure internal bucket. You'll receive an email with a download link when the export is complete.", "responses": { "202": { "description": "Migration", @@ -22952,11 +22952,6 @@ "description": "Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.", "x-example": "" }, - "bucketId": { - "type": "string", - "description": "Storage bucket unique ID where the exported CSV will be stored.", - "x-example": "" - }, "filename": { "type": "string", "description": "The name of the file to be created for the export, excluding the .csv extension.", @@ -23006,7 +23001,6 @@ }, "required": [ "resourceId", - "bucketId", "filename" ] } diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 287c33e5ec..ccf430cb28 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -22902,7 +22902,7 @@ "tags": [ "migrations" ], - "description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in an Appwrite Storage bucket.", + "description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in a secure internal bucket. You'll receive an email with a download link when the export is complete.", "responses": { "202": { "description": "Migration", @@ -22952,11 +22952,6 @@ "description": "Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.", "x-example": "" }, - "bucketId": { - "type": "string", - "description": "Storage bucket unique ID where the exported CSV will be stored.", - "x-example": "" - }, "filename": { "type": "string", "description": "The name of the file to be created for the export, excluding the .csv extension.", @@ -23006,7 +23001,6 @@ }, "required": [ "resourceId", - "bucketId", "filename" ] } diff --git a/app/config/specs/swagger2-1.8.x-console.json b/app/config/specs/swagger2-1.8.x-console.json index 4562120363..2d52da1db7 100644 --- a/app/config/specs/swagger2-1.8.x-console.json +++ b/app/config/specs/swagger2-1.8.x-console.json @@ -23005,7 +23005,7 @@ "tags": [ "migrations" ], - "description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in an Appwrite Storage bucket.", + "description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in a secure internal bucket. You'll receive an email with a download link when the export is complete.", "responses": { "202": { "description": "Migration", @@ -23053,12 +23053,6 @@ "default": null, "x-example": "" }, - "bucketId": { - "type": "string", - "description": "Storage bucket unique ID where the exported CSV will be stored.", - "default": null, - "x-example": "" - }, "filename": { "type": "string", "description": "The name of the file to be created for the export, excluding the .csv extension.", @@ -23116,7 +23110,6 @@ }, "required": [ "resourceId", - "bucketId", "filename" ] } diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index 4562120363..2d52da1db7 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -23005,7 +23005,7 @@ "tags": [ "migrations" ], - "description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in an Appwrite Storage bucket.", + "description": "Export documents to a CSV file from your Appwrite database. This endpoint allows you to export documents to a CSV file stored in a secure internal bucket. You'll receive an email with a download link when the export is complete.", "responses": { "202": { "description": "Migration", @@ -23053,12 +23053,6 @@ "default": null, "x-example": "" }, - "bucketId": { - "type": "string", - "description": "Storage bucket unique ID where the exported CSV will be stored.", - "default": null, - "x-example": "" - }, "filename": { "type": "string", "description": "The name of the file to be created for the export, excluding the .csv extension.", @@ -23116,7 +23110,6 @@ }, "required": [ "resourceId", - "bucketId", "filename" ] } From 3df2efb7aec7515ac51ecbdb1722694c9ce25079 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 14 Nov 2025 00:15:18 +1300 Subject: [PATCH 11/16] Fix file reads --- app/controllers/api/storage.php | 13 ++++++++----- src/Appwrite/Platform/Workers/Migrations.php | 15 +++++++++++---- tests/e2e/Services/Migrations/MigrationsBase.php | 15 +++++++-------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 72acd7b4a2..d4adfe27cf 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1477,12 +1477,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/push') ->inject('response') ->inject('request') ->inject('dbForProject') + ->inject('dbForPlatform') ->inject('project') ->inject('mode') ->inject('deviceForFiles') - ->action(function (string $bucketId, string $fileId, string $jwt, Response $response, Request $request, Database $dbForProject, Document $project, string $mode, Device $deviceForFiles) { - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - + ->action(function (string $bucketId, string $fileId, string $jwt, Response $response, Request $request, Database $dbForProject, Database $dbForPlatform, Document $project, string $mode, Device $deviceForFiles) { $decoder = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0); try { @@ -1499,15 +1498,19 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/push') throw new Exception(Exception::USER_UNAUTHORIZED); } + // Check if this is an internal/platform file based on JWT flag + $isInternal = $decoded['internal'] ?? false; + $db = $isInternal ? $dbForPlatform : $dbForProject; + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); + $bucket = Authorization::skip(fn () => $db->getDocument('buckets', $bucketId)); if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - + $file = Authorization::skip(fn () => $db->getDocument('bucket_' . $bucket->getSequence(), $fileId)); if ($file->isEmpty()) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); } diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 42d81d8999..875dca57ce 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -11,7 +11,7 @@ use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Exception\Authorization; +use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; @@ -19,6 +19,7 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; use Utopia\Locale\Locale; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; @@ -225,7 +226,7 @@ class Migrations extends Action } /** - * @throws Authorization + * @throws AuthorizationException * @throws Structure * @throws Conflict * @throws \Utopia\Database\Exception @@ -284,7 +285,7 @@ class Migrations extends Action } /** - * @throws Authorization + * @throws AuthorizationException * @throws Conflict * @throws Restricted * @throws Structure @@ -423,7 +424,7 @@ class Migrations extends Action * @param Document $migration * @param Mail $queueForMails * @return void - * @throws Authorization + * @throws AuthorizationException * @throws Structure * @throws \Utopia\Database\Exception * @throws Exception @@ -445,6 +446,11 @@ class Migrations extends Action throw new \Exception('User ' . $userInternalId . ' not found'); } + $bucket = Authorization::skip(fn () => $this->dbForPlatform->getDocument('buckets', $bucketId)); + if ($bucket->isEmpty()) { + throw new \Exception('Bucket not found'); + } + $path = $this->deviceForFiles->getPath($bucketId . '/' . $this->sanitizeFilename($filename) . '.csv'); $size = $this->deviceForFiles->getFileSize($path); $mime = $this->deviceForFiles->getFileMimeType($path); @@ -517,6 +523,7 @@ class Migrations extends Action 'bucketId' => $bucketId, 'fileId' => $fileId, 'projectId' => $project->getId(), + 'internal' => true, ]); // Generate download URL with JWT diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index a6ee9c15d5..f16864960e 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1341,16 +1341,15 @@ trait MigrationsBase $path = \str_replace('/v1', '', $components['path']); $downloadWithJwt = $this->client->call(Client::METHOD_GET, $path . '?project=' . $queryParams['project'] . '&jwt=' . $queryParams['jwt']); $this->assertEquals(200, $downloadWithJwt['headers']['status-code'], 'Failed to download file with JWT'); - $this->assertEquals($csvContent, $downloadWithJwt['body'], 'Downloaded content differs from original'); - // Test that download without JWT fails - $downloadWithoutJwt = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download'); - $this->assertEquals(404, $downloadWithoutJwt['headers']['status-code'], 'File should not be downloadable without JWT'); + // Verify the downloaded content is valid CSV + $csvData = $downloadWithJwt['body']; + $this->assertNotEmpty($csvData, 'CSV export should not be empty'); + $this->assertStringContainsString('name', $csvData, 'CSV should contain the name column header'); + $this->assertStringContainsString('email', $csvData, 'CSV should contain the email column header'); + $this->assertStringContainsString('Test User 1', $csvData, 'CSV should contain test data'); - $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [ - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]); + // Cleanup $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId, [ 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] From b9211d7141cb932214272b6a80fd87de7976ae76 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 14 Nov 2025 15:55:41 +1300 Subject: [PATCH 12/16] Add delete old CSV job for maintenance --- app/controllers/api/storage.php | 7 ++-- app/init/constants.php | 1 + src/Appwrite/Platform/Tasks/Maintenance.php | 8 +++++ src/Appwrite/Platform/Workers/Deletes.php | 38 +++++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 9cd3575b52..13234513c0 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1499,19 +1499,18 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/push') throw new Exception(Exception::USER_UNAUTHORIZED); } - // Check if this is an internal/platform file based on JWT flag $isInternal = $decoded['internal'] ?? false; - $db = $isInternal ? $dbForPlatform : $dbForProject; + $dbForProject = $isInternal ? $dbForPlatform : $dbForProject; $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); - $bucket = Authorization::skip(fn () => $db->getDocument('buckets', $bucketId)); + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } - $file = Authorization::skip(fn () => $db->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); if ($file->isEmpty()) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); } diff --git a/app/init/constants.php b/app/init/constants.php index 3b81785690..e11fdf9a54 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -138,6 +138,7 @@ const DELETE_TYPE_TOPIC = 'topic'; const DELETE_TYPE_TARGET = 'target'; const DELETE_TYPE_EXPIRED_TARGETS = 'invalid_targets'; const DELETE_TYPE_SESSION_TARGETS = 'session_targets'; +const DELETE_TYPE_CSV_EXPORTS = 'csv_exports'; const DELETE_TYPE_MAINTENANCE = 'maintenance'; // Message types diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 036e8783d4..f5785d0bb4 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -95,6 +95,7 @@ class Maintenance extends Action $this->renewCertificates($dbForPlatform, $queueForCertificates); $this->notifyDeleteCache($cacheRetention, $queueForDeletes); $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes); + $this->notifyDeleteCSVExports($queueForDeletes); }, $interval, $delay); } @@ -106,6 +107,13 @@ class Maintenance extends Action ->trigger(); } + private function notifyDeleteCSVExports(Delete $queueForDeletes): void + { + $queueForDeletes + ->setType(DELETE_TYPE_CSV_EXPORTS) + ->trigger(); + } + private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void { $time = DatabaseDateTime::now(); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 331a2668a3..964eddaad3 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -179,6 +179,9 @@ class Deletes extends Action case DELETE_TYPE_SESSION_TARGETS: $this->deleteSessionTargets($project, $getProjectDB, $document); break; + case DELETE_TYPE_CSV_EXPORTS: + $this->deleteOldCSVExports($dbForPlatform, $deviceForFiles); + break; case DELETE_TYPE_MAINTENANCE: $this->deleteExpiredTargets($project, $getProjectDB); $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention); @@ -720,6 +723,41 @@ class Deletes extends Action ], $dbForProject); } + /** + * @param Database $dbForPlatform + * @param Device $deviceForFiles + * @return void + * @throws Exception|Throwable + */ + private function deleteOldCSVExports(Database $dbForPlatform, Device $deviceForFiles): void + { + $bucket = $dbForPlatform->getDocument('buckets', 'default'); + + if ($bucket->isEmpty()) { + Console::warning('Default bucket not found, skipping CSV export cleanup'); + return; + } + + $oneWeekAgo = DateTime::addSeconds(new \DateTime(), -1 * 60 * 60 * 24 * 7); // 1 week + + Console::info("Deleting CSV export files older than " . $oneWeekAgo); + + $this->deleteByGroup('bucket_' . $bucket->getSequence(), [ + Query::select([...$this->selects, '$createdAt', 'filename', 'path']), + Query::equal('bucketId', ['default']), + Query::endsWith('filename', ['.csv']), + Query::createdBefore($oneWeekAgo), + Query::orderDesc('$createdAt'), + Query::orderDesc(), + ], $dbForPlatform, function (Document $file) use ($deviceForFiles) { + $path = $file->getAttribute('path'); + if ($deviceForFiles->exists($path)) { + $deviceForFiles->delete($path); + Console::success('Deleted CSV file: ' . $file->getAttribute('name')); + } + }); + } + /** * @param Database $dbForPlatform * @param string $datetime From 0588fb405d8c4cb745396e7f8cfd2fa635f2dd1a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 14 Nov 2025 16:16:06 +1300 Subject: [PATCH 13/16] Fix keys --- src/Appwrite/Platform/Workers/Deletes.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 964eddaad3..7df2770ac6 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -743,10 +743,10 @@ class Deletes extends Action Console::info("Deleting CSV export files older than " . $oneWeekAgo); $this->deleteByGroup('bucket_' . $bucket->getSequence(), [ - Query::select([...$this->selects, '$createdAt', 'filename', 'path']), + Query::select([...$this->selects, '$createdAt', 'name', 'path']), Query::equal('bucketId', ['default']), - Query::endsWith('filename', ['.csv']), Query::createdBefore($oneWeekAgo), + Query::endsWith('name', ['.csv']), Query::orderDesc('$createdAt'), Query::orderDesc(), ], $dbForPlatform, function (Document $file) use ($deviceForFiles) { From 72c71785a60a31579c4a7b90eef316252bc9abdf Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 14 Nov 2025 16:35:10 +1300 Subject: [PATCH 14/16] Fix doc --- src/Appwrite/Platform/Workers/Migrations.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 875dca57ce..0bd5c50e04 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -546,7 +546,7 @@ class Migrations extends Action * * @param bool $success Whether the export was successful * @param Document $project - * @param string $user The user who triggered the operation + * @param Document $user The user who triggered the operation * @param array $options Migration options * @param Mail $queueForMails * @param string $downloadUrl Download URL for successful exports From 2ab36f4cd6d74001bbd91dac2eb0962799d9d6e0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 14 Nov 2025 16:40:00 +1300 Subject: [PATCH 15/16] Use helper for tests --- .../TablesDB/Transactions/TransactionsBase.php | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php b/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php index 09f62d165d..7e223644b2 100644 --- a/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php +++ b/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php @@ -2,6 +2,7 @@ namespace Tests\E2E\Services\Databases\TablesDB\Transactions; +use Appwrite\Operator; use Tests\E2E\Client; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -5645,7 +5646,7 @@ trait TransactionsBase ], $this->getHeaders()), [ 'transactionId' => $transactionId, 'data' => [ - 'items' => '{"method":"arrayRemove","attribute":"","values":["item2"]}' + 'items' => Operator::arrayRemove('item2') ] ]); @@ -5658,7 +5659,7 @@ trait TransactionsBase ], $this->getHeaders()), [ 'transactionId' => $transactionId, 'data' => [ - 'items' => '{"method":"arrayInsert","attribute":"","values":[2,"newItem"]}' + 'items' => Operator::arrayInsert(2, 'newItem') ] ]); @@ -5773,7 +5774,7 @@ trait TransactionsBase 'tableId' => $tableId, 'rowId' => 'doc1', 'data' => [ - 'tags' => '{"method":"arrayRemove","attribute":"","values":["javascript"]}' + 'tags' => Operator::arrayRemove('javascript') ] ], [ @@ -5782,7 +5783,7 @@ trait TransactionsBase 'tableId' => $tableId, 'rowId' => 'doc1', 'data' => [ - 'tags' => '{"method":"arrayAppend","attribute":"","values":["go","rust"]}' + 'tags' => Operator::arrayAppend(['go', 'rust']) ] ] ] @@ -5905,7 +5906,7 @@ trait TransactionsBase ], $this->getHeaders()), [ 'transactionId' => $transactionId, 'data' => [ - 'list1' => '{"method":"arrayPrepend","attribute":"","values":["z"]}' + 'list1' => Operator::arrayPrepend(['z']) ] ]); @@ -5916,7 +5917,7 @@ trait TransactionsBase ], $this->getHeaders()), [ 'transactionId' => $transactionId, 'data' => [ - 'list2' => '{"method":"arrayAppend","attribute":"","values":["w"]}' + 'list2' => Operator::arrayAppend(['w']) ] ]); @@ -5927,7 +5928,7 @@ trait TransactionsBase ], $this->getHeaders()), [ 'transactionId' => $transactionId, 'data' => [ - 'list3' => '{"method":"arrayRemove","attribute":"","values":["3"]}' + 'list3' => Operator::arrayRemove('3') ] ]); From ab90563989a85e83b156b39d22754bdb9bd1ff37 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 14 Nov 2025 17:31:17 +1300 Subject: [PATCH 16/16] Fix tests --- .../TablesDB/Transactions/TransactionsBase.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php b/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php index 7e223644b2..488dc60239 100644 --- a/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php +++ b/tests/e2e/Services/Databases/TablesDB/Transactions/TransactionsBase.php @@ -2,11 +2,11 @@ namespace Tests\E2E\Services\Databases\TablesDB\Transactions; -use Appwrite\Operator; use Tests\E2E\Client; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; +use Utopia\Database\Operator; use Utopia\Database\Query; trait TransactionsBase @@ -5646,7 +5646,7 @@ trait TransactionsBase ], $this->getHeaders()), [ 'transactionId' => $transactionId, 'data' => [ - 'items' => Operator::arrayRemove('item2') + 'items' => Operator::arrayRemove('item2')->toString() ] ]); @@ -5659,7 +5659,7 @@ trait TransactionsBase ], $this->getHeaders()), [ 'transactionId' => $transactionId, 'data' => [ - 'items' => Operator::arrayInsert(2, 'newItem') + 'items' => Operator::arrayInsert(2, 'newItem')->toString() ] ]); @@ -5774,7 +5774,7 @@ trait TransactionsBase 'tableId' => $tableId, 'rowId' => 'doc1', 'data' => [ - 'tags' => Operator::arrayRemove('javascript') + 'tags' => Operator::arrayRemove('javascript')->toString() ] ], [ @@ -5783,7 +5783,7 @@ trait TransactionsBase 'tableId' => $tableId, 'rowId' => 'doc1', 'data' => [ - 'tags' => Operator::arrayAppend(['go', 'rust']) + 'tags' => Operator::arrayAppend(['go', 'rust'])->toString() ] ] ] @@ -5906,7 +5906,7 @@ trait TransactionsBase ], $this->getHeaders()), [ 'transactionId' => $transactionId, 'data' => [ - 'list1' => Operator::arrayPrepend(['z']) + 'list1' => Operator::arrayPrepend(['z'])->toString() ] ]); @@ -5917,7 +5917,7 @@ trait TransactionsBase ], $this->getHeaders()), [ 'transactionId' => $transactionId, 'data' => [ - 'list2' => Operator::arrayAppend(['w']) + 'list2' => Operator::arrayAppend(['w'])->toString() ] ]); @@ -5928,7 +5928,7 @@ trait TransactionsBase ], $this->getHeaders()), [ 'transactionId' => $transactionId, 'data' => [ - 'list3' => Operator::arrayRemove('3') + 'list3' => Operator::arrayRemove('3')->toString() ] ]);