From 61ca4e3969827ddcd6d5a90d6e47ae244a57b258 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 4 May 2026 14:15:01 +0400 Subject: [PATCH 01/25] Fix parallel storage chunk upload state --- composer.json | 7 + composer.lock | 79 +++- .../Storage/Http/Buckets/Files/Create.php | 423 ++++++++++-------- tests/e2e/Services/Storage/StorageBase.php | 151 +++++++ 4 files changed, 471 insertions(+), 189 deletions(-) diff --git a/composer.json b/composer.json index 735955d980..a5b00a8b8b 100644 --- a/composer.json +++ b/composer.json @@ -72,6 +72,7 @@ "utopia-php/validators": "0.2.*", "utopia-php/image": "0.8.*", "utopia-php/locale": "0.8.*", + "utopia-php/lock": "dev-main", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.22.*", "utopia-php/migration": "1.9.*", @@ -111,6 +112,12 @@ "provide": { "ext-phpiredis": "*" }, + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/utopia-php/lock" + } + ], "config": { "platform": { }, diff --git a/composer.lock b/composer.lock index 9f34d2dbe1..4d920f1eee 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "bd45829c252971301370d62300be106d", + "content-hash": "fab1d5b01931f0e2545c36c50b3963b7", "packages": [ { "name": "adhocore/jwt", @@ -4423,6 +4423,79 @@ }, "time": "2025-08-12T12:58:26+00:00" }, + { + "name": "utopia-php/lock", + "version": "dev-main", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/lock.git", + "reference": "22eda789528da4cc71e767c9ad9e9fd4fa2c9467" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/lock/zipball/22eda789528da4cc71e767c9ad9e9fd4fa2c9467", + "reference": "22eda789528da4cc71e767c9ad9e9fd4fa2c9467", + "shasum": "" + }, + "require": { + "php": ">=8.3" + }, + "require-dev": { + "laravel/pint": "1.*", + "phpstan/phpstan": "2.*", + "phpunit/phpunit": "11.*", + "swoole/ide-helper": "*" + }, + "suggest": { + "ext-pcntl": "Required to run the File lock tests", + "ext-redis": "Required for the Distributed lock", + "ext-swoole": "Required for the Mutex and Semaphore locks (>=6.0)" + }, + "default-branch": true, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Lock\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\Lock\\Tests\\": "tests/" + } + }, + "scripts": { + "test": [ + "vendor/bin/phpunit" + ], + "lint": [ + "vendor/bin/pint --test" + ], + "format": [ + "vendor/bin/pint" + ], + "format:check": [ + "vendor/bin/pint --test" + ], + "analyze": [ + "vendor/bin/phpstan analyse --memory-limit=512M" + ] + }, + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Appwrite Team", + "email": "team@appwrite.io" + } + ], + "description": "A simple lock library to coordinate access to shared resources across coroutines, processes and hosts", + "support": { + "source": "https://github.com/utopia-php/lock/tree/main", + "issues": "https://github.com/utopia-php/lock/issues" + }, + "time": "2026-04-28T10:07:10+00:00" + }, { "name": "utopia-php/logger", "version": "0.6.2", @@ -8444,7 +8517,9 @@ ], "aliases": [], "minimum-stability": "dev", - "stability-flags": {}, + "stability-flags": { + "utopia-php/lock": 20 + }, "prefer-stable": true, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index 2ce5ef97f5..f9854a17a4 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -29,6 +29,8 @@ use Utopia\Database\Validator\Authorization\Input; use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; +use Utopia\Lock\Distributed; +use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; use Utopia\Storage\Device; @@ -86,12 +88,14 @@ class Create extends Action ->inject('request') ->inject('response') ->inject('dbForProject') + ->inject('project') ->inject('user') ->inject('queueForEvents') ->inject('mode') ->inject('deviceForFiles') ->inject('deviceForLocal') ->inject('authorization') + ->inject('redis') ->callback($this->action(...)); } @@ -103,12 +107,14 @@ class Create extends Action Request $request, Response $response, Database $dbForProject, + Document $project, User $user, Event $queueForEvents, string $mode, Device $deviceForFiles, Device $deviceForLocal, - Authorization $authorization + Authorization $authorization, + \Redis $redis ) { $bucket = $authorization->skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); @@ -234,24 +240,43 @@ class Create extends Action $path = $deviceForFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); $path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + $lock = new Distributed( + $redis, + 'storage:file:' . $project->getId() . ':' . $bucket->getId() . ':' . $fileId, + ttl: 600, + ); $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; - if (!$file->isEmpty()) { - $chunks = $file->getAttribute('chunksTotal', 1); - $uploaded = $file->getAttribute('chunksUploaded', 0); - $metadata = $file->getAttribute('metadata', []); + $completed = false; - if ($uploaded === $chunks) { - if (empty($contentRange)) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + try { + $lock->withLock(function () use ($bucket, &$chunks, $contentRange, $dbForProject, $fileId, &$metadata, &$completed, $response): void { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + if (!$file->isEmpty()) { + $chunks = $file->getAttribute('chunksTotal', 1); + $uploaded = $file->getAttribute('chunksUploaded', 0); + $metadata = $file->getAttribute('metadata', []); + + if ($uploaded === $chunks) { + if (empty($contentRange)) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } + + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->dynamic($file, Response::MODEL_FILE); + + $completed = true; + return; + } } + }, timeout: 120.0); + } catch (LockContention) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'File upload is busy. Try again.'); + } - $response - ->setStatusCode(Response::STATUS_CODE_OK) - ->dynamic($file, Response::MODEL_FILE); - return; - } + if ($completed) { + return; } $chunksUploaded = $deviceForFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata); @@ -260,187 +285,211 @@ class Create extends Action throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file'); } - if ($chunksUploaded === $chunks) { - if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) { - $antivirus = new Network( - System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), - (int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) - ); + try { + $lock->withLock(function () use ($authorization, $bucket, &$chunks, $chunksUploaded, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $queueForEvents, $response): void { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - if (!$antivirus->fileScan($path)) { - $deviceForFiles->delete($path); - throw new Exception(Exception::STORAGE_INVALID_FILE); + if (!$file->isEmpty()) { + $chunks = $file->getAttribute('chunksTotal', 1); + $uploaded = $file->getAttribute('chunksUploaded', 0); + $metadata = \array_merge($file->getAttribute('metadata', []), $metadata); + + if ($uploaded === $chunks) { + if (empty($contentRange)) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } + + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->dynamic($file, Response::MODEL_FILE); + + return; + } } - } - $mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption - $fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption - $data = ''; - $iv = ''; - $tag = null; - // Compression - $algorithm = $bucket->getAttribute('compression', Compression::NONE); - if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) { - $data = $deviceForFiles->read($path); - switch ($algorithm) { - case Compression::ZSTD: - $compressor = new Zstd(); - break; - case Compression::GZIP: - default: - $compressor = new GZIP(); - break; + if ($chunksUploaded === $chunks) { + if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) { + $antivirus = new Network( + System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), + (int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) + ); + + if (!$antivirus->fileScan($path)) { + $deviceForFiles->delete($path); + throw new Exception(Exception::STORAGE_INVALID_FILE); + } + } + + $mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption + $fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption + $data = ''; + $iv = ''; + $tag = null; + // Compression + $algorithm = $bucket->getAttribute('compression', Compression::NONE); + if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) { + $data = $deviceForFiles->read($path); + switch ($algorithm) { + case Compression::ZSTD: + $compressor = new Zstd(); + break; + case Compression::GZIP: + default: + $compressor = new GZIP(); + break; + } + $data = $compressor->compress($data); + } else { + // reset the algorithm to none as we do not compress the file + // if file size exceedes the APP_STORAGE_READ_BUFFER + // regardless the bucket compression algoorithm + $algorithm = Compression::NONE; + } + + if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { + if (empty($data)) { + $data = $deviceForFiles->read($path); + } + $key = System::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); + } + + if (!empty($data)) { + if (!$deviceForFiles->write($path, $data, $mimeType)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file'); + } + } + + $sizeActual = $deviceForFiles->getFileSize($path); + + $openSSLVersion = null; + $openSSLCipher = null; + $openSSLTag = null; + $openSSLIV = null; + + if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { + $openSSLVersion = '1'; + $openSSLCipher = OpenSSL::CIPHER_AES_128_GCM; + $openSSLTag = \bin2hex($tag); + $openSSLIV = \bin2hex($iv); + } + + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => $fileId, + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $fileName, + 'path' => $path, + 'signature' => $fileHash, + 'mimeType' => $mimeType, + 'sizeOriginal' => $fileSize, + 'sizeActual' => $sizeActual, + 'algorithm' => $algorithm, + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'openSSLVersion' => $openSSLVersion, + 'openSSLCipher' => $openSSLCipher, + 'openSSLTag' => $openSSLTag, + 'openSSLIV' => $openSSLIV, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); + + try { + $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } else { + /** + * Skip authorization in updateDocument. + * Without this, the file creation will fail when user doesn't have update permission. + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we rely on the create-permission check performed earlier. + */ + $file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([ + '$permissions' => $permissions, + 'signature' => $fileHash, + 'mimeType' => $mimeType, + 'sizeActual' => $sizeActual, + 'algorithm' => $algorithm, + 'openSSLVersion' => $openSSLVersion, + 'openSSLCipher' => $openSSLCipher, + 'openSSLTag' => $openSSLTag, + 'openSSLIV' => $openSSLIV, + 'metadata' => $metadata, + 'chunksUploaded' => $chunksUploaded, + ]))); + } + + // Trigger after create success hook + $this->afterCreateSuccess($file); + } else { + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => ID::custom($fileId), + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $fileName, + 'path' => $path, + 'signature' => '', + 'mimeType' => '', + 'sizeOriginal' => $fileSize, + 'sizeActual' => 0, + 'algorithm' => '', + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); + + try { + $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } else { + /** + * Skip authorization in updateDocument. + * Without this, the file creation will fail when user doesn't have update permission. + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we rely on the create-permission check performed earlier. + */ + try { + $file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([ + 'chunksUploaded' => $chunksUploaded, + 'metadata' => $metadata, + ]))); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } } - $data = $compressor->compress($data); - } else { - // reset the algorithm to none as we do not compress the file - // if file size exceedes the APP_STORAGE_READ_BUFFER - // regardless the bucket compression algoorithm - $algorithm = Compression::NONE; - } - if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { - if (empty($data)) { - $data = $deviceForFiles->read($path); - } - $key = System::getEnv('_APP_OPENSSL_KEY_V1'); - $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); - $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); - } + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setParam('fileId', $file->getId()) + ->setContext('bucket', $bucket); - if (!empty($data)) { - if (!$deviceForFiles->write($path, $data, $mimeType)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file'); - } - } + $metadata = null; // was causing leaks as it was passed by reference - $sizeActual = $deviceForFiles->getFileSize($path); - - $openSSLVersion = null; - $openSSLCipher = null; - $openSSLTag = null; - $openSSLIV = null; - - if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { - $openSSLVersion = '1'; - $openSSLCipher = OpenSSL::CIPHER_AES_128_GCM; - $openSSLTag = \bin2hex($tag); - $openSSLIV = \bin2hex($iv); - } - - if ($file->isEmpty()) { - $doc = new Document([ - '$id' => $fileId, - '$permissions' => $permissions, - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getSequence(), - 'name' => $fileName, - 'path' => $path, - 'signature' => $fileHash, - 'mimeType' => $mimeType, - 'sizeOriginal' => $fileSize, - 'sizeActual' => $sizeActual, - 'algorithm' => $algorithm, - 'comment' => '', - 'chunksTotal' => $chunks, - 'chunksUploaded' => $chunksUploaded, - 'openSSLVersion' => $openSSLVersion, - 'openSSLCipher' => $openSSLCipher, - 'openSSLTag' => $openSSLTag, - 'openSSLIV' => $openSSLIV, - 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => $metadata, - ]); - - try { - $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); - } catch (DuplicateException) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } else { - $file = $file - ->setAttribute('$permissions', $permissions) - ->setAttribute('signature', $fileHash) - ->setAttribute('mimeType', $mimeType) - ->setAttribute('sizeActual', $sizeActual) - ->setAttribute('algorithm', $algorithm) - ->setAttribute('openSSLVersion', $openSSLVersion) - ->setAttribute('openSSLCipher', $openSSLCipher) - ->setAttribute('openSSLTag', $openSSLTag) - ->setAttribute('openSSLIV', $openSSLIV) - ->setAttribute('metadata', $metadata) - ->setAttribute('chunksUploaded', $chunksUploaded); - - /** - * Skip authorization in updateDocument. - * Without this, the file creation will fail when user doesn't have update permission. - * However as with chunk upload even if we are updating, we are essentially creating a file - * adding it's new chunk so we rely on the create-permission check performed earlier. - */ - $file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); - } - - // Trigger after create success hook - $this->afterCreateSuccess($file); - } else { - if ($file->isEmpty()) { - $doc = new Document([ - '$id' => ID::custom($fileId), - '$permissions' => $permissions, - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getSequence(), - 'name' => $fileName, - 'path' => $path, - 'signature' => '', - 'mimeType' => '', - 'sizeOriginal' => $fileSize, - 'sizeActual' => 0, - 'algorithm' => '', - 'comment' => '', - 'chunksTotal' => $chunks, - 'chunksUploaded' => $chunksUploaded, - 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => $metadata, - ]); - - try { - $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); - } catch (DuplicateException) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } else { - $file = $file - ->setAttribute('chunksUploaded', $chunksUploaded) - ->setAttribute('metadata', $metadata); - - /** - * Skip authorization in updateDocument. - * Without this, the file creation will fail when user doesn't have update permission. - * However as with chunk upload even if we are updating, we are essentially creating a file - * adding it's new chunk so we rely on the create-permission check performed earlier. - */ - try { - $file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file)); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($file, Response::MODEL_FILE); + }, timeout: 120.0); + } catch (LockContention) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'File upload is busy. Try again.'); } - - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ->setParam('fileId', $file->getId()) - ->setContext('bucket', $bucket); - - $metadata = null; // was causing leaks as it was passed by reference - - $response - ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($file, Response::MODEL_FILE); } /** diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 29f7d70435..a7f1693214 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -1374,6 +1374,157 @@ trait StorageBase ]); } + public function testCreateBucketFileParallelChunksLargeFile(): void + { + $totalSize = (int) ($_ENV['APPWRITE_TEST_PARALLEL_UPLOAD_SIZE'] ?? 20 * 1024 * 1024); + $chunkSize = 5 * 1024 * 1024; + $chunksTotal = (int) ceil($totalSize / $chunkSize); + + $this->assertGreaterThanOrEqual(4, $chunksTotal, 'Test file must span at least 4 chunks'); + + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket Parallel Chunk Upload', + 'fileSecurity' => true, + 'maximumFileSize' => $totalSize, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $bucket['headers']['status-code']); + + $bucketId = $bucket['body']['$id']; + $fileId = ID::unique(); + $tmpDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'appwrite-parallel-upload-' . $fileId; + $source = $tmpDirectory . DIRECTORY_SEPARATOR . 'large-parallel-upload.bin'; + + mkdir($tmpDirectory); + + try { + $handle = fopen($source, 'wb'); + $this->assertNotFalse($handle, 'Could not create test file'); + + $remaining = $totalSize; + $block = str_repeat(hash('sha256', $fileId, binary: true), 1024); + while ($remaining > 0) { + $bytes = substr($block, 0, min(strlen($block), $remaining)); + fwrite($handle, $bytes); + $remaining -= strlen($bytes); + } + fclose($handle); + + $multi = curl_multi_init(); + $handles = []; + + $sourceHandle = fopen($source, 'rb'); + $this->assertNotFalse($sourceHandle, 'Could not open test file'); + + for ($i = 0; $i < $chunksTotal; $i++) { + $start = $i * $chunkSize; + $end = min($start + $chunkSize, $totalSize) - 1; + $length = $end - $start + 1; + $chunkPath = $tmpDirectory . DIRECTORY_SEPARATOR . 'chunk-' . $i . '.part'; + + fseek($sourceHandle, $start); + file_put_contents($chunkPath, fread($sourceHandle, $length)); + + $headers = array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + 'content-range' => 'bytes ' . $start . '-' . $end . '/' . $totalSize, + ]); + + $formattedHeaders = []; + foreach ($headers as $key => $value) { + $formattedHeaders[] = $key . ': ' . $value; + } + + $ch = curl_init($this->client->getEndpoint() . '/storage/buckets/' . $bucketId . '/files'); + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, Client::METHOD_POST); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); + curl_setopt($ch, CURLOPT_HTTPHEADER, $formattedHeaders); + curl_setopt($ch, CURLOPT_POSTFIELDS, [ + 'fileId' => $fileId, + 'file' => new CURLFile($chunkPath, 'application/octet-stream', 'large-parallel-upload.bin'), + 'permissions[0]' => Permission::read(Role::any()), + 'permissions[1]' => Permission::delete(Role::any()), + ]); + curl_setopt($ch, CURLOPT_TIMEOUT, 300); + $handles[] = $ch; + curl_multi_add_handle($multi, $ch); + } + fclose($sourceHandle); + + do { + $status = curl_multi_exec($multi, $running); + if ($running > 0) { + curl_multi_select($multi, 1.0); + } + } while ($running > 0 && $status === CURLM_OK); + + foreach ($handles as $ch) { + $body = curl_multi_getcontent($ch); + $statusCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + $error = curl_error($ch); + curl_multi_remove_handle($multi, $ch); + + $this->assertSame('', $error); + $this->assertContains($statusCode, [200, 201], $body); + } + curl_multi_close($multi); + + $uploadedFile = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ])); + + $this->assertEquals(200, $uploadedFile['headers']['status-code']); + $this->assertEquals($chunksTotal, $uploadedFile['body']['chunksTotal']); + $this->assertEquals($chunksTotal, $uploadedFile['body']['chunksUploaded']); + + $download = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ])); + + $this->assertEquals(200, $download['headers']['status-code']); + $this->assertEquals($totalSize, strlen($download['body'])); + $this->assertEquals(hash_file('sha256', $source), hash('sha256', $download['body'])); + } finally { + if (isset($bucketId)) { + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $fileId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ])); + + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + + foreach (glob($tmpDirectory . DIRECTORY_SEPARATOR . '*') ?: [] as $file) { + unlink($file); + } + + if (is_dir($tmpDirectory)) { + rmdir($tmpDirectory); + } + } + } + public function testDeleteBucketFile(): void { // Create a fresh file just for deletion testing (not using cache since we delete it) From 3db776e2e5a43f59e1f237ae9c7112645bbfcc61 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 4 May 2026 14:24:01 +0400 Subject: [PATCH 02/25] Update parallel upload test concurrency --- tests/e2e/Services/Storage/StorageBase.php | 96 +++++++++++++--------- 1 file changed, 56 insertions(+), 40 deletions(-) diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index a7f1693214..3e2e8b4dea 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -1420,8 +1420,7 @@ trait StorageBase } fclose($handle); - $multi = curl_multi_init(); - $handles = []; + $requests = []; $sourceHandle = fopen($source, 'rb'); $this->assertNotFalse($sourceHandle, 'Could not open test file'); @@ -1435,51 +1434,68 @@ trait StorageBase fseek($sourceHandle, $start); file_put_contents($chunkPath, fread($sourceHandle, $length)); - $headers = array_merge([ - 'content-type' => 'multipart/form-data', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'], - 'content-range' => 'bytes ' . $start . '-' . $end . '/' . $totalSize, - ]); - - $formattedHeaders = []; - foreach ($headers as $key => $value) { - $formattedHeaders[] = $key . ': ' . $value; - } - - $ch = curl_init($this->client->getEndpoint() . '/storage/buckets/' . $bucketId . '/files'); - curl_setopt($ch, CURLOPT_CUSTOMREQUEST, Client::METHOD_POST); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); - curl_setopt($ch, CURLOPT_HTTPHEADER, $formattedHeaders); - curl_setopt($ch, CURLOPT_POSTFIELDS, [ - 'fileId' => $fileId, - 'file' => new CURLFile($chunkPath, 'application/octet-stream', 'large-parallel-upload.bin'), - 'permissions[0]' => Permission::read(Role::any()), - 'permissions[1]' => Permission::delete(Role::any()), - ]); - curl_setopt($ch, CURLOPT_TIMEOUT, 300); - $handles[] = $ch; - curl_multi_add_handle($multi, $ch); + $requests[] = [ + 'headers' => [ + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + 'content-range' => 'bytes ' . $start . '-' . $end . '/' . $totalSize, + ], + 'chunkPath' => $chunkPath, + ]; } fclose($sourceHandle); - do { - $status = curl_multi_exec($multi, $running); - if ($running > 0) { - curl_multi_select($multi, 1.0); + $responses = []; + $endpoint = parse_url($this->client->getEndpoint()); + $scheme = $endpoint['scheme'] ?? 'http'; + $host = $endpoint['host'] ?? 'appwrite'; + $port = $endpoint['port'] ?? ($scheme === 'https' ? 443 : 80); + $basePath = rtrim($endpoint['path'] ?? '', '/'); + + \Swoole\Coroutine\run(function () use ($basePath, $bucketId, $fileId, $host, $port, $requests, $scheme, &$responses): void { + $wg = new \Swoole\Coroutine\WaitGroup(); + + foreach ($requests as $index => $request) { + $wg->add(); + \Swoole\Coroutine::create(function () use ($basePath, $bucketId, $fileId, $host, $index, $port, $request, &$responses, $scheme, $wg): void { + $client = new \Swoole\Coroutine\Http\Client($host, (int) $port, $scheme === 'https'); + $client->set([ + 'timeout' => 300, + 'ssl_verify_peer' => false, + 'ssl_verify_host' => false, + ]); + $client->setHeaders($request['headers']); + $client->setMethod(Client::METHOD_POST); + $client->setData([ + 'fileId' => $fileId, + 'permissions[0]' => Permission::read(Role::any()), + 'permissions[1]' => Permission::delete(Role::any()), + ]); + $client->addFile($request['chunkPath'], 'file', 'application/octet-stream', 'large-parallel-upload.bin'); + $client->execute($basePath . '/storage/buckets/' . $bucketId . '/files'); + + try { + $responses[$index] = [ + 'body' => $client->body, + 'error' => $client->errMsg, + 'statusCode' => $client->statusCode, + ]; + } finally { + $client->close(); + $wg->done(); + } + }); } - } while ($running > 0 && $status === CURLM_OK); - foreach ($handles as $ch) { - $body = curl_multi_getcontent($ch); - $statusCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - $error = curl_error($ch); - curl_multi_remove_handle($multi, $ch); + $wg->wait(); + }); - $this->assertSame('', $error); - $this->assertContains($statusCode, [200, 201], $body); + ksort($responses); + + foreach ($responses as $response) { + $this->assertSame('', $response['error']); + $this->assertContains($response['statusCode'], [200, 201], (string) $response['body']); } - curl_multi_close($multi); $uploadedFile = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId, array_merge([ 'content-type' => 'application/json', From 9927e4bd8e878bbe60e8f75131b7f6efa139aeef Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 4 May 2026 14:34:42 +0400 Subject: [PATCH 03/25] Fix storage upload lock contention handling --- .../Modules/Storage/Http/Buckets/Files/Create.php | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index f9854a17a4..4e98c4c8d5 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -243,7 +243,7 @@ class Create extends Action $lock = new Distributed( $redis, 'storage:file:' . $project->getId() . ':' . $bucket->getId() . ':' . $fileId, - ttl: 600, + ttl: 120, ); $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; @@ -272,7 +272,8 @@ class Create extends Action } }, timeout: 120.0); } catch (LockContention) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'File upload is busy. Try again.'); + $response->addHeader('Retry-After', '5'); + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'File upload is busy. Try again.'); } if ($completed) { @@ -288,6 +289,7 @@ class Create extends Action try { $lock->withLock(function () use ($authorization, $bucket, &$chunks, $chunksUploaded, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $queueForEvents, $response): void { $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + $uploaded = 0; if (!$file->isEmpty()) { $chunks = $file->getAttribute('chunksTotal', 1); @@ -307,7 +309,9 @@ class Create extends Action } } - if ($chunksUploaded === $chunks) { + $chunksUploaded = max($uploaded, $chunksUploaded); + + if ($chunksUploaded === $chunks && $uploaded < $chunks) { if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) { $antivirus = new Network( System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), @@ -488,7 +492,8 @@ class Create extends Action ->dynamic($file, Response::MODEL_FILE); }, timeout: 120.0); } catch (LockContention) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'File upload is busy. Try again.'); + $response->addHeader('Retry-After', '5'); + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'File upload is busy. Try again.'); } } From d58929b621b4c227c0595c4d3b42696ee3290624 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 4 May 2026 15:00:16 +0400 Subject: [PATCH 04/25] Restore storage upload lock TTL --- .../Platform/Modules/Storage/Http/Buckets/Files/Create.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index 4e98c4c8d5..a85fb72a23 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -91,7 +91,6 @@ class Create extends Action ->inject('project') ->inject('user') ->inject('queueForEvents') - ->inject('mode') ->inject('deviceForFiles') ->inject('deviceForLocal') ->inject('authorization') @@ -110,7 +109,6 @@ class Create extends Action Document $project, User $user, Event $queueForEvents, - string $mode, Device $deviceForFiles, Device $deviceForLocal, Authorization $authorization, @@ -243,7 +241,7 @@ class Create extends Action $lock = new Distributed( $redis, 'storage:file:' . $project->getId() . ':' . $bucket->getId() . ':' . $fileId, - ttl: 120, + ttl: 600, ); $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; From 6e19db130e03e09dcdfc68eb3af572162247ff5e Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 4 May 2026 15:09:52 +0400 Subject: [PATCH 05/25] Fix storage chunk upload events --- .../Storage/Http/Buckets/Files/Create.php | 10 ++-- tests/e2e/Services/Storage/StorageBase.php | 55 +++++++++++-------- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index a85fb72a23..c26e30474f 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -478,10 +478,12 @@ class Create extends Action } } - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ->setParam('fileId', $file->getId()) - ->setContext('bucket', $bucket); + if ($chunksUploaded === $chunks) { + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setParam('fileId', $file->getId()) + ->setContext('bucket', $bucket); + } $metadata = null; // was causing leaks as it was passed by reference diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 3e2e8b4dea..009910fb38 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -1458,30 +1458,41 @@ trait StorageBase foreach ($requests as $index => $request) { $wg->add(); \Swoole\Coroutine::create(function () use ($basePath, $bucketId, $fileId, $host, $index, $port, $request, &$responses, $scheme, $wg): void { - $client = new \Swoole\Coroutine\Http\Client($host, (int) $port, $scheme === 'https'); - $client->set([ - 'timeout' => 300, - 'ssl_verify_peer' => false, - 'ssl_verify_host' => false, - ]); - $client->setHeaders($request['headers']); - $client->setMethod(Client::METHOD_POST); - $client->setData([ - 'fileId' => $fileId, - 'permissions[0]' => Permission::read(Role::any()), - 'permissions[1]' => Permission::delete(Role::any()), - ]); - $client->addFile($request['chunkPath'], 'file', 'application/octet-stream', 'large-parallel-upload.bin'); - $client->execute($basePath . '/storage/buckets/' . $bucketId . '/files'); - try { - $responses[$index] = [ - 'body' => $client->body, - 'error' => $client->errMsg, - 'statusCode' => $client->statusCode, - ]; + for ($attempt = 0; $attempt < 3; $attempt++) { + $client = new \Swoole\Coroutine\Http\Client($host, (int) $port, $scheme === 'https'); + $client->set([ + 'timeout' => 300, + 'ssl_verify_peer' => false, + 'ssl_verify_host' => false, + ]); + $client->setHeaders($request['headers']); + $client->setMethod(Client::METHOD_POST); + $client->setData([ + 'fileId' => $fileId, + 'permissions[0]' => Permission::read(Role::any()), + 'permissions[1]' => Permission::delete(Role::any()), + ]); + $client->addFile($request['chunkPath'], 'file', 'application/octet-stream', 'large-parallel-upload.bin'); + $client->execute($basePath . '/storage/buckets/' . $bucketId . '/files'); + + $responses[$index] = [ + 'body' => $client->body, + 'error' => $client->errMsg, + 'headers' => $client->headers ?? [], + 'statusCode' => $client->statusCode, + ]; + + $client->close(); + + if ($responses[$index]['statusCode'] !== 429) { + break; + } + + $retryAfter = (float) ($responses[$index]['headers']['retry-after'] ?? 0.1); + \Swoole\Coroutine::sleep(max($retryAfter, 0.1)); + } } finally { - $client->close(); $wg->done(); } }); From fd830902157b19c6d178b8a4fd09c6f765eebdec Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 4 May 2026 17:34:53 +0400 Subject: [PATCH 06/25] Fix parallel deployment chunk uploads --- .../Functions/Http/Deployments/Create.php | 278 +++++++----- .../Modules/Sites/Http/Deployments/Create.php | 398 ++++++++++-------- .../Functions/FunctionsCustomServerTest.php | 138 ++++++ .../Services/Sites/SitesCustomServerTest.php | 139 ++++++ tests/e2e/Services/Storage/StorageBase.php | 2 +- 5 files changed, 665 insertions(+), 290 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php index 757edc0484..ac735a8b0a 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php @@ -20,6 +20,8 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; +use Utopia\Lock\Distributed; +use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; use Utopia\Storage\Device; @@ -90,6 +92,7 @@ class Create extends Action ->inject('queueForBuilds') ->inject('plan') ->inject('authorization') + ->inject('redis') ->callback($this->action(...)); } @@ -108,7 +111,8 @@ class Create extends Action Device $deviceForLocal, Build $queueForBuilds, array $plan, - Authorization $authorization + Authorization $authorization, + \Redis $redis ) { $activate = \strval($activate) === 'true' || \strval($activate) === '1'; @@ -190,20 +194,40 @@ class Create extends Action // Save to storage $fileSize ??= $deviceForLocal->getFileSize($fileTmpName); $path = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); - $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + $lockKey = 'functions:deployment:' . $project->getId() . ':' . $functionId . ':' . $deploymentId; + $checkLock = new Distributed($redis, $lockKey, ttl: 120); + $stateLock = new Distributed($redis, $lockKey, ttl: 600); $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; - if (!$deployment->isEmpty()) { - $chunks = $deployment->getAttribute('sourceChunksTotal', 1); - $uploaded = $deployment->getAttribute('sourceChunksUploaded', 0); - $metadata = $deployment->getAttribute('sourceMetadata', []); + $completed = false; - if ($uploaded === $chunks) { - $response - ->setStatusCode(Response::STATUS_CODE_ACCEPTED) - ->dynamic($deployment, Response::MODEL_DEPLOYMENT); - return; - } + try { + $checkLock->withLock(function () use (&$chunks, $contentRange, $dbForProject, $deploymentId, &$metadata, &$completed, $response): void { + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + if (!$deployment->isEmpty()) { + $chunks = $deployment->getAttribute('sourceChunksTotal', 1); + $uploaded = $deployment->getAttribute('sourceChunksUploaded', 0); + $metadata = $deployment->getAttribute('sourceMetadata', []); + + if ($uploaded === $chunks) { + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($deployment, Response::MODEL_DEPLOYMENT); + + $completed = true; + return; + } + } + }, timeout: 120.0); + } catch (LockContention) { + $response->addHeader('Retry-After', '5'); + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Deployment upload is busy. Try again.'); + } + + if ($completed) { + return; } $chunksUploaded = $deviceForFunctions->upload($fileTmpName, $path, $chunk, $chunks, $metadata); @@ -214,115 +238,141 @@ class Create extends Action $type = $request->getHeader('x-sdk-language') === 'cli' ? 'cli' : 'manual'; - if ($chunksUploaded === $chunks) { - if ($activate) { - // Remove deploy for all other deployments. - $activeDeployments = $dbForProject->find('deployments', [ - Query::equal('activate', [true]), - Query::equal('resourceId', [$functionId]), - Query::equal('resourceType', ['functions']) - ]); + try { + $stateLock->withLock(function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $queueForBuilds, $queueForEvents, $response, $type): void { + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + $uploaded = 0; - foreach ($activeDeployments as $activeDeployment) { - $activeDeployment->setAttribute('activate', false); - $dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document([ - 'activate' => false, - ])); + if (!$deployment->isEmpty()) { + $chunks = $deployment->getAttribute('sourceChunksTotal', 1); + $uploaded = $deployment->getAttribute('sourceChunksUploaded', 0); + $metadata = \array_merge($deployment->getAttribute('sourceMetadata', []), $metadata); + + if ($uploaded === $chunks) { + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($deployment, Response::MODEL_DEPLOYMENT); + return; + } } - } - $fileSize = $deviceForFunctions->getFileSize($path); + $chunksUploaded = max($uploaded, $chunksUploaded); - if ($deployment->isEmpty()) { - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceInternalId' => $function->getSequence(), - 'resourceId' => $function->getId(), - 'resourceType' => 'functions', - 'entrypoint' => $entrypoint, - 'buildCommands' => $commands, - 'startCommand' => $function->getAttribute('startCommand', ''), - 'sourcePath' => $path, - 'sourceSize' => $fileSize, - 'totalSize' => $fileSize, - 'sourceChunksTotal' => $chunks, - 'sourceChunksUploaded' => $chunksUploaded, - 'activate' => $activate, - 'sourceMetadata' => $metadata, - 'type' => $type - ])); + if ($chunksUploaded === $chunks && $uploaded < $chunks) { + if ($activate) { + // Remove deploy for all other deployments. + $activeDeployments = $dbForProject->find('deployments', [ + Query::equal('activate', [true]), + Query::equal('resourceId', [$functionId]), + Query::equal('resourceType', ['functions']) + ]); - $function = $dbForProject->updateDocument('functions', $function->getId(), new Document([ - 'latestDeploymentId' => $deployment->getId(), - 'latestDeploymentInternalId' => $deployment->getSequence(), - 'latestDeploymentCreatedAt' => $deployment->getCreatedAt(), - 'latestDeploymentStatus' => $deployment->getAttribute('status', ''), - ])); - } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ - 'sourceSize' => $fileSize, - 'sourceChunksUploaded' => $chunksUploaded, - 'sourceMetadata' => $metadata, - ])); - } + foreach ($activeDeployments as $activeDeployment) { + $dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document([ + 'activate' => false, + ])); + } + } - // Start the build - $queueForBuilds - ->setType(BUILD_TYPE_DEPLOYMENT) - ->setResource($function) - ->setDeployment($deployment); - } else { - if ($deployment->isEmpty()) { - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceInternalId' => $function->getSequence(), - 'resourceId' => $function->getId(), - 'resourceType' => 'functions', - 'entrypoint' => $entrypoint, - 'buildCommands' => $commands, - 'startCommand' => $function->getAttribute('startCommand', ''), - 'sourcePath' => $path, - 'sourceSize' => $fileSize, - 'totalSize' => $fileSize, - 'sourceChunksTotal' => $chunks, - 'sourceChunksUploaded' => $chunksUploaded, - 'activate' => $activate, - 'sourceMetadata' => $metadata, - 'type' => $type - ])); + $fileSize = $deviceForFunctions->getFileSize($path); - $function = $dbForProject->updateDocument('functions', $function->getId(), new Document([ - 'latestDeploymentId' => $deployment->getId(), - 'latestDeploymentInternalId' => $deployment->getSequence(), - 'latestDeploymentCreatedAt' => $deployment->getCreatedAt(), - 'latestDeploymentStatus' => $deployment->getAttribute('status', ''), - ])); - } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ - 'sourceChunksUploaded' => $chunksUploaded, - 'sourceMetadata' => $metadata, - ])); - } + if ($deployment->isEmpty()) { + $deployment = $dbForProject->createDocument('deployments', new Document([ + '$id' => $deploymentId, + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'resourceInternalId' => $function->getSequence(), + 'resourceId' => $function->getId(), + 'resourceType' => 'functions', + 'entrypoint' => $entrypoint, + 'buildCommands' => $commands, + 'startCommand' => $function->getAttribute('startCommand', ''), + 'sourcePath' => $path, + 'sourceSize' => $fileSize, + 'totalSize' => $fileSize, + 'sourceChunksTotal' => $chunks, + 'sourceChunksUploaded' => $chunksUploaded, + 'activate' => $activate, + 'sourceMetadata' => $metadata, + 'type' => $type + ])); + + $function = $dbForProject->updateDocument('functions', $function->getId(), new Document([ + 'latestDeploymentId' => $deployment->getId(), + 'latestDeploymentInternalId' => $deployment->getSequence(), + 'latestDeploymentCreatedAt' => $deployment->getCreatedAt(), + 'latestDeploymentStatus' => $deployment->getAttribute('status', ''), + ])); + } else { + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ + 'sourceSize' => $fileSize, + 'sourceChunksUploaded' => $chunksUploaded, + 'sourceMetadata' => $metadata, + ])); + } + + // Start the build + $queueForBuilds + ->setType(BUILD_TYPE_DEPLOYMENT) + ->setResource($function) + ->setDeployment($deployment); + } else { + if ($deployment->isEmpty()) { + $deployment = $dbForProject->createDocument('deployments', new Document([ + '$id' => $deploymentId, + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'resourceInternalId' => $function->getSequence(), + 'resourceId' => $function->getId(), + 'resourceType' => 'functions', + 'entrypoint' => $entrypoint, + 'buildCommands' => $commands, + 'startCommand' => $function->getAttribute('startCommand', ''), + 'sourcePath' => $path, + 'sourceSize' => $fileSize, + 'totalSize' => $fileSize, + 'sourceChunksTotal' => $chunks, + 'sourceChunksUploaded' => $chunksUploaded, + 'activate' => $activate, + 'sourceMetadata' => $metadata, + 'type' => $type + ])); + + $function = $dbForProject->updateDocument('functions', $function->getId(), new Document([ + 'latestDeploymentId' => $deployment->getId(), + 'latestDeploymentInternalId' => $deployment->getSequence(), + 'latestDeploymentCreatedAt' => $deployment->getCreatedAt(), + 'latestDeploymentStatus' => $deployment->getAttribute('status', ''), + ])); + } else { + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ + 'sourceChunksUploaded' => $chunksUploaded, + 'sourceMetadata' => $metadata, + ])); + } + } + + $metadata = null; + + if ($chunksUploaded === $chunks) { + $queueForEvents + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()); + } + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($deployment, Response::MODEL_DEPLOYMENT); + }, timeout: 120.0); + } catch (LockContention) { + $response->addHeader('Retry-After', '5'); + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Deployment upload is busy. Try again.'); } - - $metadata = null; - - $queueForEvents - ->setParam('functionId', $function->getId()) - ->setParam('deploymentId', $deployment->getId()); - - $response - ->setStatusCode(Response::STATUS_CODE_ACCEPTED) - ->dynamic($deployment, Response::MODEL_DEPLOYMENT); } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php index 71ea5ceb2f..531962d33a 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php @@ -20,6 +20,8 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; +use Utopia\Lock\Distributed; +use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; use Utopia\Storage\Device; @@ -89,6 +91,7 @@ class Create extends Action ->inject('plan') ->inject('authorization') ->inject('platform') + ->inject('redis') ->callback($this->action(...)); } @@ -111,6 +114,7 @@ class Create extends Action array $plan, Authorization $authorization, array $platform, + \Redis $redis, ) { $activate = \strval($activate) === 'true' || \strval($activate) === '1'; @@ -192,20 +196,40 @@ class Create extends Action // Save to storage $fileSize ??= $deviceForLocal->getFileSize($fileTmpName); $path = $deviceForSites->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); - $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + $lockKey = 'sites:deployment:' . $project->getId() . ':' . $siteId . ':' . $deploymentId; + $checkLock = new Distributed($redis, $lockKey, ttl: 120); + $stateLock = new Distributed($redis, $lockKey, ttl: 600); $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; - if (!$deployment->isEmpty()) { - $chunks = $deployment->getAttribute('sourceChunksTotal', 1); - $uploaded = $deployment->getAttribute('sourceChunksUploaded', 0); - $metadata = $deployment->getAttribute('sourceMetadata', []); + $completed = false; - if ($uploaded === $chunks) { - $response - ->setStatusCode(Response::STATUS_CODE_ACCEPTED) - ->dynamic($deployment, Response::MODEL_DEPLOYMENT); - return; - } + try { + $checkLock->withLock(function () use (&$chunks, $dbForProject, $deploymentId, &$metadata, &$completed, $response): void { + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + if (!$deployment->isEmpty()) { + $chunks = $deployment->getAttribute('sourceChunksTotal', 1); + $uploaded = $deployment->getAttribute('sourceChunksUploaded', 0); + $metadata = $deployment->getAttribute('sourceMetadata', []); + + if ($uploaded === $chunks) { + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($deployment, Response::MODEL_DEPLOYMENT); + + $completed = true; + return; + } + } + }, timeout: 120.0); + } catch (LockContention) { + $response->addHeader('Retry-After', '5'); + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Deployment upload is busy. Try again.'); + } + + if ($completed) { + return; } $chunksUploaded = $deviceForSites->upload($fileTmpName, $path, $chunk, $chunks, $metadata); @@ -224,181 +248,205 @@ class Create extends Action $commands[] = $buildCommand; } - if ($chunksUploaded === $chunks) { - if ($activate) { - // Remove deploy for all other deployments. - $activeDeployments = $dbForProject->find('deployments', [ - Query::equal('activate', [true]), - Query::equal('resourceId', [$siteId]), - Query::equal('resourceType', ['sites']) - ]); + try { + $stateLock->withLock(function () use ($activate, $authorization, $commands, &$chunks, $chunksUploaded, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $outputDirectory, $path, $platform, $project, $queueForBuilds, $queueForEvents, $response, &$site, $siteId, $type): void { + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + $uploaded = 0; - foreach ($activeDeployments as $activeDeployment) { - $activeDeployment->setAttribute('activate', false); - $dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document(['activate' => false])); + if (!$deployment->isEmpty()) { + $chunks = $deployment->getAttribute('sourceChunksTotal', 1); + $uploaded = $deployment->getAttribute('sourceChunksUploaded', 0); + $metadata = \array_merge($deployment->getAttribute('sourceMetadata', []), $metadata); + + if ($uploaded === $chunks) { + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($deployment, Response::MODEL_DEPLOYMENT); + return; + } } - } - $fileSize = $deviceForSites->getFileSize($path); + $chunksUploaded = max($uploaded, $chunksUploaded); - if ($deployment->isEmpty()) { - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceInternalId' => $site->getSequence(), - 'resourceId' => $site->getId(), - 'resourceType' => 'sites', - 'buildCommands' => \implode(' && ', $commands), - 'startCommand' => $site->getAttribute('startCommand', ''), - 'buildOutput' => $outputDirectory, - 'adapter' => $site->getAttribute('adapter', ''), - 'fallbackFile' => $site->getAttribute('fallbackFile', ''), - 'sourcePath' => $path, - 'sourceSize' => $fileSize, - 'totalSize' => $fileSize, - 'sourceChunksTotal' => $chunks, - 'sourceChunksUploaded' => $chunksUploaded, - 'activate' => $activate, - 'sourceMetadata' => $metadata, - 'type' => $type, - ])); + if ($chunksUploaded === $chunks && $uploaded < $chunks) { + if ($activate) { + // Remove deploy for all other deployments. + $activeDeployments = $dbForProject->find('deployments', [ + Query::equal('activate', [true]), + Query::equal('resourceId', [$siteId]), + Query::equal('resourceType', ['sites']) + ]); - $site = $site - ->setAttribute('latestDeploymentId', $deployment->getId()) - ->setAttribute('latestDeploymentInternalId', $deployment->getSequence()) - ->setAttribute('latestDeploymentCreatedAt', $deployment->getCreatedAt()) - ->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', '')); - $dbForProject->updateDocument('sites', $site->getId(), new Document([ - 'latestDeploymentId' => $deployment->getId(), - 'latestDeploymentInternalId' => $deployment->getSequence(), - 'latestDeploymentCreatedAt' => $deployment->getCreatedAt(), - 'latestDeploymentStatus' => $deployment->getAttribute('status', ''), - ])); + foreach ($activeDeployments as $activeDeployment) { + $dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document(['activate' => false])); + } + } - $sitesDomain = $platform['sitesDomain']; - $domain = ID::unique() . "." . $sitesDomain; + $fileSize = $deviceForSites->getFileSize($path); - // TODO: (@Meldiron) Remove after 1.7.x migration - $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; - $ruleId = $isMd5 ? md5($domain) : ID::unique(); + if ($deployment->isEmpty()) { + $deployment = $dbForProject->createDocument('deployments', new Document([ + '$id' => $deploymentId, + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'resourceInternalId' => $site->getSequence(), + 'resourceId' => $site->getId(), + 'resourceType' => 'sites', + 'buildCommands' => \implode(' && ', $commands), + 'startCommand' => $site->getAttribute('startCommand', ''), + 'buildOutput' => $outputDirectory, + 'adapter' => $site->getAttribute('adapter', ''), + 'fallbackFile' => $site->getAttribute('fallbackFile', ''), + 'sourcePath' => $path, + 'sourceSize' => $fileSize, + 'totalSize' => $fileSize, + 'sourceChunksTotal' => $chunks, + 'sourceChunksUploaded' => $chunksUploaded, + 'activate' => $activate, + 'sourceMetadata' => $metadata, + 'type' => $type, + ])); - $authorization->skip( - fn () => $dbForPlatform->createDocument('rules', new Document([ - '$id' => $ruleId, - 'projectId' => $project->getId(), - 'projectInternalId' => $project->getSequence(), - 'domain' => $domain, - 'type' => 'deployment', - 'trigger' => 'deployment', - 'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(), - 'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(), - 'deploymentResourceType' => 'site', - 'deploymentResourceId' => $site->getId(), - 'deploymentResourceInternalId' => $site->getSequence(), - 'status' => 'verified', - 'certificateId' => '', - 'search' => implode(' ', [$ruleId, $domain]), - 'owner' => 'Appwrite', - 'region' => $project->getAttribute('region') - ])) - ); - } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ - 'sourceSize' => $fileSize, - 'sourceChunksUploaded' => $chunksUploaded, - 'sourceMetadata' => $metadata, - ])); - } + $site = $site + ->setAttribute('latestDeploymentId', $deployment->getId()) + ->setAttribute('latestDeploymentInternalId', $deployment->getSequence()) + ->setAttribute('latestDeploymentCreatedAt', $deployment->getCreatedAt()) + ->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', '')); + $dbForProject->updateDocument('sites', $site->getId(), new Document([ + 'latestDeploymentId' => $deployment->getId(), + 'latestDeploymentInternalId' => $deployment->getSequence(), + 'latestDeploymentCreatedAt' => $deployment->getCreatedAt(), + 'latestDeploymentStatus' => $deployment->getAttribute('status', ''), + ])); - // Start the build - $queueForBuilds - ->setType(BUILD_TYPE_DEPLOYMENT) - ->setResource($site) - ->setDeployment($deployment); - } else { - if ($deployment->isEmpty()) { - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceInternalId' => $site->getSequence(), - 'resourceId' => $site->getId(), - 'resourceType' => 'sites', - 'buildCommands' => \implode(' && ', $commands), - 'startCommand' => $site->getAttribute('startCommand', ''), - 'buildOutput' => $outputDirectory, - 'adapter' => $site->getAttribute('adapter', ''), - 'fallbackFile' => $site->getAttribute('fallbackFile', ''), - 'sourcePath' => $path, - 'sourceSize' => $fileSize, - 'totalSize' => $fileSize, - 'sourceChunksTotal' => $chunks, - 'sourceChunksUploaded' => $chunksUploaded, - 'activate' => $activate, - 'sourceMetadata' => $metadata, - 'type' => $type, - ])); + $sitesDomain = $platform['sitesDomain']; + $domain = ID::unique() . "." . $sitesDomain; - $site = $site - ->setAttribute('latestDeploymentId', $deployment->getId()) - ->setAttribute('latestDeploymentInternalId', $deployment->getSequence()) - ->setAttribute('latestDeploymentCreatedAt', $deployment->getCreatedAt()) - ->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', '')); - $dbForProject->updateDocument('sites', $site->getId(), new Document([ - 'latestDeploymentId' => $site->getAttribute('latestDeploymentId'), - 'latestDeploymentInternalId' => $site->getAttribute('latestDeploymentInternalId'), - 'latestDeploymentCreatedAt' => $site->getAttribute('latestDeploymentCreatedAt'), - 'latestDeploymentStatus' => $site->getAttribute('latestDeploymentStatus'), - ])); + // TODO: (@Meldiron) Remove after 1.7.x migration + $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5'; + $ruleId = $isMd5 ? md5($domain) : ID::unique(); - $sitesDomain = $platform['sitesDomain']; - $domain = ID::unique() . "." . $sitesDomain; - $ruleId = md5($domain); - $authorization->skip( - fn () => $dbForPlatform->createDocument('rules', new Document([ - '$id' => $ruleId, - 'projectId' => $project->getId(), - 'projectInternalId' => $project->getSequence(), - 'domain' => $domain, - 'type' => 'deployment', - 'trigger' => 'deployment', - 'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(), - 'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(), - 'deploymentResourceType' => 'site', - 'deploymentResourceId' => $site->getId(), - 'deploymentResourceInternalId' => $site->getSequence(), - 'status' => 'verified', - 'certificateId' => '', - 'search' => implode(' ', [$ruleId, $domain]), - 'owner' => 'Appwrite', - 'region' => $project->getAttribute('region') - ])) - ); - } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ - 'sourceChunksUploaded' => $chunksUploaded, - 'sourceMetadata' => $metadata, - ])); - } + $authorization->skip( + fn () => $dbForPlatform->createDocument('rules', new Document([ + '$id' => $ruleId, + 'projectId' => $project->getId(), + 'projectInternalId' => $project->getSequence(), + 'domain' => $domain, + 'type' => 'deployment', + 'trigger' => 'deployment', + 'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(), + 'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(), + 'deploymentResourceType' => 'site', + 'deploymentResourceId' => $site->getId(), + 'deploymentResourceInternalId' => $site->getSequence(), + 'status' => 'verified', + 'certificateId' => '', + 'search' => implode(' ', [$ruleId, $domain]), + 'owner' => 'Appwrite', + 'region' => $project->getAttribute('region') + ])) + ); + } else { + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ + 'sourceSize' => $fileSize, + 'sourceChunksUploaded' => $chunksUploaded, + 'sourceMetadata' => $metadata, + ])); + } + + // Start the build + $queueForBuilds + ->setType(BUILD_TYPE_DEPLOYMENT) + ->setResource($site) + ->setDeployment($deployment); + } else { + if ($deployment->isEmpty()) { + $deployment = $dbForProject->createDocument('deployments', new Document([ + '$id' => $deploymentId, + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'resourceInternalId' => $site->getSequence(), + 'resourceId' => $site->getId(), + 'resourceType' => 'sites', + 'buildCommands' => \implode(' && ', $commands), + 'startCommand' => $site->getAttribute('startCommand', ''), + 'buildOutput' => $outputDirectory, + 'adapter' => $site->getAttribute('adapter', ''), + 'fallbackFile' => $site->getAttribute('fallbackFile', ''), + 'sourcePath' => $path, + 'sourceSize' => $fileSize, + 'totalSize' => $fileSize, + 'sourceChunksTotal' => $chunks, + 'sourceChunksUploaded' => $chunksUploaded, + 'activate' => $activate, + 'sourceMetadata' => $metadata, + 'type' => $type, + ])); + + $site = $site + ->setAttribute('latestDeploymentId', $deployment->getId()) + ->setAttribute('latestDeploymentInternalId', $deployment->getSequence()) + ->setAttribute('latestDeploymentCreatedAt', $deployment->getCreatedAt()) + ->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', '')); + $dbForProject->updateDocument('sites', $site->getId(), new Document([ + 'latestDeploymentId' => $site->getAttribute('latestDeploymentId'), + 'latestDeploymentInternalId' => $site->getAttribute('latestDeploymentInternalId'), + 'latestDeploymentCreatedAt' => $site->getAttribute('latestDeploymentCreatedAt'), + 'latestDeploymentStatus' => $site->getAttribute('latestDeploymentStatus'), + ])); + + $sitesDomain = $platform['sitesDomain']; + $domain = ID::unique() . "." . $sitesDomain; + $ruleId = md5($domain); + $authorization->skip( + fn () => $dbForPlatform->createDocument('rules', new Document([ + '$id' => $ruleId, + 'projectId' => $project->getId(), + 'projectInternalId' => $project->getSequence(), + 'domain' => $domain, + 'type' => 'deployment', + 'trigger' => 'deployment', + 'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(), + 'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(), + 'deploymentResourceType' => 'site', + 'deploymentResourceId' => $site->getId(), + 'deploymentResourceInternalId' => $site->getSequence(), + 'status' => 'verified', + 'certificateId' => '', + 'search' => implode(' ', [$ruleId, $domain]), + 'owner' => 'Appwrite', + 'region' => $project->getAttribute('region') + ])) + ); + } else { + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ + 'sourceChunksUploaded' => $chunksUploaded, + 'sourceMetadata' => $metadata, + ])); + } + } + + $metadata = null; + + if ($chunksUploaded === $chunks) { + $queueForEvents + ->setParam('siteId', $site->getId()) + ->setParam('deploymentId', $deployment->getId()); + } + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($deployment, Response::MODEL_DEPLOYMENT); + }, timeout: 120.0); + } catch (LockContention) { + $response->addHeader('Retry-After', '5'); + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Deployment upload is busy. Try again.'); } - - - - $metadata = null; - - $queueForEvents - ->setParam('siteId', $site->getId()) - ->setParam('deploymentId', $deployment->getId()); - - $response - ->setStatusCode(Response::STATUS_CODE_ACCEPTED) - ->dynamic($deployment, Response::MODEL_DEPLOYMENT); } } diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 899c0ff71f..8e784b3d5b 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -1191,6 +1191,144 @@ class FunctionsCustomServerTest extends Scope }, 120000, 500); } + public function testCreateDeploymentParallelChunksLargeFile(): void + { + $functionId = $this->setupFunction([ + 'functionId' => ID::unique(), + 'name' => 'Test Parallel Chunk Deployment', + 'execute' => [Role::user($this->getUser()['$id'])->toString()], + 'runtime' => 'node-22', + 'entrypoint' => 'index.js', + 'timeout' => 10, + ]); + + $deploymentId = ID::unique(); + $tmpDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'appwrite-parallel-function-deployment-' . $deploymentId; + + mkdir($tmpDirectory); + + try { + copy(__DIR__ . '/../../../resources/functions/basic/index.js', $tmpDirectory . DIRECTORY_SEPARATOR . 'index.js'); + file_put_contents($tmpDirectory . DIRECTORY_SEPARATOR . 'large.bin', random_bytes(20 * 1024 * 1024)); + + $source = $tmpDirectory . DIRECTORY_SEPARATOR . 'code.tar.gz'; + Console::execute('cd ' . $tmpDirectory . ' && tar --exclude code.tar.gz -czf code.tar.gz .', '', $this->stdout, $this->stderr); + + $totalSize = filesize($source); + $chunkSize = 5 * 1024 * 1024; + $chunksTotal = (int) ceil($totalSize / $chunkSize); + + $this->assertGreaterThanOrEqual(4, $chunksTotal, 'Test deployment must span at least 4 chunks'); + + $requests = []; + $sourceHandle = fopen($source, 'rb'); + $this->assertNotFalse($sourceHandle, 'Could not open deployment package'); + + try { + for ($i = 0; $i < $chunksTotal; $i++) { + $start = $i * $chunkSize; + $end = min($start + $chunkSize, $totalSize) - 1; + $length = $end - $start + 1; + $chunkPath = $tmpDirectory . DIRECTORY_SEPARATOR . 'chunk-' . $i . '.part'; + + fseek($sourceHandle, $start); + file_put_contents($chunkPath, fread($sourceHandle, $length)); + + $requests[] = [ + 'headers' => [ + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + 'x-appwrite-id' => $deploymentId, + 'content-range' => 'bytes ' . $start . '-' . $end . '/' . $totalSize, + ], + 'chunkPath' => $chunkPath, + ]; + } + } finally { + fclose($sourceHandle); + } + + $responses = []; + $endpoint = parse_url($this->client->getEndpoint()); + $scheme = $endpoint['scheme'] ?? 'http'; + $host = $endpoint['host'] ?? 'appwrite'; + $port = $endpoint['port'] ?? ($scheme === 'https' ? 443 : 80); + $basePath = rtrim($endpoint['path'] ?? '', '/'); + + \Swoole\Coroutine\run(function () use ($basePath, $functionId, $host, $port, $requests, $scheme, &$responses): void { + $wg = new \Swoole\Coroutine\WaitGroup(); + + foreach ($requests as $index => $request) { + $wg->add(); + \Swoole\Coroutine::create(function () use ($basePath, $functionId, $host, $index, $port, $request, &$responses, $scheme, $wg): void { + try { + for ($attempt = 0; $attempt < 3; $attempt++) { + $client = new \Swoole\Coroutine\Http\Client($host, (int) $port, $scheme === 'https'); + $client->set([ + 'timeout' => 300, + 'ssl_verify_peer' => false, + 'ssl_verify_host' => false, + ]); + $client->setHeaders($request['headers']); + $client->setMethod(Client::METHOD_POST); + $client->setData([ + 'entrypoint' => 'index.js', + 'activate' => true, + ]); + $client->addFile($request['chunkPath'], 'code', 'application/x-gzip', 'code.tar.gz'); + $client->execute($basePath . '/functions/' . $functionId . '/deployments'); + + $responses[$index] = [ + 'body' => $client->body, + 'error' => $client->errMsg, + 'headers' => $client->headers ?? [], + 'statusCode' => $client->statusCode, + ]; + + $client->close(); + + if ($responses[$index]['statusCode'] !== 429) { + break; + } + + $retryAfter = (float) ($responses[$index]['headers']['retry-after'] ?? 0.1); + \Swoole\Coroutine::sleep(max($retryAfter, 0.1)); + } + } finally { + $wg->done(); + } + }); + } + + $wg->wait(); + }); + + ksort($responses); + + foreach ($responses as $response) { + $this->assertSame('', $response['error']); + $this->assertContains($response['statusCode'], [202], (string) $response['body']); + } + + $this->assertEventually(function () use ($functionId, $deploymentId) { + $deployment = $this->getDeployment($functionId, $deploymentId); + + $this->assertEquals(200, $deployment['headers']['status-code']); + $this->assertEquals('ready', $deployment['body']['status']); + $this->assertEquals($deploymentId, $deployment['body']['$id']); + }, 120000, 500); + } finally { + $this->cleanupFunction($functionId); + + if (is_dir($tmpDirectory)) { + foreach (glob($tmpDirectory . DIRECTORY_SEPARATOR . '*') ?: [] as $file) { + unlink($file); + } + rmdir($tmpDirectory); + } + } + } + public function testUpdateDeployment(): void { $data = $this->setupTestDeployment(); diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index 7d9257c699..653b5919eb 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -1034,6 +1034,145 @@ class SitesCustomServerTest extends Scope $this->cleanupSite($siteId); } + public function testCreateDeploymentParallelChunksLargeFile(): void + { + $siteId = $this->setupSite([ + 'buildRuntime' => 'node-22', + 'fallbackFile' => '', + 'framework' => 'other', + 'name' => 'Test Site Parallel Chunk Deployment', + 'outputDirectory' => './', + 'providerBranch' => 'main', + 'providerRootDirectory' => './', + 'siteId' => ID::unique() + ]); + + $deploymentId = ID::unique(); + $tmpDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'appwrite-parallel-site-deployment-' . $deploymentId; + + mkdir($tmpDirectory); + + try { + file_put_contents($tmpDirectory . DIRECTORY_SEPARATOR . 'index.html', 'Hello World'); + file_put_contents($tmpDirectory . DIRECTORY_SEPARATOR . 'large.bin', random_bytes(20 * 1024 * 1024)); + + $source = $tmpDirectory . DIRECTORY_SEPARATOR . 'code.tar.gz'; + Console::execute('cd ' . $tmpDirectory . ' && tar --exclude code.tar.gz -czf code.tar.gz .', '', $this->stdout, $this->stderr); + + $totalSize = filesize($source); + $chunkSize = 5 * 1024 * 1024; + $chunksTotal = (int) ceil($totalSize / $chunkSize); + + $this->assertGreaterThanOrEqual(4, $chunksTotal, 'Test deployment must span at least 4 chunks'); + + $requests = []; + $sourceHandle = fopen($source, 'rb'); + $this->assertNotFalse($sourceHandle, 'Could not open deployment package'); + + try { + for ($i = 0; $i < $chunksTotal; $i++) { + $start = $i * $chunkSize; + $end = min($start + $chunkSize, $totalSize) - 1; + $length = $end - $start + 1; + $chunkPath = $tmpDirectory . DIRECTORY_SEPARATOR . 'chunk-' . $i . '.part'; + + fseek($sourceHandle, $start); + file_put_contents($chunkPath, fread($sourceHandle, $length)); + + $requests[] = [ + 'headers' => [ + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + 'x-appwrite-id' => $deploymentId, + 'content-range' => 'bytes ' . $start . '-' . $end . '/' . $totalSize, + ], + 'chunkPath' => $chunkPath, + ]; + } + } finally { + fclose($sourceHandle); + } + + $responses = []; + $endpoint = parse_url($this->client->getEndpoint()); + $scheme = $endpoint['scheme'] ?? 'http'; + $host = $endpoint['host'] ?? 'appwrite'; + $port = $endpoint['port'] ?? ($scheme === 'https' ? 443 : 80); + $basePath = rtrim($endpoint['path'] ?? '', '/'); + + \Swoole\Coroutine\run(function () use ($basePath, $host, $port, $requests, $scheme, $siteId, &$responses): void { + $wg = new \Swoole\Coroutine\WaitGroup(); + + foreach ($requests as $index => $request) { + $wg->add(); + \Swoole\Coroutine::create(function () use ($basePath, $host, $index, $port, $request, &$responses, $scheme, $siteId, $wg): void { + try { + for ($attempt = 0; $attempt < 3; $attempt++) { + $client = new \Swoole\Coroutine\Http\Client($host, (int) $port, $scheme === 'https'); + $client->set([ + 'timeout' => 300, + 'ssl_verify_peer' => false, + 'ssl_verify_host' => false, + ]); + $client->setHeaders($request['headers']); + $client->setMethod(Client::METHOD_POST); + $client->setData([ + 'activate' => true, + ]); + $client->addFile($request['chunkPath'], 'code', 'application/x-gzip', 'code.tar.gz'); + $client->execute($basePath . '/sites/' . $siteId . '/deployments'); + + $responses[$index] = [ + 'body' => $client->body, + 'error' => $client->errMsg, + 'headers' => $client->headers ?? [], + 'statusCode' => $client->statusCode, + ]; + + $client->close(); + + if ($responses[$index]['statusCode'] !== 429) { + break; + } + + $retryAfter = (float) ($responses[$index]['headers']['retry-after'] ?? 0.1); + \Swoole\Coroutine::sleep(max($retryAfter, 0.1)); + } + } finally { + $wg->done(); + } + }); + } + + $wg->wait(); + }); + + ksort($responses); + + foreach ($responses as $response) { + $this->assertSame('', $response['error']); + $this->assertContains($response['statusCode'], [202], (string) $response['body']); + } + + $this->assertEventually(function () use ($siteId, $deploymentId) { + $deployment = $this->getDeployment($siteId, $deploymentId); + + $this->assertEquals(200, $deployment['headers']['status-code']); + $this->assertEquals('ready', $deployment['body']['status']); + $this->assertEquals($deploymentId, $deployment['body']['$id']); + }, 120000, 500); + } finally { + $this->cleanupSite($siteId); + + if (is_dir($tmpDirectory)) { + foreach (glob($tmpDirectory . DIRECTORY_SEPARATOR . '*') ?: [] as $file) { + unlink($file); + } + rmdir($tmpDirectory); + } + } + } + public function testCreateDeployment() { $siteId = $this->setupSite([ diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 009910fb38..af6aa62564 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -1376,7 +1376,7 @@ trait StorageBase public function testCreateBucketFileParallelChunksLargeFile(): void { - $totalSize = (int) ($_ENV['APPWRITE_TEST_PARALLEL_UPLOAD_SIZE'] ?? 20 * 1024 * 1024); + $totalSize = 20 * 1024 * 1024; $chunkSize = 5 * 1024 * 1024; $chunksTotal = (int) ceil($totalSize / $chunkSize); From 5c7e08fdff44bc7b5cf957fbf7c86c5695f46727 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 4 May 2026 17:59:27 +0400 Subject: [PATCH 07/25] Fix deployment upload analyze warning --- .../Platform/Modules/Functions/Http/Deployments/Create.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php index ac735a8b0a..de468e1bb0 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php @@ -203,7 +203,7 @@ class Create extends Action $completed = false; try { - $checkLock->withLock(function () use (&$chunks, $contentRange, $dbForProject, $deploymentId, &$metadata, &$completed, $response): void { + $checkLock->withLock(function () use (&$chunks, $dbForProject, $deploymentId, &$metadata, &$completed, $response): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); if (!$deployment->isEmpty()) { From c5123529ee81be645ac570758422392025535a1f Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 15 May 2026 13:54:24 +0400 Subject: [PATCH 08/25] Update graphql-php to 15.32 and fix storage tests - Bump graphql-php to 15.32 in composer.json and update composer.lock - Extend Storage/Files/Create with mergeUploadMetadata and pass path and deviceForFiles into the lock callback - Remove obsolete type hints in Storage/Files/Preview/Get - Update Storage tests to tolerate S3-backed signatures --- composer.json | 2 +- composer.lock | 246 +++++----- .../Storage/Http/Buckets/Files/Create.php | 441 +++++++++--------- .../Http/Buckets/Files/Preview/Get.php | 2 - tests/e2e/Services/Storage/StorageBase.php | 10 +- 5 files changed, 369 insertions(+), 332 deletions(-) diff --git a/composer.json b/composer.json index a5b00a8b8b..ca1919a062 100644 --- a/composer.json +++ b/composer.json @@ -94,7 +94,7 @@ "chillerlan/php-qrcode": "4.3.*", "adhocore/jwt": "1.1.*", "spomky-labs/otphp": "11.*", - "webonyx/graphql-php": "15.31.*", + "webonyx/graphql-php": "15.32.*", "league/csv": "9.14.*", "enshrined/svg-sanitize": "0.22.*" }, diff --git a/composer.lock b/composer.lock index 4d920f1eee..43f3dbf228 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fab1d5b01931f0e2545c36c50b3963b7", + "content-hash": "acb2a97c8a37ef8145b70cc385b1091e", "packages": [ { "name": "adhocore/jwt", @@ -2641,16 +2641,16 @@ }, { "name": "symfony/deprecation-contracts", - "version": "v3.6.0", + "version": "v3.7.0", "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "63afe740e99a13ba87ec199bb07bbdee937a5b62" + "reference": "50f59d1f3ca46d41ac911f97a78626b6756af35b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/63afe740e99a13ba87ec199bb07bbdee937a5b62", - "reference": "63afe740e99a13ba87ec199bb07bbdee937a5b62", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/50f59d1f3ca46d41ac911f97a78626b6756af35b", + "reference": "50f59d1f3ca46d41ac911f97a78626b6756af35b", "shasum": "" }, "require": { @@ -2663,7 +2663,7 @@ "name": "symfony/contracts" }, "branch-alias": { - "dev-main": "3.6-dev" + "dev-main": "3.7-dev" } }, "autoload": { @@ -2688,7 +2688,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.6.0" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.7.0" }, "funding": [ { @@ -2699,25 +2699,29 @@ "url": "https://github.com/fabpot", "type": "github" }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" } ], - "time": "2024-09-25T14:21:43+00:00" + "time": "2026-04-13T15:52:40+00:00" }, { "name": "symfony/http-client", - "version": "v7.4.8", + "version": "v7.4.9", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "01933e626c3de76bea1e22641e205e78f6a34342" + "reference": "7e941c6abf4e3bf7dca160bf0e11ef36a9f832f6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/01933e626c3de76bea1e22641e205e78f6a34342", - "reference": "01933e626c3de76bea1e22641e205e78f6a34342", + "url": "https://api.github.com/repos/symfony/http-client/zipball/7e941c6abf4e3bf7dca160bf0e11ef36a9f832f6", + "reference": "7e941c6abf4e3bf7dca160bf0e11ef36a9f832f6", "shasum": "" }, "require": { @@ -2785,7 +2789,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.4.8" + "source": "https://github.com/symfony/http-client/tree/v7.4.9" }, "funding": [ { @@ -2805,20 +2809,20 @@ "type": "tidelift" } ], - "time": "2026-03-30T12:55:43+00:00" + "time": "2026-04-29T13:25:15+00:00" }, { "name": "symfony/http-client-contracts", - "version": "v3.6.0", + "version": "v3.7.0", "source": { "type": "git", "url": "https://github.com/symfony/http-client-contracts.git", - "reference": "75d7043853a42837e68111812f4d964b01e5101c" + "reference": "4a2d00c37651c0bdc2b9e1c773487a8bf4edb12d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/75d7043853a42837e68111812f4d964b01e5101c", - "reference": "75d7043853a42837e68111812f4d964b01e5101c", + "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/4a2d00c37651c0bdc2b9e1c773487a8bf4edb12d", + "reference": "4a2d00c37651c0bdc2b9e1c773487a8bf4edb12d", "shasum": "" }, "require": { @@ -2831,7 +2835,7 @@ "name": "symfony/contracts" }, "branch-alias": { - "dev-main": "3.6-dev" + "dev-main": "3.7-dev" } }, "autoload": { @@ -2867,7 +2871,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/http-client-contracts/tree/v3.6.0" + "source": "https://github.com/symfony/http-client-contracts/tree/v3.7.0" }, "funding": [ { @@ -2878,12 +2882,16 @@ "url": "https://github.com/fabpot", "type": "github" }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" } ], - "time": "2025-04-29T11:18:49+00:00" + "time": "2026-03-06T13:17:50+00:00" }, { "name": "symfony/polyfill-mbstring", @@ -3212,16 +3220,16 @@ }, { "name": "symfony/service-contracts", - "version": "v3.6.1", + "version": "v3.7.0", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43" + "reference": "d25d82433a80eba6aa0e6c24b61d7370d99e444a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/45112560a3ba2d715666a509a0bc9521d10b6c43", - "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/d25d82433a80eba6aa0e6c24b61d7370d99e444a", + "reference": "d25d82433a80eba6aa0e6c24b61d7370d99e444a", "shasum": "" }, "require": { @@ -3239,7 +3247,7 @@ "name": "symfony/contracts" }, "branch-alias": { - "dev-main": "3.6-dev" + "dev-main": "3.7-dev" } }, "autoload": { @@ -3275,7 +3283,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.6.1" + "source": "https://github.com/symfony/service-contracts/tree/v3.7.0" }, "funding": [ { @@ -3295,7 +3303,7 @@ "type": "tidelift" } ], - "time": "2025-07-15T11:30:57+00:00" + "time": "2026-03-28T09:44:51+00:00" }, { "name": "tbachert/spi", @@ -3606,16 +3614,16 @@ }, { "name": "utopia-php/cache", - "version": "1.0.1", + "version": "1.0.3", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "05ceba981436a4022553f7aaa2a05fa049d0f71c" + "reference": "ef52a04e8bfa314c621e3d3326ffcf50db3dfdfa" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/05ceba981436a4022553f7aaa2a05fa049d0f71c", - "reference": "05ceba981436a4022553f7aaa2a05fa049d0f71c", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/ef52a04e8bfa314c621e3d3326ffcf50db3dfdfa", + "reference": "ef52a04e8bfa314c621e3d3326ffcf50db3dfdfa", "shasum": "" }, "require": { @@ -3652,9 +3660,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/1.0.1" + "source": "https://github.com/utopia-php/cache/tree/1.0.3" }, - "time": "2026-03-12T03:39:09+00:00" + "time": "2026-05-11T11:02:13+00:00" }, { "name": "utopia-php/cli", @@ -3850,22 +3858,23 @@ }, { "name": "utopia-php/database", - "version": "5.4.1", + "version": "5.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "688d9422b5ff42ac2ecc29397d94891cfd772e93" + "reference": "eb35e68f7f90932d5a60bd72e70158ae7a4e0511" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/688d9422b5ff42ac2ecc29397d94891cfd772e93", - "reference": "688d9422b5ff42ac2ecc29397d94891cfd772e93", + "url": "https://api.github.com/repos/utopia-php/database/zipball/eb35e68f7f90932d5a60bd72e70158ae7a4e0511", + "reference": "eb35e68f7f90932d5a60bd72e70158ae7a4e0511", "shasum": "" }, "require": { "ext-mbstring": "*", "ext-mongodb": "*", "ext-pdo": "*", + "ext-redis": "*", "php": ">=8.4", "utopia-php/cache": "1.*", "utopia-php/console": "0.1.*", @@ -3903,9 +3912,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/5.4.1" + "source": "https://github.com/utopia-php/database/tree/5.7.0" }, - "time": "2026-04-29T07:32:59+00:00" + "time": "2026-05-06T01:04:08+00:00" }, { "name": "utopia-php/detector", @@ -4171,22 +4180,21 @@ }, { "name": "utopia-php/emails", - "version": "0.6.9", + "version": "0.6.10", "source": { "type": "git", "url": "https://github.com/utopia-php/emails.git", - "reference": "3a59fb392a03a88f5497e5fdb0ea84a252a4dfdf" + "reference": "2e397754ce68c2ba918564b9f31d9923c0a90429" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/emails/zipball/3a59fb392a03a88f5497e5fdb0ea84a252a4dfdf", - "reference": "3a59fb392a03a88f5497e5fdb0ea84a252a4dfdf", + "url": "https://api.github.com/repos/utopia-php/emails/zipball/2e397754ce68c2ba918564b9f31d9923c0a90429", + "reference": "2e397754ce68c2ba918564b9f31d9923c0a90429", "shasum": "" }, "require": { "php": ">=8.0", "utopia-php/domains": "^1.0", - "utopia-php/fetch": "^0.5", "utopia-php/validators": "0.*" }, "require-dev": { @@ -4194,7 +4202,8 @@ "phpstan/phpstan": "^1.10", "phpunit/phpunit": "^9.3", "utopia-php/cli": "^0.22", - "utopia-php/console": "0.*" + "utopia-php/console": "0.*", + "utopia-php/fetch": "^1.1" }, "type": "library", "autoload": { @@ -4226,9 +4235,9 @@ ], "support": { "issues": "https://github.com/utopia-php/emails/issues", - "source": "https://github.com/utopia-php/emails/tree/0.6.9" + "source": "https://github.com/utopia-php/emails/tree/0.6.10" }, - "time": "2026-03-14T13:52:56+00:00" + "time": "2026-05-08T10:16:22+00:00" }, { "name": "utopia-php/fetch", @@ -4552,16 +4561,16 @@ }, { "name": "utopia-php/messaging", - "version": "0.22.0", + "version": "0.22.2", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "a6ac04fd204fb6a16bf8c75a84d0b9fc10aa5030" + "reference": "f99feceab575243f3a86ee2e90cd1a6407805def" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/a6ac04fd204fb6a16bf8c75a84d0b9fc10aa5030", - "reference": "a6ac04fd204fb6a16bf8c75a84d0b9fc10aa5030", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/f99feceab575243f3a86ee2e90cd1a6407805def", + "reference": "f99feceab575243f3a86ee2e90cd1a6407805def", "shasum": "" }, "require": { @@ -4597,22 +4606,22 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.22.0" + "source": "https://github.com/utopia-php/messaging/tree/0.22.2" }, - "time": "2026-04-02T04:09:19+00:00" + "time": "2026-05-14T08:51:26+00:00" }, { "name": "utopia-php/migration", - "version": "1.9.5", + "version": "1.9.7", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "952a4dfe232702f80e45c35129466a8d8cb4c599" + "reference": "81b608a6871f56b70496803d12010823300aab6e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/952a4dfe232702f80e45c35129466a8d8cb4c599", - "reference": "952a4dfe232702f80e45c35129466a8d8cb4c599", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/81b608a6871f56b70496803d12010823300aab6e", + "reference": "81b608a6871f56b70496803d12010823300aab6e", "shasum": "" }, "require": { @@ -4652,9 +4661,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.9.5" + "source": "https://github.com/utopia-php/migration/tree/1.9.7" }, - "time": "2026-04-29T11:19:13+00:00" + "time": "2026-05-05T07:18:48+00:00" }, { "name": "utopia-php/mongo", @@ -5093,16 +5102,16 @@ }, { "name": "utopia-php/storage", - "version": "2.0.1", + "version": "2.0.3", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "8a2e3a86fd01aaed675884146665308c2122264e" + "reference": "37129cf0bfcc03210172000e4388d4d3495ae013" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/8a2e3a86fd01aaed675884146665308c2122264e", - "reference": "8a2e3a86fd01aaed675884146665308c2122264e", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/37129cf0bfcc03210172000e4388d4d3495ae013", + "reference": "37129cf0bfcc03210172000e4388d4d3495ae013", "shasum": "" }, "require": { @@ -5139,22 +5148,22 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/2.0.1" + "source": "https://github.com/utopia-php/storage/tree/2.0.3" }, - "time": "2026-04-29T09:05:48+00:00" + "time": "2026-05-15T09:42:32+00:00" }, { "name": "utopia-php/system", - "version": "0.10.1", + "version": "0.10.2", "source": { "type": "git", "url": "https://github.com/utopia-php/system.git", - "reference": "7c1669533bb9c285de19191270c8c1439161a78a" + "reference": "04229a822b147c1abaf1a92fb42c2d7aad4625df" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/system/zipball/7c1669533bb9c285de19191270c8c1439161a78a", - "reference": "7c1669533bb9c285de19191270c8c1439161a78a", + "url": "https://api.github.com/repos/utopia-php/system/zipball/04229a822b147c1abaf1a92fb42c2d7aad4625df", + "reference": "04229a822b147c1abaf1a92fb42c2d7aad4625df", "shasum": "" }, "require": { @@ -5195,9 +5204,9 @@ ], "support": { "issues": "https://github.com/utopia-php/system/issues", - "source": "https://github.com/utopia-php/system/tree/0.10.1" + "source": "https://github.com/utopia-php/system/tree/0.10.2" }, - "time": "2026-03-15T21:07:41+00:00" + "time": "2026-05-05T14:33:41+00:00" }, { "name": "utopia-php/telemetry", @@ -5256,16 +5265,16 @@ }, { "name": "utopia-php/validators", - "version": "0.2.2", + "version": "0.2.3", "source": { "type": "git", "url": "https://github.com/utopia-php/validators.git", - "reference": "5d7d494e64457cd4eb67fdcfd9481f2c89796aa6" + "reference": "9770269c8ed8e6909934965fa8722103c7434c23" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/validators/zipball/5d7d494e64457cd4eb67fdcfd9481f2c89796aa6", - "reference": "5d7d494e64457cd4eb67fdcfd9481f2c89796aa6", + "url": "https://api.github.com/repos/utopia-php/validators/zipball/9770269c8ed8e6909934965fa8722103c7434c23", + "reference": "9770269c8ed8e6909934965fa8722103c7434c23", "shasum": "" }, "require": { @@ -5295,9 +5304,9 @@ ], "support": { "issues": "https://github.com/utopia-php/validators/issues", - "source": "https://github.com/utopia-php/validators/tree/0.2.2" + "source": "https://github.com/utopia-php/validators/tree/0.2.3" }, - "time": "2026-04-27T16:30:24+00:00" + "time": "2026-05-14T08:05:44+00:00" }, { "name": "utopia-php/vcs", @@ -5457,16 +5466,16 @@ }, { "name": "webonyx/graphql-php", - "version": "v15.31.5", + "version": "v15.32.3", "source": { "type": "git", "url": "https://github.com/webonyx/graphql-php.git", - "reference": "089c4ef7e112df85788cfe06596278a8f99f4aa9" + "reference": "993bf0bea17f870412ad8a90f60c41cb8d5f1145" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/webonyx/graphql-php/zipball/089c4ef7e112df85788cfe06596278a8f99f4aa9", - "reference": "089c4ef7e112df85788cfe06596278a8f99f4aa9", + "url": "https://api.github.com/repos/webonyx/graphql-php/zipball/993bf0bea17f870412ad8a90f60c41cb8d5f1145", + "reference": "993bf0bea17f870412ad8a90f60c41cb8d5f1145", "shasum": "" }, "require": { @@ -5475,16 +5484,16 @@ "php": "^7.4 || ^8" }, "require-dev": { - "amphp/amp": "^2.6", - "amphp/http-server": "^2.1", + "amphp/amp": "^2.6 || ^3", + "amphp/http-server": "^2.1 || ^3", "dms/phpunit-arraysubset-asserts": "dev-master", "ergebnis/composer-normalize": "^2.28", - "friendsofphp/php-cs-fixer": "3.94.2", + "friendsofphp/php-cs-fixer": "3.95.1", "mll-lab/php-cs-fixer-config": "5.13.0", "nyholm/psr7": "^1.5", "phpbench/phpbench": "^1.2", "phpstan/extension-installer": "^1.1", - "phpstan/phpstan": "2.1.46", + "phpstan/phpstan": "2.1.51", "phpstan/phpstan-phpunit": "2.0.16", "phpstan/phpstan-strict-rules": "2.0.10", "phpunit/phpunit": "^9.5 || ^10.5.21 || ^11", @@ -5498,6 +5507,7 @@ "ticketswap/phpstan-error-formatter": "1.3.0" }, "suggest": { + "amphp/amp": "To leverage async resolving on AMPHP platform (v3 with AmpFutureAdapter, v2 with AmpPromiseAdapter)", "amphp/http-server": "To leverage async resolving with webserver on AMPHP platform", "psr/http-message": "To use standard GraphQL server", "react/promise": "To leverage async resolving on React PHP platform" @@ -5520,7 +5530,7 @@ ], "support": { "issues": "https://github.com/webonyx/graphql-php/issues", - "source": "https://github.com/webonyx/graphql-php/tree/v15.31.5" + "source": "https://github.com/webonyx/graphql-php/tree/v15.32.3" }, "funding": [ { @@ -5532,22 +5542,22 @@ "type": "open_collective" } ], - "time": "2026-04-11T18:06:15+00:00" + "time": "2026-04-24T13:49:35+00:00" } ], "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.25.1", + "version": "1.29.5", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "f21a556b9acdbf75bbdcdc90a078af641646eade" + "reference": "e670edcdfb9ffcec36125b1eb3e4473dce30b620" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/f21a556b9acdbf75bbdcdc90a078af641646eade", - "reference": "f21a556b9acdbf75bbdcdc90a078af641646eade", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/e670edcdfb9ffcec36125b1eb3e4473dce30b620", + "reference": "e670edcdfb9ffcec36125b1eb3e4473dce30b620", "shasum": "" }, "require": { @@ -5583,9 +5593,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.25.1" + "source": "https://github.com/appwrite/sdk-generator/tree/1.29.5" }, - "time": "2026-04-28T11:12:22+00:00" + "time": "2026-05-15T06:49:05+00:00" }, { "name": "brianium/paratest", @@ -6692,16 +6702,16 @@ }, { "name": "phpunit/phpunit", - "version": "12.5.23", + "version": "12.5.25", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "c54fcf3d6bcb6e96ac2f7e40097dc37b5f139969" + "reference": "792c2980442dfce319226b88fa845b8b6de3b333" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c54fcf3d6bcb6e96ac2f7e40097dc37b5f139969", - "reference": "c54fcf3d6bcb6e96ac2f7e40097dc37b5f139969", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/792c2980442dfce319226b88fa845b8b6de3b333", + "reference": "792c2980442dfce319226b88fa845b8b6de3b333", "shasum": "" }, "require": { @@ -6770,7 +6780,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/12.5.23" + "source": "https://github.com/sebastianbergmann/phpunit/tree/12.5.25" }, "funding": [ { @@ -6778,7 +6788,7 @@ "type": "other" } ], - "time": "2026-04-18T06:12:49+00:00" + "time": "2026-05-13T03:56:57+00:00" }, { "name": "sebastian/cli-parser", @@ -7763,16 +7773,16 @@ }, { "name": "symfony/console", - "version": "v8.0.8", + "version": "v8.0.11", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "5b66d385dc58f69652e56f78a4184615e3f2b7f7" + "reference": "3156577f46a38aa1b9323aad223de7a9cd426782" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/5b66d385dc58f69652e56f78a4184615e3f2b7f7", - "reference": "5b66d385dc58f69652e56f78a4184615e3f2b7f7", + "url": "https://api.github.com/repos/symfony/console/zipball/3156577f46a38aa1b9323aad223de7a9cd426782", + "reference": "3156577f46a38aa1b9323aad223de7a9cd426782", "shasum": "" }, "require": { @@ -7829,7 +7839,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v8.0.8" + "source": "https://github.com/symfony/console/tree/v8.0.11" }, "funding": [ { @@ -7849,7 +7859,7 @@ "type": "tidelift" } ], - "time": "2026-03-30T15:14:47+00:00" + "time": "2026-05-13T12:07:53+00:00" }, { "name": "symfony/polyfill-ctype", @@ -8183,16 +8193,16 @@ }, { "name": "symfony/process", - "version": "v8.0.8", + "version": "v8.0.11", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "cb8939aff03470d1a9d1d1b66d08c6fa71b3bbdc" + "reference": "26d89e459f037d2873300605d0a07e7a8ef84db0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/cb8939aff03470d1a9d1d1b66d08c6fa71b3bbdc", - "reference": "cb8939aff03470d1a9d1d1b66d08c6fa71b3bbdc", + "url": "https://api.github.com/repos/symfony/process/zipball/26d89e459f037d2873300605d0a07e7a8ef84db0", + "reference": "26d89e459f037d2873300605d0a07e7a8ef84db0", "shasum": "" }, "require": { @@ -8224,7 +8234,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v8.0.8" + "source": "https://github.com/symfony/process/tree/v8.0.11" }, "funding": [ { @@ -8244,20 +8254,20 @@ "type": "tidelift" } ], - "time": "2026-03-30T15:14:47+00:00" + "time": "2026-05-11T16:56:32+00:00" }, { "name": "symfony/string", - "version": "v8.0.8", + "version": "v8.0.11", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "ae9488f874d7603f9d2dfbf120203882b645d963" + "reference": "39be2ad058a3c0bd558edca23e65f009865d75ff" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/ae9488f874d7603f9d2dfbf120203882b645d963", - "reference": "ae9488f874d7603f9d2dfbf120203882b645d963", + "url": "https://api.github.com/repos/symfony/string/zipball/39be2ad058a3c0bd558edca23e65f009865d75ff", + "reference": "39be2ad058a3c0bd558edca23e65f009865d75ff", "shasum": "" }, "require": { @@ -8314,7 +8324,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v8.0.8" + "source": "https://github.com/symfony/string/tree/v8.0.11" }, "funding": [ { @@ -8334,7 +8344,7 @@ "type": "tidelift" } ], - "time": "2026-03-30T15:14:47+00:00" + "time": "2026-05-13T12:07:53+00:00" }, { "name": "textalk/websocket", diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index c26e30474f..e9f17fb801 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -247,8 +247,25 @@ class Create extends Action $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; $completed = false; + $mergeUploadMetadata = function (array $stored, array $current): array { + $merged = \array_merge($stored, $current); + + if (isset($stored['parts']) || isset($current['parts'])) { + $parts = $stored['parts'] ?? []; + foreach (($current['parts'] ?? []) as $part => $value) { + $parts[(int) $part] = $value; + } + \ksort($parts); + + $merged['parts'] = $parts; + $merged['chunks'] = \count($parts); + } + + return $merged; + }; + try { - $lock->withLock(function () use ($bucket, &$chunks, $contentRange, $dbForProject, $fileId, &$metadata, &$completed, $response): void { + $lock->withLock(function () use ($bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, &$metadata, &$completed, $path, $response): void { $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); if (!$file->isEmpty()) { $chunks = $file->getAttribute('chunksTotal', 1); @@ -268,6 +285,8 @@ class Create extends Action return; } } + + $deviceForFiles->prepareUpload($path, $metadata['content_type'] ?? '', $chunks, $metadata); }, timeout: 120.0); } catch (LockContention) { $response->addHeader('Retry-After', '5'); @@ -278,219 +297,223 @@ class Create extends Action return; } - $chunksUploaded = $deviceForFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata); + $finalizeUpload = function (int $chunksUploaded) use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $mergeUploadMetadata, $path, $permissions, $queueForEvents, $response): void { + $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + $uploaded = 0; - if (empty($chunksUploaded)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file'); - } + if (!$file->isEmpty()) { + $chunks = $file->getAttribute('chunksTotal', 1); + $uploaded = $file->getAttribute('chunksUploaded', 0); + $metadata = $mergeUploadMetadata($file->getAttribute('metadata', []), $metadata); + + if ($uploaded === $chunks) { + if (empty($contentRange)) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } + + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->dynamic($file, Response::MODEL_FILE); + + return; + } + } + + $chunksUploaded = max($uploaded, $chunksUploaded, (int) ($metadata['chunks'] ?? 0)); + + if ($chunksUploaded === $chunks && $uploaded < $chunks) { + $deviceForFiles->finalizeUpload($path, $chunks, $metadata); + + if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) { + $antivirus = new Network( + System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), + (int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) + ); + + if (!$antivirus->fileScan($path)) { + $deviceForFiles->delete($path); + throw new Exception(Exception::STORAGE_INVALID_FILE); + } + } + + $mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption + $fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption + $data = ''; + $iv = ''; + $tag = null; + // Compression + $algorithm = $bucket->getAttribute('compression', Compression::NONE); + if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) { + $data = $deviceForFiles->read($path); + switch ($algorithm) { + case Compression::ZSTD: + $compressor = new Zstd(); + break; + case Compression::GZIP: + default: + $compressor = new GZIP(); + break; + } + $data = $compressor->compress($data); + } else { + // reset the algorithm to none as we do not compress the file + // if file size exceedes the APP_STORAGE_READ_BUFFER + // regardless the bucket compression algoorithm + $algorithm = Compression::NONE; + } + + if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { + if (empty($data)) { + $data = $deviceForFiles->read($path); + } + $key = System::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); + } + + if (!empty($data)) { + if (!$deviceForFiles->write($path, $data, $mimeType)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file'); + } + } + + $sizeActual = $deviceForFiles->getFileSize($path); + + $openSSLVersion = null; + $openSSLCipher = null; + $openSSLTag = null; + $openSSLIV = null; + + if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { + $openSSLVersion = '1'; + $openSSLCipher = OpenSSL::CIPHER_AES_128_GCM; + $openSSLTag = \bin2hex($tag); + $openSSLIV = \bin2hex($iv); + } + + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => $fileId, + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $fileName, + 'path' => $path, + 'signature' => $fileHash, + 'mimeType' => $mimeType, + 'sizeOriginal' => $fileSize, + 'sizeActual' => $sizeActual, + 'algorithm' => $algorithm, + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'openSSLVersion' => $openSSLVersion, + 'openSSLCipher' => $openSSLCipher, + 'openSSLTag' => $openSSLTag, + 'openSSLIV' => $openSSLIV, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); + + try { + $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } else { + /** + * Skip authorization in updateDocument. + * Without this, the file creation will fail when user doesn't have update permission. + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we rely on the create-permission check performed earlier. + */ + $file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([ + '$permissions' => $permissions, + 'signature' => $fileHash, + 'mimeType' => $mimeType, + 'sizeActual' => $sizeActual, + 'algorithm' => $algorithm, + 'openSSLVersion' => $openSSLVersion, + 'openSSLCipher' => $openSSLCipher, + 'openSSLTag' => $openSSLTag, + 'openSSLIV' => $openSSLIV, + 'metadata' => $metadata, + 'chunksUploaded' => $chunksUploaded, + ]))); + } + + // Trigger after create success hook + $this->afterCreateSuccess($file); + } else { + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => ID::custom($fileId), + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $fileName, + 'path' => $path, + 'signature' => '', + 'mimeType' => '', + 'sizeOriginal' => $fileSize, + 'sizeActual' => 0, + 'algorithm' => '', + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); + + try { + $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } else { + /** + * Skip authorization in updateDocument. + * Without this, the file creation will fail when user doesn't have update permission. + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we rely on the create-permission check performed earlier. + */ + try { + $file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([ + 'chunksUploaded' => $chunksUploaded, + 'metadata' => $metadata, + ]))); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } + } + + if ($chunksUploaded === $chunks) { + $queueForEvents + ->setParam('bucketId', $bucket->getId()) + ->setParam('fileId', $file->getId()) + ->setContext('bucket', $bucket); + } + + $metadata = null; // was causing leaks as it was passed by reference + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($file, Response::MODEL_FILE); + }; try { - $lock->withLock(function () use ($authorization, $bucket, &$chunks, $chunksUploaded, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $queueForEvents, $response): void { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); - $uploaded = 0; + $chunksUploaded = $deviceForFiles->uploadChunk($fileTmpName, $path, $chunk, $chunks, $metadata); - if (!$file->isEmpty()) { - $chunks = $file->getAttribute('chunksTotal', 1); - $uploaded = $file->getAttribute('chunksUploaded', 0); - $metadata = \array_merge($file->getAttribute('metadata', []), $metadata); + if (empty($chunksUploaded)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file'); + } - if ($uploaded === $chunks) { - if (empty($contentRange)) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } - - $response - ->setStatusCode(Response::STATUS_CODE_OK) - ->dynamic($file, Response::MODEL_FILE); - - return; - } - } - - $chunksUploaded = max($uploaded, $chunksUploaded); - - if ($chunksUploaded === $chunks && $uploaded < $chunks) { - if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) { - $antivirus = new Network( - System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), - (int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) - ); - - if (!$antivirus->fileScan($path)) { - $deviceForFiles->delete($path); - throw new Exception(Exception::STORAGE_INVALID_FILE); - } - } - - $mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption - $fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption - $data = ''; - $iv = ''; - $tag = null; - // Compression - $algorithm = $bucket->getAttribute('compression', Compression::NONE); - if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) { - $data = $deviceForFiles->read($path); - switch ($algorithm) { - case Compression::ZSTD: - $compressor = new Zstd(); - break; - case Compression::GZIP: - default: - $compressor = new GZIP(); - break; - } - $data = $compressor->compress($data); - } else { - // reset the algorithm to none as we do not compress the file - // if file size exceedes the APP_STORAGE_READ_BUFFER - // regardless the bucket compression algoorithm - $algorithm = Compression::NONE; - } - - if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { - if (empty($data)) { - $data = $deviceForFiles->read($path); - } - $key = System::getEnv('_APP_OPENSSL_KEY_V1'); - $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); - $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); - } - - if (!empty($data)) { - if (!$deviceForFiles->write($path, $data, $mimeType)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file'); - } - } - - $sizeActual = $deviceForFiles->getFileSize($path); - - $openSSLVersion = null; - $openSSLCipher = null; - $openSSLTag = null; - $openSSLIV = null; - - if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) { - $openSSLVersion = '1'; - $openSSLCipher = OpenSSL::CIPHER_AES_128_GCM; - $openSSLTag = \bin2hex($tag); - $openSSLIV = \bin2hex($iv); - } - - if ($file->isEmpty()) { - $doc = new Document([ - '$id' => $fileId, - '$permissions' => $permissions, - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getSequence(), - 'name' => $fileName, - 'path' => $path, - 'signature' => $fileHash, - 'mimeType' => $mimeType, - 'sizeOriginal' => $fileSize, - 'sizeActual' => $sizeActual, - 'algorithm' => $algorithm, - 'comment' => '', - 'chunksTotal' => $chunks, - 'chunksUploaded' => $chunksUploaded, - 'openSSLVersion' => $openSSLVersion, - 'openSSLCipher' => $openSSLCipher, - 'openSSLTag' => $openSSLTag, - 'openSSLIV' => $openSSLIV, - 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => $metadata, - ]); - - try { - $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); - } catch (DuplicateException) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } else { - /** - * Skip authorization in updateDocument. - * Without this, the file creation will fail when user doesn't have update permission. - * However as with chunk upload even if we are updating, we are essentially creating a file - * adding it's new chunk so we rely on the create-permission check performed earlier. - */ - $file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([ - '$permissions' => $permissions, - 'signature' => $fileHash, - 'mimeType' => $mimeType, - 'sizeActual' => $sizeActual, - 'algorithm' => $algorithm, - 'openSSLVersion' => $openSSLVersion, - 'openSSLCipher' => $openSSLCipher, - 'openSSLTag' => $openSSLTag, - 'openSSLIV' => $openSSLIV, - 'metadata' => $metadata, - 'chunksUploaded' => $chunksUploaded, - ]))); - } - - // Trigger after create success hook - $this->afterCreateSuccess($file); - } else { - if ($file->isEmpty()) { - $doc = new Document([ - '$id' => ID::custom($fileId), - '$permissions' => $permissions, - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getSequence(), - 'name' => $fileName, - 'path' => $path, - 'signature' => '', - 'mimeType' => '', - 'sizeOriginal' => $fileSize, - 'sizeActual' => 0, - 'algorithm' => '', - 'comment' => '', - 'chunksTotal' => $chunks, - 'chunksUploaded' => $chunksUploaded, - 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => $metadata, - ]); - - try { - $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); - } catch (DuplicateException) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } else { - /** - * Skip authorization in updateDocument. - * Without this, the file creation will fail when user doesn't have update permission. - * However as with chunk upload even if we are updating, we are essentially creating a file - * adding it's new chunk so we rely on the create-permission check performed earlier. - */ - try { - $file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([ - 'chunksUploaded' => $chunksUploaded, - 'metadata' => $metadata, - ]))); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } - } - - if ($chunksUploaded === $chunks) { - $queueForEvents - ->setParam('bucketId', $bucket->getId()) - ->setParam('fileId', $file->getId()) - ->setContext('bucket', $bucket); - } - - $metadata = null; // was causing leaks as it was passed by reference - - $response - ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($file, Response::MODEL_FILE); - }, timeout: 120.0); + $lock->withLock(fn () => $finalizeUpload($chunksUploaded), timeout: 120.0); } catch (LockContention) { $response->addHeader('Retry-After', '5'); throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'File upload is busy. Try again.'); diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php index 4fa5006db8..ff4844468c 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Preview/Get.php @@ -127,7 +127,6 @@ class Get extends Action throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); } - /* @type Document $bucket */ $bucket = $authorization->skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); $isAPIKey = $user->isApp($authorization->getRoles()); @@ -151,7 +150,6 @@ class Get extends Action if ($fileSecurity && !$valid && !$isToken) { $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); } else { - /* @type Document $file */ $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); } diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index af6aa62564..a3aa88db31 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -11,6 +11,8 @@ use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Database\Validator\Datetime as DatetimeValidator; +use Utopia\Storage\Storage; +use Utopia\System\System; trait StorageBase { @@ -287,7 +289,11 @@ trait StorageBase $this->assertEquals('large-file.mp4', $largeFile['body']['name']); $this->assertEquals('video/mp4', $largeFile['body']['mimeType']); $this->assertEquals($totalSize, $largeFile['body']['sizeOriginal']); - $this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $largeFile['body']['signature']); // should validate that the file is not encrypted + if (System::getEnv('_APP_STORAGE_DEVICE') === Storage::DEVICE_S3 || str_starts_with(System::getEnv('_APP_CONNECTIONS_STORAGE', ''), Storage::DEVICE_S3 . '://')) { + $this->assertNotEmpty($largeFile['body']['signature']); + } else { + $this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $largeFile['body']['signature']); // should validate that the file is not encrypted + } /** * Failure @@ -391,7 +397,7 @@ trait StorageBase 'bucketId' => ID::unique(), 'name' => 'Test Bucket 2', 'fileSecurity' => true, - 'maximumFileSize' => 6000000000, //6GB + 'maximumFileSize' => (int) System::getEnv('_APP_STORAGE_LIMIT', 0) + 1, 'allowedFileExtensions' => ["jpg", "png"], 'permissions' => [ Permission::read(Role::any()), From e6f30611033dd7be7a2d8143333e6501d6a1712c Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 15 May 2026 14:15:08 +0400 Subject: [PATCH 09/25] Pin utopia-php/lock to 0.2.x --- composer.json | 2 +- composer.lock | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/composer.json b/composer.json index fe6ec5dd0d..646112438c 100644 --- a/composer.json +++ b/composer.json @@ -72,7 +72,7 @@ "utopia-php/validators": "0.2.*", "utopia-php/image": "0.8.*", "utopia-php/locale": "0.8.*", - "utopia-php/lock": "dev-main", + "utopia-php/lock": "0.2.*", "utopia-php/logger": "0.8.*", "utopia-php/messaging": "0.22.*", "utopia-php/migration": "1.*", diff --git a/composer.lock b/composer.lock index 8ccfb9e744..ff112ce5b1 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "864ea52ffc77c76363c5b487f49a1c77", + "content-hash": "6bc4a0d273ff6b032b36fe205847d02a", "packages": [ { "name": "adhocore/jwt", @@ -4500,16 +4500,16 @@ }, { "name": "utopia-php/lock", - "version": "dev-main", + "version": "0.2.0", "source": { "type": "git", "url": "https://github.com/utopia-php/lock.git", - "reference": "22eda789528da4cc71e767c9ad9e9fd4fa2c9467" + "reference": "49317c9493d8f747e4299aa24c22862aa5f6e106" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/lock/zipball/22eda789528da4cc71e767c9ad9e9fd4fa2c9467", - "reference": "22eda789528da4cc71e767c9ad9e9fd4fa2c9467", + "url": "https://api.github.com/repos/utopia-php/lock/zipball/49317c9493d8f747e4299aa24c22862aa5f6e106", + "reference": "49317c9493d8f747e4299aa24c22862aa5f6e106", "shasum": "" }, "require": { @@ -4526,7 +4526,6 @@ "ext-redis": "Required for the Distributed lock", "ext-swoole": "Required for the Mutex and Semaphore locks (>=6.0)" }, - "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -4564,12 +4563,12 @@ "email": "team@appwrite.io" } ], - "description": "A simple lock library to coordinate access to shared resources across coroutines, processes and hosts", + "description": "Mutex, semaphore, file and distributed locks for PHP — one interface, four backends.", "support": { - "source": "https://github.com/utopia-php/lock/tree/main", + "source": "https://github.com/utopia-php/lock/tree/0.2.0", "issues": "https://github.com/utopia-php/lock/issues" }, - "time": "2026-04-28T10:07:10+00:00" + "time": "2026-04-24T10:47:56+00:00" }, { "name": "utopia-php/logger", @@ -8640,8 +8639,7 @@ "aliases": [], "minimum-stability": "dev", "stability-flags": { - "utopia-php/http": 5, - "utopia-php/lock": 20 + "utopia-php/http": 5 }, "prefer-stable": true, "prefer-lowest": false, From 97d66566870784510e9a1142ebd94fd766cf9d4a Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 15 May 2026 14:17:14 +0400 Subject: [PATCH 10/25] Fix deployment upload lock TTL --- .../Platform/Modules/Functions/Http/Deployments/Create.php | 2 +- src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php index 74e32e01c9..1718f7359f 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php @@ -199,7 +199,7 @@ class Create extends Action $path = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); $lockKey = 'functions:deployment:' . $project->getId() . ':' . $functionId . ':' . $deploymentId; - $checkLock = new Distributed($redis, $lockKey, ttl: 120); + $checkLock = new Distributed($redis, $lockKey, ttl: 600); $stateLock = new Distributed($redis, $lockKey, ttl: 600); $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php index df61d1809d..2c2c34e913 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php @@ -199,7 +199,7 @@ class Create extends Action $path = $deviceForSites->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); $lockKey = 'sites:deployment:' . $project->getId() . ':' . $siteId . ':' . $deploymentId; - $checkLock = new Distributed($redis, $lockKey, ttl: 120); + $checkLock = new Distributed($redis, $lockKey, ttl: 600); $stateLock = new Distributed($redis, $lockKey, ttl: 600); $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; From 69735b383b6eed5c40b7284053a182415e974d93 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Fri, 15 May 2026 17:09:58 +0530 Subject: [PATCH 11/25] feat: add email template reset and get-default endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add GET /v1/project/templates/email/:templateId/default that returns Appwrite built-in defaults, ignoring any custom project overrides - Add reset param to PATCH /v1/project/templates/email — when reset=true, clears the custom template entry and returns defaults without requiring SMTP to be enabled --- .../Project/Templates/Email/GetDefault.php | 118 ++++++++++++++++++ .../Http/Project/Templates/Email/Update.php | 102 ++++++++++++--- .../Modules/Project/Services/Http.php | 2 + 3 files changed, 207 insertions(+), 15 deletions(-) create mode 100644 src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/GetDefault.php diff --git a/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/GetDefault.php b/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/GetDefault.php new file mode 100644 index 0000000000..c89c39e0d6 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/GetDefault.php @@ -0,0 +1,118 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/console/templates/email/:templateId') + ->desc('Get default project email template') + ->groups(['api', 'project']) + ->label('scope', 'templates.read') + ->label('sdk', new Method( + namespace: 'console', + group: 'templates', + name: 'getEmailTemplate', + description: <<param('templateId', '', new WhiteList(Config::getParam('locale-templates')['email'] ?? [], true), 'Email template type. Can be one of: ' . \implode(', ', Config::getParam('locale-templates')['email'] ?? [])) + ->param('locale', '', fn ($localeCodes) => new WhiteList($localeCodes), 'Template locale. If left empty, the fallback locale (en) will be used.', optional: true, injections: ['localeCodes']) + ->inject('response') + ->callback($this->action(...)); + } + + public function action( + string $templateId, + string $locale, + Response $response, + ): void { + $locale = $locale ?: System::getEnv('_APP_LOCALE', 'en'); + + $localeObj = new Locale($locale); + $localeObj->setFallback(System::getEnv('_APP_LOCALE', 'en')); + + $response->dynamic(new Document([ + 'templateId' => $templateId, + 'locale' => $locale, + 'subject' => $localeObj->getText('emails.' . $templateId . '.subject'), + 'message' => $this->getDefaultMessage($templateId, $localeObj), + 'senderName' => '', + 'senderEmail' => '', + 'replyToEmail' => '', + 'replyToName' => '', + ]), Response::MODEL_EMAIL_TEMPLATE); + } + + private function getDefaultMessage(string $templateId, Locale $localeObj): string + { + $templateConfigs = [ + 'magicSession' => [ + 'file' => 'email-magic-url.tpl', + 'placeholders' => ['optionButton', 'buttonText', 'optionUrl', 'clientInfo', 'securityPhrase'] + ], + 'mfaChallenge' => [ + 'file' => 'email-mfa-challenge.tpl', + 'placeholders' => ['description', 'clientInfo'] + ], + 'otpSession' => [ + 'file' => 'email-otp.tpl', + 'placeholders' => ['description', 'clientInfo', 'securityPhrase'] + ], + 'sessionAlert' => [ + 'file' => 'email-session-alert.tpl', + 'placeholders' => ['body', 'listDevice', 'listIpAddress', 'listCountry', 'footer'] + ], + ]; + + $config = $templateConfigs[$templateId] ?? [ + 'file' => 'email-inner-base.tpl', + 'placeholders' => ['buttonText', 'body', 'footer'] + ]; + + $templateString = file_get_contents(APP_CE_CONFIG_DIR . '/locale/templates/' . $config['file']); + $message = Template::fromString($templateString); + + foreach ($config['placeholders'] as $param) { + $escapeHtml = !in_array($param, ['clientInfo', 'body', 'footer', 'description']); + $message->setParam("{{{$param}}}", $localeObj->getText("emails.{$templateId}.{$param}"), escapeHtml: $escapeHtml); + } + + $message + ->setParam('{{hello}}', $localeObj->getText("emails.{$templateId}.hello")) + ->setParam('{{thanks}}', $localeObj->getText("emails.{$templateId}.thanks")) + ->setParam('{{signature}}', $localeObj->getText("emails.{$templateId}.signature")); + + return $message->render(useContent: true); + } +} diff --git a/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php b/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php index ef93abf683..55824f2102 100644 --- a/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php +++ b/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php @@ -7,15 +7,18 @@ use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Template\Template; use Appwrite\Utopia\Response; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; use Utopia\Emails\Validator\Email; +use Utopia\Locale\Locale; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; use Utopia\System\System; +use Utopia\Validator\Boolean; use Utopia\Validator\Nullable; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; @@ -64,6 +67,7 @@ class Update extends Action ->param('senderEmail', null, new Nullable(new Email()), 'Email of the sender.', optional: true) ->param('replyToEmail', null, new Nullable(new Email()), 'Reply to email.', optional: true) ->param('replyToName', null, new Nullable(new Text(255, 0)), 'Reply to name.', optional: true) + ->param('reset', false, new Boolean(), 'Reset template to Appwrite default, removing any custom override.', optional: true) ->inject('response') ->inject('queueForEvents') ->inject('dbForPlatform') @@ -81,6 +85,7 @@ class Update extends Action ?string $senderEmail, ?string $replyToEmail, ?string $replyToName, + bool $reset, Response $response, QueueEvent $queueForEvents, Database $dbForPlatform, @@ -89,14 +94,40 @@ class Update extends Action ) { $locale = $locale ?: System::getEnv('_APP_LOCALE', 'en'); + $templates = $project->getAttribute('templates', []); + + if ($reset) { + unset($templates['email.' . $templateId . '-' . $locale]); + $authorization->skip(fn () => $dbForPlatform->updateDocument( + 'projects', + $project->getId(), + new Document(['templates' => $templates]) + )); + + $queueForEvents->setParam('templateId', $templateId); + + $localeObj = new Locale($locale); + $localeObj->setFallback(System::getEnv('_APP_LOCALE', 'en')); + + $response->dynamic(new Document([ + 'templateId' => $templateId, + 'locale' => $locale, + 'subject' => $localeObj->getText('emails.' . $templateId . '.subject'), + 'message' => $this->getDefaultMessage($templateId, $localeObj), + 'senderName' => '', + 'senderEmail' => '', + 'replyToEmail' => '', + 'replyToName' => '', + ]), Response::MODEL_EMAIL_TEMPLATE); + return; + } + // Prevent template update if custom SMTP is not configured $smtp = $project->getAttribute('smtp', []); if (($smtp['enabled'] ?? false) !== true) { throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'SMTP must be enabled on the project to configure custom email templates.'); } - // Fetch current configuration - $templates = $project->getAttribute('templates', []); $template = $templates['email.' . $templateId . '-' . $locale] ?? []; // Apply changes @@ -120,25 +151,66 @@ class Update extends Action } } - // Save configuration $templates['email.' . $templateId . '-' . $locale] = $template; - $updates = new Document([ - 'templates' => $templates, - ]); - - $project = $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $updates)); + $authorization->skip(fn () => $dbForPlatform->updateDocument( + 'projects', + $project->getId(), + new Document(['templates' => $templates]) + )); $queueForEvents->setParam('templateId', $templateId); $response->dynamic(new Document([ - 'templateId' => $templateId, - 'locale' => $locale, - 'subject' => $template['subject'], - 'message' => $template['message'], - 'senderName' => $template['senderName'] ?? '', - 'senderEmail' => $template['senderEmail'] ?? '', + 'templateId' => $templateId, + 'locale' => $locale, + 'subject' => $template['subject'], + 'message' => $template['message'], + 'senderName' => $template['senderName'] ?? '', + 'senderEmail' => $template['senderEmail'] ?? '', 'replyToEmail' => $template['replyToEmail'] ?? '', - 'replyToName' => $template['replyToName'] ?? '', + 'replyToName' => $template['replyToName'] ?? '', ]), Response::MODEL_EMAIL_TEMPLATE); } + + private function getDefaultMessage(string $templateId, Locale $localeObj): string + { + $templateConfigs = [ + 'magicSession' => [ + 'file' => 'email-magic-url.tpl', + 'placeholders' => ['optionButton', 'buttonText', 'optionUrl', 'clientInfo', 'securityPhrase'] + ], + 'mfaChallenge' => [ + 'file' => 'email-mfa-challenge.tpl', + 'placeholders' => ['description', 'clientInfo'] + ], + 'otpSession' => [ + 'file' => 'email-otp.tpl', + 'placeholders' => ['description', 'clientInfo', 'securityPhrase'] + ], + 'sessionAlert' => [ + 'file' => 'email-session-alert.tpl', + 'placeholders' => ['body', 'listDevice', 'listIpAddress', 'listCountry', 'footer'] + ], + ]; + + $config = $templateConfigs[$templateId] ?? [ + 'file' => 'email-inner-base.tpl', + 'placeholders' => ['buttonText', 'body', 'footer'] + ]; + + $templateString = file_get_contents(APP_CE_CONFIG_DIR . '/locale/templates/' . $config['file']); + $message = Template::fromString($templateString); + + foreach ($config['placeholders'] as $param) { + $escapeHtml = !in_array($param, ['clientInfo', 'body', 'footer', 'description']); + $message->setParam("{{{$param}}}", $localeObj->getText("emails.{$templateId}.{$param}"), escapeHtml: $escapeHtml); + } + + $message + ->setParam('{{hello}}', $localeObj->getText("emails.{$templateId}.hello")) + ->setParam('{{thanks}}', $localeObj->getText("emails.{$templateId}.thanks")) + ->setParam('{{signature}}', $localeObj->getText("emails.{$templateId}.signature")); + + return $message->render(useContent: true); + } } diff --git a/src/Appwrite/Platform/Modules/Project/Services/Http.php b/src/Appwrite/Platform/Modules/Project/Services/Http.php index 3fe9f63d9e..17eba57735 100644 --- a/src/Appwrite/Platform/Modules/Project/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Project/Services/Http.php @@ -90,6 +90,7 @@ use Appwrite\Platform\Modules\Project\Http\Project\Protocols\Update as UpdatePro use Appwrite\Platform\Modules\Project\Http\Project\Services\Update as UpdateProjectService; use Appwrite\Platform\Modules\Project\Http\Project\SMTP\Tests\Create as CreateSMTPTest; use Appwrite\Platform\Modules\Project\Http\Project\SMTP\Update as UpdateSMTP; +use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\GetDefault as DefaultTemplate; use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\Get as GetTemplate; use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\Update as UpdateTemplate; use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\XList as ListTemplates; @@ -124,6 +125,7 @@ class Http extends Service $this->addAction(ListTemplates::getName(), new ListTemplates()); $this->addAction(GetTemplate::getName(), new GetTemplate()); $this->addAction(UpdateTemplate::getName(), new UpdateTemplate()); + $this->addAction(DefaultTemplate::getName(), new DefaultTemplate()); // Variables $this->addAction(CreateVariable::getName(), new CreateVariable()); From 0e157efabe3aa7ee70ad1aa81f421c1bedb5c583 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Fri, 15 May 2026 17:17:32 +0530 Subject: [PATCH 12/25] chore: fix import ordering in Http.php --- src/Appwrite/Platform/Modules/Project/Services/Http.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Project/Services/Http.php b/src/Appwrite/Platform/Modules/Project/Services/Http.php index 17eba57735..4357f38f4b 100644 --- a/src/Appwrite/Platform/Modules/Project/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Project/Services/Http.php @@ -90,8 +90,8 @@ use Appwrite\Platform\Modules\Project\Http\Project\Protocols\Update as UpdatePro use Appwrite\Platform\Modules\Project\Http\Project\Services\Update as UpdateProjectService; use Appwrite\Platform\Modules\Project\Http\Project\SMTP\Tests\Create as CreateSMTPTest; use Appwrite\Platform\Modules\Project\Http\Project\SMTP\Update as UpdateSMTP; -use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\GetDefault as DefaultTemplate; use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\Get as GetTemplate; +use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\GetDefault as DefaultTemplate; use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\Update as UpdateTemplate; use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\XList as ListTemplates; use Appwrite\Platform\Modules\Project\Http\Project\Variables\Create as CreateVariable; From b7bf14a18a93368077158feb4c5252f1b156efc1 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 15 May 2026 15:58:48 +0400 Subject: [PATCH 13/25] Fix storage multipart upload preparation --- .../Storage/Http/Buckets/Files/Create.php | 86 ++++++++++--------- tests/e2e/Services/Storage/StorageBase.php | 2 +- 2 files changed, 45 insertions(+), 43 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index e9f17fb801..58f7381001 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -265,7 +265,7 @@ class Create extends Action }; try { - $lock->withLock(function () use ($bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, &$metadata, &$completed, $path, $response): void { + $lock->withLock(function () use ($bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $response, &$completed): void { $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); if (!$file->isEmpty()) { $chunks = $file->getAttribute('chunksTotal', 1); @@ -286,7 +286,38 @@ class Create extends Action } } - $deviceForFiles->prepareUpload($path, $metadata['content_type'] ?? '', $chunks, $metadata); + if ($file->isEmpty()) { + $deviceForFiles->prepareUpload($path, $metadata['content_type'] ?? '', $chunks, $metadata); + + if (!empty($contentRange)) { + $doc = new Document([ + '$id' => ID::custom($fileId), + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $fileName, + 'path' => $path, + 'signature' => '', + 'mimeType' => '', + 'sizeOriginal' => $fileSize, + 'sizeActual' => 0, + 'algorithm' => '', + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => 0, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); + + try { + $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + } + } }, timeout: 120.0); } catch (LockContention) { $response->addHeader('Retry-After', '5'); @@ -447,48 +478,19 @@ class Create extends Action // Trigger after create success hook $this->afterCreateSuccess($file); } else { - if ($file->isEmpty()) { - $doc = new Document([ - '$id' => ID::custom($fileId), - '$permissions' => $permissions, - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getSequence(), - 'name' => $fileName, - 'path' => $path, - 'signature' => '', - 'mimeType' => '', - 'sizeOriginal' => $fileSize, - 'sizeActual' => 0, - 'algorithm' => '', - 'comment' => '', - 'chunksTotal' => $chunks, + /** + * Skip authorization in updateDocument. + * Without this, the file creation will fail when user doesn't have update permission. + * However as with chunk upload even if we are updating, we are essentially creating a file + * adding it's new chunk so we rely on the create-permission check performed earlier. + */ + try { + $file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([ 'chunksUploaded' => $chunksUploaded, - 'search' => implode(' ', [$fileId, $fileName]), 'metadata' => $metadata, - ]); - - try { - $file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); - } catch (DuplicateException) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } else { - /** - * Skip authorization in updateDocument. - * Without this, the file creation will fail when user doesn't have update permission. - * However as with chunk upload even if we are updating, we are essentially creating a file - * adding it's new chunk so we rely on the create-permission check performed earlier. - */ - try { - $file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([ - 'chunksUploaded' => $chunksUploaded, - 'metadata' => $metadata, - ]))); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } + ]))); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } } diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index b2d7155208..3049cae5ca 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -397,7 +397,7 @@ trait StorageBase 'bucketId' => ID::unique(), 'name' => 'Test Bucket 2', 'fileSecurity' => true, - 'maximumFileSize' => (int) System::getEnv('_APP_STORAGE_LIMIT', 0) + 1, + 'maximumFileSize' => ((int) System::getEnv('_APP_STORAGE_LIMIT', 0) ?: 6000000000) + 1, 'allowedFileExtensions' => ["jpg", "png"], 'permissions' => [ Permission::read(Role::any()), From a614afe5c7ff4536e7d5dbdb4e89803e2fad9118 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Fri, 15 May 2026 18:03:12 +0530 Subject: [PATCH 14/25] refactor: move email template defaults to console module and add tests - Move GetDefault endpoint to Console module as Get.php (GET /v1/console/templates/email/:templateId, console.getEmailTemplate) - Remove from Project module - Add e2e tests for console.getEmailTemplate and reset=true --- .../Http/Templates/Email/Get.php} | 16 +-- .../Modules/Console/Services/Http.php | 2 + .../Modules/Project/Services/Http.php | 2 - tests/e2e/Services/Project/TemplatesBase.php | 136 ++++++++++++++++++ 4 files changed, 146 insertions(+), 10 deletions(-) rename src/Appwrite/Platform/Modules/{Project/Http/Project/Templates/Email/GetDefault.php => Console/Http/Templates/Email/Get.php} (91%) diff --git a/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/GetDefault.php b/src/Appwrite/Platform/Modules/Console/Http/Templates/Email/Get.php similarity index 91% rename from src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/GetDefault.php rename to src/Appwrite/Platform/Modules/Console/Http/Templates/Email/Get.php index c89c39e0d6..97d48603df 100644 --- a/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/GetDefault.php +++ b/src/Appwrite/Platform/Modules/Console/Http/Templates/Email/Get.php @@ -1,6 +1,6 @@ setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) ->setHttpPath('/v1/console/templates/email/:templateId') - ->desc('Get default project email template') - ->groups(['api', 'project']) - ->label('scope', 'templates.read') + ->desc('Get email template') + ->groups(['api', 'projects']) + ->label('scope', 'projects.read') ->label('sdk', new Method( namespace: 'console', group: 'templates', name: 'getEmailTemplate', description: <<addAction(Web::getName(), new Web()); $this->addAction(GetVariables::getName(), new GetVariables()); + $this->addAction(GetEmailTemplate::getName(), new GetEmailTemplate()); $this->addAction(ListOAuth2Providers::getName(), new ListOAuth2Providers()); $this->addAction(ListKeyScopes::getName(), new ListKeyScopes()); $this->addAction(ListOrganizationScopes::getName(), new ListOrganizationScopes()); diff --git a/src/Appwrite/Platform/Modules/Project/Services/Http.php b/src/Appwrite/Platform/Modules/Project/Services/Http.php index 4357f38f4b..3fe9f63d9e 100644 --- a/src/Appwrite/Platform/Modules/Project/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Project/Services/Http.php @@ -91,7 +91,6 @@ use Appwrite\Platform\Modules\Project\Http\Project\Services\Update as UpdateProj use Appwrite\Platform\Modules\Project\Http\Project\SMTP\Tests\Create as CreateSMTPTest; use Appwrite\Platform\Modules\Project\Http\Project\SMTP\Update as UpdateSMTP; use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\Get as GetTemplate; -use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\GetDefault as DefaultTemplate; use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\Update as UpdateTemplate; use Appwrite\Platform\Modules\Project\Http\Project\Templates\Email\XList as ListTemplates; use Appwrite\Platform\Modules\Project\Http\Project\Variables\Create as CreateVariable; @@ -125,7 +124,6 @@ class Http extends Service $this->addAction(ListTemplates::getName(), new ListTemplates()); $this->addAction(GetTemplate::getName(), new GetTemplate()); $this->addAction(UpdateTemplate::getName(), new UpdateTemplate()); - $this->addAction(DefaultTemplate::getName(), new DefaultTemplate()); // Variables $this->addAction(CreateVariable::getName(), new CreateVariable()); diff --git a/tests/e2e/Services/Project/TemplatesBase.php b/tests/e2e/Services/Project/TemplatesBase.php index b240c945b3..a1e46debe7 100644 --- a/tests/e2e/Services/Project/TemplatesBase.php +++ b/tests/e2e/Services/Project/TemplatesBase.php @@ -1147,6 +1147,142 @@ trait TemplatesBase return $this->client->call(Client::METHOD_PATCH, '/project/templates/email', $headers, $params); } + // Console email template (default) tests + + public function testGetConsoleEmailTemplate(): void + { + $response = $this->getConsoleEmailTemplate('verification', 'en'); + + $this->assertSame(200, $response['headers']['status-code']); + $this->assertSame('verification', $response['body']['templateId']); + $this->assertSame('en', $response['body']['locale']); + $this->assertNotEmpty($response['body']['subject']); + $this->assertNotEmpty($response['body']['message']); + $this->assertSame('', $response['body']['senderName']); + $this->assertSame('', $response['body']['senderEmail']); + $this->assertSame('', $response['body']['replyToEmail']); + $this->assertSame('', $response['body']['replyToName']); + } + + public function testGetConsoleEmailTemplateIgnoresCustomOverride(): void + { + $this->ensureSMTPEnabled(); + + // Set a custom override on the project template. + $this->updateEmailTemplate( + templateId: 'recovery', + locale: 'en', + subject: 'Custom subject', + message: 'Custom message', + senderName: 'Custom Sender', + senderEmail: 'custom@appwrite.io', + ); + + // Console endpoint must always return the built-in default, not the override. + $response = $this->getConsoleEmailTemplate('recovery', 'en'); + + $this->assertSame(200, $response['headers']['status-code']); + $this->assertSame('recovery', $response['body']['templateId']); + $this->assertNotSame('Custom subject', $response['body']['subject']); + $this->assertSame('', $response['body']['senderName']); + $this->assertSame('', $response['body']['senderEmail']); + } + + public function testGetConsoleEmailTemplateDefaultLocale(): void + { + $response = $this->getConsoleEmailTemplate('magicSession'); + + $this->assertSame(200, $response['headers']['status-code']); + $this->assertSame('en', $response['body']['locale']); + $this->assertNotEmpty($response['body']['subject']); + } + + public function testGetConsoleEmailTemplateAllTypes(): void + { + $types = [ + 'verification', + 'magicSession', + 'recovery', + 'invitation', + 'mfaChallenge', + 'sessionAlert', + 'otpSession', + ]; + + foreach ($types as $type) { + $response = $this->getConsoleEmailTemplate($type, 'en'); + $this->assertSame(200, $response['headers']['status-code'], "type={$type}"); + $this->assertNotEmpty($response['body']['subject'], "type={$type} must have subject"); + $this->assertNotEmpty($response['body']['message'], "type={$type} must have message"); + } + } + + public function testResetEmailTemplate(): void + { + $this->ensureSMTPEnabled(); + + // Apply a custom override. + $this->updateEmailTemplate( + templateId: 'invitation', + locale: 'en', + subject: 'Custom invite subject', + message: 'Custom invite message', + senderName: 'Bad Sender', + senderEmail: 'bad@example.com', + ); + + $custom = $this->getEmailTemplate('invitation', 'en'); + $this->assertSame('Custom invite subject', $custom['body']['subject']); + $this->assertSame('Bad Sender', $custom['body']['senderName']); + + // Reset to default. + $response = $this->resetEmailTemplate('invitation', 'en'); + $this->assertSame(200, $response['headers']['status-code']); + $this->assertSame('', $response['body']['senderName']); + $this->assertSame('', $response['body']['senderEmail']); + $this->assertNotSame('Custom invite subject', $response['body']['subject']); + + // Confirm the project template now reflects the default. + $after = $this->getEmailTemplate('invitation', 'en'); + $this->assertSame('', $after['body']['senderName']); + $this->assertSame('', $after['body']['senderEmail']); + } + + public function testResetEmailTemplateWithoutSMTP(): void + { + // Reset must work even without SMTP enabled. + $response = $this->resetEmailTemplate('recovery', 'en'); + $this->assertSame(200, $response['headers']['status-code']); + $this->assertSame('', $response['body']['senderName']); + $this->assertSame('', $response['body']['senderEmail']); + } + + protected function getConsoleEmailTemplate(string $templateId, ?string $locale = null): mixed + { + $params = []; + if ($locale !== null) { + $params['locale'] = $locale; + } + + return $this->client->call(Client::METHOD_GET, '/console/templates/email/' . $templateId, \array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), $params); + } + + protected function resetEmailTemplate(string $templateId, ?string $locale = null): mixed + { + $params = ['templateId' => $templateId, 'reset' => true]; + if ($locale !== null) { + $params['locale'] = $locale; + } + + return $this->client->call(Client::METHOD_PATCH, '/project/templates/email', \array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), $params); + } + protected function ensureSMTPEnabled(): void { $this->client->call( From 8781942633aa99f5e6cdfbb58f70d1925b5834c4 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Fri, 15 May 2026 18:18:10 +0530 Subject: [PATCH 15/25] Fix console email template spec enum names --- src/Appwrite/SDK/Specification/Format.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Appwrite/SDK/Specification/Format.php b/src/Appwrite/SDK/Specification/Format.php index fc67dedb13..d236ee53ce 100644 --- a/src/Appwrite/SDK/Specification/Format.php +++ b/src/Appwrite/SDK/Specification/Format.php @@ -466,6 +466,14 @@ abstract class Format return 'ConsoleResourceValue'; } break; + case 'getEmailTemplate': + switch ($param) { + case 'templateId': + return 'ConsoleEmailTemplateId'; + case 'locale': + return 'ConsoleEmailTemplateLocale'; + } + break; } break; case 'account': From 0e7c50c181ef68b4b81a36a99e0c27c098f08e4e Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Fri, 15 May 2026 18:45:38 +0530 Subject: [PATCH 16/25] Align console email template endpoint metadata --- .../Platform/Modules/Console/Http/Templates/Email/Get.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Console/Http/Templates/Email/Get.php b/src/Appwrite/Platform/Modules/Console/Http/Templates/Email/Get.php index 97d48603df..41b56f2aa6 100644 --- a/src/Appwrite/Platform/Modules/Console/Http/Templates/Email/Get.php +++ b/src/Appwrite/Platform/Modules/Console/Http/Templates/Email/Get.php @@ -29,16 +29,16 @@ class Get extends Action $this->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) ->setHttpPath('/v1/console/templates/email/:templateId') ->desc('Get email template') - ->groups(['api', 'projects']) - ->label('scope', 'projects.read') + ->groups(['api']) + ->label('scope', 'public') ->label('sdk', new Method( namespace: 'console', - group: 'templates', + group: null, name: 'getEmailTemplate', description: << Date: Fri, 15 May 2026 18:51:22 +0530 Subject: [PATCH 17/25] Remove email template reset API changes --- .../Http/Project/Templates/Email/Update.php | 74 ------------------- src/Appwrite/SDK/Specification/Format.php | 4 +- tests/e2e/Services/Project/TemplatesBase.php | 60 +++------------ 3 files changed, 13 insertions(+), 125 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php b/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php index 55824f2102..8fc2f6eec9 100644 --- a/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php +++ b/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php @@ -7,18 +7,14 @@ use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; -use Appwrite\Template\Template; use Appwrite\Utopia\Response; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; use Utopia\Emails\Validator\Email; -use Utopia\Locale\Locale; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; -use Utopia\System\System; -use Utopia\Validator\Boolean; use Utopia\Validator\Nullable; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; @@ -67,7 +63,6 @@ class Update extends Action ->param('senderEmail', null, new Nullable(new Email()), 'Email of the sender.', optional: true) ->param('replyToEmail', null, new Nullable(new Email()), 'Reply to email.', optional: true) ->param('replyToName', null, new Nullable(new Text(255, 0)), 'Reply to name.', optional: true) - ->param('reset', false, new Boolean(), 'Reset template to Appwrite default, removing any custom override.', optional: true) ->inject('response') ->inject('queueForEvents') ->inject('dbForPlatform') @@ -85,7 +80,6 @@ class Update extends Action ?string $senderEmail, ?string $replyToEmail, ?string $replyToName, - bool $reset, Response $response, QueueEvent $queueForEvents, Database $dbForPlatform, @@ -96,32 +90,6 @@ class Update extends Action $templates = $project->getAttribute('templates', []); - if ($reset) { - unset($templates['email.' . $templateId . '-' . $locale]); - $authorization->skip(fn () => $dbForPlatform->updateDocument( - 'projects', - $project->getId(), - new Document(['templates' => $templates]) - )); - - $queueForEvents->setParam('templateId', $templateId); - - $localeObj = new Locale($locale); - $localeObj->setFallback(System::getEnv('_APP_LOCALE', 'en')); - - $response->dynamic(new Document([ - 'templateId' => $templateId, - 'locale' => $locale, - 'subject' => $localeObj->getText('emails.' . $templateId . '.subject'), - 'message' => $this->getDefaultMessage($templateId, $localeObj), - 'senderName' => '', - 'senderEmail' => '', - 'replyToEmail' => '', - 'replyToName' => '', - ]), Response::MODEL_EMAIL_TEMPLATE); - return; - } - // Prevent template update if custom SMTP is not configured $smtp = $project->getAttribute('smtp', []); if (($smtp['enabled'] ?? false) !== true) { @@ -171,46 +139,4 @@ class Update extends Action 'replyToName' => $template['replyToName'] ?? '', ]), Response::MODEL_EMAIL_TEMPLATE); } - - private function getDefaultMessage(string $templateId, Locale $localeObj): string - { - $templateConfigs = [ - 'magicSession' => [ - 'file' => 'email-magic-url.tpl', - 'placeholders' => ['optionButton', 'buttonText', 'optionUrl', 'clientInfo', 'securityPhrase'] - ], - 'mfaChallenge' => [ - 'file' => 'email-mfa-challenge.tpl', - 'placeholders' => ['description', 'clientInfo'] - ], - 'otpSession' => [ - 'file' => 'email-otp.tpl', - 'placeholders' => ['description', 'clientInfo', 'securityPhrase'] - ], - 'sessionAlert' => [ - 'file' => 'email-session-alert.tpl', - 'placeholders' => ['body', 'listDevice', 'listIpAddress', 'listCountry', 'footer'] - ], - ]; - - $config = $templateConfigs[$templateId] ?? [ - 'file' => 'email-inner-base.tpl', - 'placeholders' => ['buttonText', 'body', 'footer'] - ]; - - $templateString = file_get_contents(APP_CE_CONFIG_DIR . '/locale/templates/' . $config['file']); - $message = Template::fromString($templateString); - - foreach ($config['placeholders'] as $param) { - $escapeHtml = !in_array($param, ['clientInfo', 'body', 'footer', 'description']); - $message->setParam("{{{$param}}}", $localeObj->getText("emails.{$templateId}.{$param}"), escapeHtml: $escapeHtml); - } - - $message - ->setParam('{{hello}}', $localeObj->getText("emails.{$templateId}.hello")) - ->setParam('{{thanks}}', $localeObj->getText("emails.{$templateId}.thanks")) - ->setParam('{{signature}}', $localeObj->getText("emails.{$templateId}.signature")); - - return $message->render(useContent: true); - } } diff --git a/src/Appwrite/SDK/Specification/Format.php b/src/Appwrite/SDK/Specification/Format.php index d236ee53ce..6c5d50e016 100644 --- a/src/Appwrite/SDK/Specification/Format.php +++ b/src/Appwrite/SDK/Specification/Format.php @@ -469,9 +469,9 @@ abstract class Format case 'getEmailTemplate': switch ($param) { case 'templateId': - return 'ConsoleEmailTemplateId'; + return 'ProjectEmailTemplateId'; case 'locale': - return 'ConsoleEmailTemplateLocale'; + return 'ProjectEmailTemplateLocale'; } break; } diff --git a/tests/e2e/Services/Project/TemplatesBase.php b/tests/e2e/Services/Project/TemplatesBase.php index a1e46debe7..15c24f74c9 100644 --- a/tests/e2e/Services/Project/TemplatesBase.php +++ b/tests/e2e/Services/Project/TemplatesBase.php @@ -1217,44 +1217,18 @@ trait TemplatesBase } } - public function testResetEmailTemplate(): void + public function testGetConsoleEmailTemplateInvalidTemplateId(): void { - $this->ensureSMTPEnabled(); + $response = $this->getConsoleEmailTemplate('invalidTemplate', 'en'); - // Apply a custom override. - $this->updateEmailTemplate( - templateId: 'invitation', - locale: 'en', - subject: 'Custom invite subject', - message: 'Custom invite message', - senderName: 'Bad Sender', - senderEmail: 'bad@example.com', - ); - - $custom = $this->getEmailTemplate('invitation', 'en'); - $this->assertSame('Custom invite subject', $custom['body']['subject']); - $this->assertSame('Bad Sender', $custom['body']['senderName']); - - // Reset to default. - $response = $this->resetEmailTemplate('invitation', 'en'); - $this->assertSame(200, $response['headers']['status-code']); - $this->assertSame('', $response['body']['senderName']); - $this->assertSame('', $response['body']['senderEmail']); - $this->assertNotSame('Custom invite subject', $response['body']['subject']); - - // Confirm the project template now reflects the default. - $after = $this->getEmailTemplate('invitation', 'en'); - $this->assertSame('', $after['body']['senderName']); - $this->assertSame('', $after['body']['senderEmail']); + $this->assertSame(400, $response['headers']['status-code']); } - public function testResetEmailTemplateWithoutSMTP(): void + public function testGetConsoleEmailTemplateInvalidLocale(): void { - // Reset must work even without SMTP enabled. - $response = $this->resetEmailTemplate('recovery', 'en'); - $this->assertSame(200, $response['headers']['status-code']); - $this->assertSame('', $response['body']['senderName']); - $this->assertSame('', $response['body']['senderEmail']); + $response = $this->getConsoleEmailTemplate('recovery', 'not-a-locale'); + + $this->assertSame(400, $response['headers']['status-code']); } protected function getConsoleEmailTemplate(string $templateId, ?string $locale = null): mixed @@ -1264,23 +1238,11 @@ trait TemplatesBase $params['locale'] = $locale; } - return $this->client->call(Client::METHOD_GET, '/console/templates/email/' . $templateId, \array_merge([ + return $this->client->call(Client::METHOD_GET, '/console/templates/email/' . $templateId, [ 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), $params); - } - - protected function resetEmailTemplate(string $templateId, ?string $locale = null): mixed - { - $params = ['templateId' => $templateId, 'reset' => true]; - if ($locale !== null) { - $params['locale'] = $locale; - } - - return $this->client->call(Client::METHOD_PATCH, '/project/templates/email', \array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), $params); + 'x-appwrite-project' => 'console', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ], $params); } protected function ensureSMTPEnabled(): void From 7b718e7dcc92a9a2ee0e58677bc0b9587338c86f Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Fri, 15 May 2026 18:57:08 +0530 Subject: [PATCH 18/25] Add email template default integration test --- .../Console/Http/Templates/Email/Get.php | 5 + .../Project/EmailTemplatesIntegrationTest.php | 194 ++++++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 tests/e2e/Services/Project/EmailTemplatesIntegrationTest.php diff --git a/src/Appwrite/Platform/Modules/Console/Http/Templates/Email/Get.php b/src/Appwrite/Platform/Modules/Console/Http/Templates/Email/Get.php index 41b56f2aa6..6906c1fd79 100644 --- a/src/Appwrite/Platform/Modules/Console/Http/Templates/Email/Get.php +++ b/src/Appwrite/Platform/Modules/Console/Http/Templates/Email/Get.php @@ -105,6 +105,11 @@ class Get extends Action foreach ($config['placeholders'] as $param) { $escapeHtml = !in_array($param, ['clientInfo', 'body', 'footer', 'description']); + if ($templateId === 'magicSession' && $param === 'securityPhrase') { + $message->setParam('{{securityPhrase}}', ''); + continue; + } + $message->setParam("{{{$param}}}", $localeObj->getText("emails.{$templateId}.{$param}"), escapeHtml: $escapeHtml); } diff --git a/tests/e2e/Services/Project/EmailTemplatesIntegrationTest.php b/tests/e2e/Services/Project/EmailTemplatesIntegrationTest.php new file mode 100644 index 0000000000..7d8a724c38 --- /dev/null +++ b/tests/e2e/Services/Project/EmailTemplatesIntegrationTest.php @@ -0,0 +1,194 @@ +clearMaildev(); + + $recipientEmail = 'magic-template-' . \uniqid() . '@appwrite.io'; + + $this->updateSMTP(['enabled' => false]); + + $firstEmail = $this->triggerMagicUrlAndGetEmail($recipientEmail); + $defaultSnapshot = $this->normalizeMagicUrlEmail($firstEmail); + + $this->updateSMTP([ + 'enabled' => true, + 'senderName' => 'Template Test Mailer', + 'senderEmail' => 'template-test@appwrite.io', + 'host' => 'maildev', + 'port' => 1025, + 'username' => 'user', + 'password' => 'password', + ]); + + $customSubject = 'Custom magic login ' . \uniqid(); + $customMarker = 'CUSTOM_MAGIC_TEMPLATE_' . \uniqid(); + $this->updateEmailTemplate([ + 'templateId' => 'magicSession', + 'locale' => 'en', + 'subject' => $customSubject, + 'message' => '

' . $customMarker . '

{{redirect}}

', + ]); + + $customEmail = $this->triggerMagicUrlAndGetEmail($recipientEmail); + $this->assertSame($customSubject, $customEmail['subject']); + $this->assertStringContainsString($customMarker, $customEmail['text']); + $this->assertStringContainsString($customMarker, $customEmail['html']); + + $defaultTemplate = $this->getConsoleEmailTemplate('magicSession', 'en'); + $this->assertSame(200, $defaultTemplate['headers']['status-code']); + + $this->updateEmailTemplate([ + 'templateId' => 'magicSession', + 'locale' => 'en', + 'subject' => $defaultTemplate['body']['subject'], + 'message' => $defaultTemplate['body']['message'], + ]); + + $restoredEmail = $this->triggerMagicUrlAndGetEmail($recipientEmail); + $restoredSnapshot = $this->normalizeMagicUrlEmail($restoredEmail); + + $this->assertSame($defaultSnapshot, $restoredSnapshot); + } + + /** + * @param array $params + */ + private function updateSMTP(array $params): void + { + $response = $this->client->call(Client::METHOD_PATCH, '/project/smtp', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $params); + + $this->assertSame(200, $response['headers']['status-code']); + } + + /** + * @param array $params + */ + private function updateEmailTemplate(array $params): void + { + $response = $this->client->call(Client::METHOD_PATCH, '/project/templates/email', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $params); + + $this->assertSame(200, $response['headers']['status-code']); + } + + private function getConsoleEmailTemplate(string $templateId, string $locale): array + { + return $this->client->call(Client::METHOD_GET, '/console/templates/email/' . $templateId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ], [ + 'locale' => $locale, + ]); + } + + private function triggerMagicUrlAndGetEmail(string $recipientEmail): array + { + $previousCount = $this->countEmailsTo($recipientEmail); + + $response = $this->client->call(Client::METHOD_POST, '/account/tokens/magic-url', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'userId' => ID::unique(), + 'email' => $recipientEmail, + ]); + + $this->assertSame(201, $response['headers']['status-code']); + + return $this->getNextEmailByAddress($recipientEmail, $previousCount); + } + + private function countEmailsTo(string $address): int + { + $emails = \json_decode(\file_get_contents('http://maildev:1080/email'), true) ?? []; + $count = 0; + + foreach ($emails as $email) { + foreach ($email['to'] ?? [] as $recipient) { + if (($recipient['address'] ?? '') === $address) { + $count++; + } + } + } + + return $count; + } + + private function clearMaildev(): void + { + $context = \stream_context_create([ + 'http' => [ + 'method' => 'DELETE', + ], + ]); + + \file_get_contents('http://maildev:1080/email/all', false, $context); + } + + private function getNextEmailByAddress(string $address, int $previousCount): array + { + $result = []; + + $this->assertEventually(function () use (&$result, $address, $previousCount) { + $emails = \json_decode(\file_get_contents('http://maildev:1080/email'), true) ?? []; + $matches = []; + + foreach ($emails as $email) { + foreach ($email['to'] ?? [] as $recipient) { + if (($recipient['address'] ?? '') === $address) { + $matches[] = $email; + break; + } + } + } + + $this->assertGreaterThan($previousCount, \count($matches), 'Expected a new email for ' . $address); + $result = $matches[\count($matches) - 1]; + }, 15_000, 500); + + return $result; + } + + /** + * @return array{subject: string, text: string, html: string} + */ + private function normalizeMagicUrlEmail(array $email): array + { + return [ + 'subject' => $this->normalizeMagicUrlContent($email['subject'] ?? ''), + 'text' => $this->normalizeMagicUrlContent($email['text'] ?? ''), + 'html' => $this->normalizeMagicUrlContent($email['html'] ?? ''), + ]; + } + + private function normalizeMagicUrlContent(string $content): string + { + $content = \html_entity_decode($content, ENT_QUOTES); + $content = \preg_replace('/([?&](?:secret|expire)=)[^&\s<"]+/', '$1{value}', $content) ?? $content; + + return \trim($content); + } +} From 6428b7396630d47f16086fe55e3d5c4a2362749a Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Fri, 15 May 2026 19:05:01 +0530 Subject: [PATCH 19/25] Fix email template update System import --- .../Modules/Project/Http/Project/Templates/Email/Update.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php b/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php index 8fc2f6eec9..256ffcf7d4 100644 --- a/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php +++ b/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php @@ -15,6 +15,7 @@ use Utopia\Database\Validator\Authorization; use Utopia\Emails\Validator\Email; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; +use Utopia\System\System; use Utopia\Validator\Nullable; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; From 6a2ddb221cfd0b0a02f5d571cc7c7c66d21e964b Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Fri, 15 May 2026 19:34:15 +0530 Subject: [PATCH 20/25] Clean up template update diff and add locale test --- .../Http/Project/Templates/Email/Update.php | 29 ++++++++++--------- tests/e2e/Services/Project/TemplatesBase.php | 11 +++++++ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php b/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php index 256ffcf7d4..ef93abf683 100644 --- a/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php +++ b/src/Appwrite/Platform/Modules/Project/Http/Project/Templates/Email/Update.php @@ -89,14 +89,14 @@ class Update extends Action ) { $locale = $locale ?: System::getEnv('_APP_LOCALE', 'en'); - $templates = $project->getAttribute('templates', []); - // Prevent template update if custom SMTP is not configured $smtp = $project->getAttribute('smtp', []); if (($smtp['enabled'] ?? false) !== true) { throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'SMTP must be enabled on the project to configure custom email templates.'); } + // Fetch current configuration + $templates = $project->getAttribute('templates', []); $template = $templates['email.' . $templateId . '-' . $locale] ?? []; // Apply changes @@ -120,24 +120,25 @@ class Update extends Action } } + // Save configuration $templates['email.' . $templateId . '-' . $locale] = $template; - $authorization->skip(fn () => $dbForPlatform->updateDocument( - 'projects', - $project->getId(), - new Document(['templates' => $templates]) - )); + $updates = new Document([ + 'templates' => $templates, + ]); + + $project = $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $updates)); $queueForEvents->setParam('templateId', $templateId); $response->dynamic(new Document([ - 'templateId' => $templateId, - 'locale' => $locale, - 'subject' => $template['subject'], - 'message' => $template['message'], - 'senderName' => $template['senderName'] ?? '', - 'senderEmail' => $template['senderEmail'] ?? '', + 'templateId' => $templateId, + 'locale' => $locale, + 'subject' => $template['subject'], + 'message' => $template['message'], + 'senderName' => $template['senderName'] ?? '', + 'senderEmail' => $template['senderEmail'] ?? '', 'replyToEmail' => $template['replyToEmail'] ?? '', - 'replyToName' => $template['replyToName'] ?? '', + 'replyToName' => $template['replyToName'] ?? '', ]), Response::MODEL_EMAIL_TEMPLATE); } } diff --git a/tests/e2e/Services/Project/TemplatesBase.php b/tests/e2e/Services/Project/TemplatesBase.php index 15c24f74c9..11dc6dc80b 100644 --- a/tests/e2e/Services/Project/TemplatesBase.php +++ b/tests/e2e/Services/Project/TemplatesBase.php @@ -1197,6 +1197,17 @@ trait TemplatesBase $this->assertNotEmpty($response['body']['subject']); } + public function testGetConsoleEmailTemplateNonDefaultLocale(): void + { + $response = $this->getConsoleEmailTemplate('verification', 'fr'); + + $this->assertSame(200, $response['headers']['status-code']); + $this->assertSame('verification', $response['body']['templateId']); + $this->assertSame('fr', $response['body']['locale']); + $this->assertNotEmpty($response['body']['subject']); + $this->assertNotEmpty($response['body']['message']); + } + public function testGetConsoleEmailTemplateAllTypes(): void { $types = [ From cb80f0e9c892e39ecf7c6bd24e97ba5cdab2205e Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Fri, 15 May 2026 19:44:20 +0530 Subject: [PATCH 21/25] Remove email template integration test --- .../Project/EmailTemplatesIntegrationTest.php | 194 ------------------ 1 file changed, 194 deletions(-) delete mode 100644 tests/e2e/Services/Project/EmailTemplatesIntegrationTest.php diff --git a/tests/e2e/Services/Project/EmailTemplatesIntegrationTest.php b/tests/e2e/Services/Project/EmailTemplatesIntegrationTest.php deleted file mode 100644 index 7d8a724c38..0000000000 --- a/tests/e2e/Services/Project/EmailTemplatesIntegrationTest.php +++ /dev/null @@ -1,194 +0,0 @@ -clearMaildev(); - - $recipientEmail = 'magic-template-' . \uniqid() . '@appwrite.io'; - - $this->updateSMTP(['enabled' => false]); - - $firstEmail = $this->triggerMagicUrlAndGetEmail($recipientEmail); - $defaultSnapshot = $this->normalizeMagicUrlEmail($firstEmail); - - $this->updateSMTP([ - 'enabled' => true, - 'senderName' => 'Template Test Mailer', - 'senderEmail' => 'template-test@appwrite.io', - 'host' => 'maildev', - 'port' => 1025, - 'username' => 'user', - 'password' => 'password', - ]); - - $customSubject = 'Custom magic login ' . \uniqid(); - $customMarker = 'CUSTOM_MAGIC_TEMPLATE_' . \uniqid(); - $this->updateEmailTemplate([ - 'templateId' => 'magicSession', - 'locale' => 'en', - 'subject' => $customSubject, - 'message' => '

' . $customMarker . '

{{redirect}}

', - ]); - - $customEmail = $this->triggerMagicUrlAndGetEmail($recipientEmail); - $this->assertSame($customSubject, $customEmail['subject']); - $this->assertStringContainsString($customMarker, $customEmail['text']); - $this->assertStringContainsString($customMarker, $customEmail['html']); - - $defaultTemplate = $this->getConsoleEmailTemplate('magicSession', 'en'); - $this->assertSame(200, $defaultTemplate['headers']['status-code']); - - $this->updateEmailTemplate([ - 'templateId' => 'magicSession', - 'locale' => 'en', - 'subject' => $defaultTemplate['body']['subject'], - 'message' => $defaultTemplate['body']['message'], - ]); - - $restoredEmail = $this->triggerMagicUrlAndGetEmail($recipientEmail); - $restoredSnapshot = $this->normalizeMagicUrlEmail($restoredEmail); - - $this->assertSame($defaultSnapshot, $restoredSnapshot); - } - - /** - * @param array $params - */ - private function updateSMTP(array $params): void - { - $response = $this->client->call(Client::METHOD_PATCH, '/project/smtp', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], $params); - - $this->assertSame(200, $response['headers']['status-code']); - } - - /** - * @param array $params - */ - private function updateEmailTemplate(array $params): void - { - $response = $this->client->call(Client::METHOD_PATCH, '/project/templates/email', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], $params); - - $this->assertSame(200, $response['headers']['status-code']); - } - - private function getConsoleEmailTemplate(string $templateId, string $locale): array - { - return $this->client->call(Client::METHOD_GET, '/console/templates/email/' . $templateId, [ - 'content-type' => 'application/json', - 'x-appwrite-project' => 'console', - 'cookie' => 'a_session_console=' . $this->getRoot()['session'], - ], [ - 'locale' => $locale, - ]); - } - - private function triggerMagicUrlAndGetEmail(string $recipientEmail): array - { - $previousCount = $this->countEmailsTo($recipientEmail); - - $response = $this->client->call(Client::METHOD_POST, '/account/tokens/magic-url', [ - 'origin' => 'http://localhost', - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], [ - 'userId' => ID::unique(), - 'email' => $recipientEmail, - ]); - - $this->assertSame(201, $response['headers']['status-code']); - - return $this->getNextEmailByAddress($recipientEmail, $previousCount); - } - - private function countEmailsTo(string $address): int - { - $emails = \json_decode(\file_get_contents('http://maildev:1080/email'), true) ?? []; - $count = 0; - - foreach ($emails as $email) { - foreach ($email['to'] ?? [] as $recipient) { - if (($recipient['address'] ?? '') === $address) { - $count++; - } - } - } - - return $count; - } - - private function clearMaildev(): void - { - $context = \stream_context_create([ - 'http' => [ - 'method' => 'DELETE', - ], - ]); - - \file_get_contents('http://maildev:1080/email/all', false, $context); - } - - private function getNextEmailByAddress(string $address, int $previousCount): array - { - $result = []; - - $this->assertEventually(function () use (&$result, $address, $previousCount) { - $emails = \json_decode(\file_get_contents('http://maildev:1080/email'), true) ?? []; - $matches = []; - - foreach ($emails as $email) { - foreach ($email['to'] ?? [] as $recipient) { - if (($recipient['address'] ?? '') === $address) { - $matches[] = $email; - break; - } - } - } - - $this->assertGreaterThan($previousCount, \count($matches), 'Expected a new email for ' . $address); - $result = $matches[\count($matches) - 1]; - }, 15_000, 500); - - return $result; - } - - /** - * @return array{subject: string, text: string, html: string} - */ - private function normalizeMagicUrlEmail(array $email): array - { - return [ - 'subject' => $this->normalizeMagicUrlContent($email['subject'] ?? ''), - 'text' => $this->normalizeMagicUrlContent($email['text'] ?? ''), - 'html' => $this->normalizeMagicUrlContent($email['html'] ?? ''), - ]; - } - - private function normalizeMagicUrlContent(string $content): string - { - $content = \html_entity_decode($content, ENT_QUOTES); - $content = \preg_replace('/([?&](?:secret|expire)=)[^&\s<"]+/', '$1{value}', $content) ?? $content; - - return \trim($content); - } -} From 5750155591322e06f300d81a51490454a87a5672 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 15 May 2026 19:54:44 +0400 Subject: [PATCH 22/25] Add pool-backed lock resource --- app/init/registers.php | 8 ++++++++ app/init/resources.php | 11 +++++++++++ .../Modules/Functions/Http/Deployments/Create.php | 11 ++++------- .../Modules/Sites/Http/Deployments/Create.php | 11 ++++------- .../Modules/Storage/Http/Buckets/Files/Create.php | 15 +++++---------- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/app/init/registers.php b/app/init/registers.php index 54c0053a33..21ce536a8b 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -240,6 +240,12 @@ $register->set('pools', function () { 'multiple' => true, 'schemes' => ['redis'], ], + 'lock' => [ + 'type' => 'lock', + 'dsns' => $fallbackForRedis, + 'multiple' => false, + 'schemes' => ['redis'], + ], ]; $maxConnections = (int) System::getEnv('_APP_CONNECTIONS_MAX', 151); @@ -369,6 +375,8 @@ $register->set('pools', function () { } return $adapter; + case 'lock': + return $resource(); default: throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); } diff --git a/app/init/resources.php b/app/init/resources.php index 92b581157f..d48a60c06c 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -29,6 +29,7 @@ use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; use Utopia\DI\Container; use Utopia\DSN\DSN; +use Utopia\Lock\Distributed; use Utopia\Pools\Group; use Utopia\Queue\Broker\Pool as BrokerPool; use Utopia\Queue\Publisher; @@ -248,6 +249,16 @@ $container->set('redis', function () { return $redis; }); +$container->set('locks', function (Group $pools) { + return function (string $key, int $ttl, callable $callback, float $timeout = 0.0) use ($pools): mixed { + return $pools->get('lock')->use(function (\Redis $redis) use ($key, $ttl, $callback, $timeout) { + $lock = new Distributed($redis, $key, ttl: $ttl); + + return $lock->withLock($callback, timeout: $timeout); + }); + }; +}, ['pools']); + $container->set('timelimit', function (\Redis $redis) { return function (string $key, int $limit, int $time) use ($redis) { return new TimeLimitRedis($key, $limit, $time, $redis); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php index 1718f7359f..9af5491598 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php @@ -21,7 +21,6 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; -use Utopia\Lock\Distributed; use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; @@ -94,7 +93,7 @@ class Create extends Action ->inject('plan') ->inject('authorization') ->inject('platform') - ->inject('redis') + ->inject('locks') ->callback($this->action(...)); } @@ -115,7 +114,7 @@ class Create extends Action array $plan, Authorization $authorization, array $platform, - \Redis $redis + callable $locks ) { $activate = \strval($activate) === 'true' || \strval($activate) === '1'; @@ -199,14 +198,12 @@ class Create extends Action $path = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); $lockKey = 'functions:deployment:' . $project->getId() . ':' . $functionId . ':' . $deploymentId; - $checkLock = new Distributed($redis, $lockKey, ttl: 600); - $stateLock = new Distributed($redis, $lockKey, ttl: 600); $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; $completed = false; try { - $checkLock->withLock(function () use (&$chunks, $dbForProject, $deploymentId, &$metadata, &$completed, $response): void { + $locks($lockKey, 600, function () use (&$chunks, $dbForProject, $deploymentId, &$metadata, &$completed, $response): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); if (!$deployment->isEmpty()) { @@ -242,7 +239,7 @@ class Create extends Action $type = $request->getHeader('x-sdk-language') === 'cli' ? 'cli' : 'manual'; try { - $stateLock->withLock(function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $platform, $project, $publisherForBuilds, $queueForEvents, $response, $type): void { + $locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $platform, $project, $publisherForBuilds, $queueForEvents, $response, $type): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); $uploaded = 0; diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php index 2c2c34e913..d27755d106 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php @@ -21,7 +21,6 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; -use Utopia\Lock\Distributed; use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; @@ -92,7 +91,7 @@ class Create extends Action ->inject('plan') ->inject('authorization') ->inject('platform') - ->inject('redis') + ->inject('locks') ->callback($this->action(...)); } @@ -115,7 +114,7 @@ class Create extends Action array $plan, Authorization $authorization, array $platform, - \Redis $redis, + callable $locks, ) { $activate = \strval($activate) === 'true' || \strval($activate) === '1'; @@ -199,14 +198,12 @@ class Create extends Action $path = $deviceForSites->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); $lockKey = 'sites:deployment:' . $project->getId() . ':' . $siteId . ':' . $deploymentId; - $checkLock = new Distributed($redis, $lockKey, ttl: 600); - $stateLock = new Distributed($redis, $lockKey, ttl: 600); $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; $completed = false; try { - $checkLock->withLock(function () use (&$chunks, $dbForProject, $deploymentId, &$metadata, &$completed, $response): void { + $locks($lockKey, 600, function () use (&$chunks, $dbForProject, $deploymentId, &$metadata, &$completed, $response): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); if (!$deployment->isEmpty()) { @@ -250,7 +247,7 @@ class Create extends Action } try { - $stateLock->withLock(function () use ($activate, $authorization, $commands, &$chunks, $chunksUploaded, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $outputDirectory, $path, $platform, $project, $publisherForBuilds, $queueForEvents, $response, &$site, $siteId, $type): void { + $locks($lockKey, 600, function () use ($activate, $authorization, $commands, &$chunks, $chunksUploaded, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $outputDirectory, $path, $platform, $project, $publisherForBuilds, $queueForEvents, $response, &$site, $siteId, $type): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); $uploaded = 0; diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index 58f7381001..8530475f0c 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -29,7 +29,6 @@ use Utopia\Database\Validator\Authorization\Input; use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; -use Utopia\Lock\Distributed; use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; @@ -94,7 +93,7 @@ class Create extends Action ->inject('deviceForFiles') ->inject('deviceForLocal') ->inject('authorization') - ->inject('redis') + ->inject('locks') ->callback($this->action(...)); } @@ -112,7 +111,7 @@ class Create extends Action Device $deviceForFiles, Device $deviceForLocal, Authorization $authorization, - \Redis $redis + callable $locks ) { $bucket = $authorization->skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); @@ -238,11 +237,7 @@ class Create extends Action $path = $deviceForFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); $path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root - $lock = new Distributed( - $redis, - 'storage:file:' . $project->getId() . ':' . $bucket->getId() . ':' . $fileId, - ttl: 600, - ); + $lockKey = 'storage:file:' . $project->getId() . ':' . $bucket->getId() . ':' . $fileId; $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; $completed = false; @@ -265,7 +260,7 @@ class Create extends Action }; try { - $lock->withLock(function () use ($bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $response, &$completed): void { + $locks($lockKey, 600, function () use ($bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $response, &$completed): void { $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); if (!$file->isEmpty()) { $chunks = $file->getAttribute('chunksTotal', 1); @@ -515,7 +510,7 @@ class Create extends Action throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file'); } - $lock->withLock(fn () => $finalizeUpload($chunksUploaded), timeout: 120.0); + $locks($lockKey, 600, fn () => $finalizeUpload($chunksUploaded), timeout: 120.0); } catch (LockContention) { $response->addHeader('Retry-After', '5'); throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'File upload is busy. Try again.'); From 27bf229524fe6e5e297830a29caafe565648cb5f Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 15 May 2026 21:12:14 +0400 Subject: [PATCH 23/25] Simplify storage test assumptions --- tests/e2e/Services/Storage/StorageBase.php | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 3049cae5ca..375e526fcf 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -11,8 +11,6 @@ use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Database\Validator\Datetime as DatetimeValidator; -use Utopia\Storage\Storage; -use Utopia\System\System; trait StorageBase { @@ -289,11 +287,7 @@ trait StorageBase $this->assertEquals('large-file.mp4', $largeFile['body']['name']); $this->assertEquals('video/mp4', $largeFile['body']['mimeType']); $this->assertEquals($totalSize, $largeFile['body']['sizeOriginal']); - if (System::getEnv('_APP_STORAGE_DEVICE') === Storage::DEVICE_S3 || str_starts_with(System::getEnv('_APP_CONNECTIONS_STORAGE', ''), Storage::DEVICE_S3 . '://')) { - $this->assertNotEmpty($largeFile['body']['signature']); - } else { - $this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $largeFile['body']['signature']); // should validate that the file is not encrypted - } + $this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $largeFile['body']['signature']); // should validate that the file is not encrypted /** * Failure @@ -397,7 +391,7 @@ trait StorageBase 'bucketId' => ID::unique(), 'name' => 'Test Bucket 2', 'fileSecurity' => true, - 'maximumFileSize' => ((int) System::getEnv('_APP_STORAGE_LIMIT', 0) ?: 6000000000) + 1, + 'maximumFileSize' => 6000000001, 'allowedFileExtensions' => ["jpg", "png"], 'permissions' => [ Permission::read(Role::any()), From 37921a569430080f6476701c8b580973bf681390 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 15 May 2026 21:20:20 +0400 Subject: [PATCH 24/25] Remove utopia-php/lock vcs repo --- composer.json | 6 ------ composer.lock | 29 ++++------------------------- 2 files changed, 4 insertions(+), 31 deletions(-) diff --git a/composer.json b/composer.json index 646112438c..400e3c1822 100644 --- a/composer.json +++ b/composer.json @@ -112,12 +112,6 @@ "provide": { "ext-phpiredis": "*" }, - "repositories": [ - { - "type": "vcs", - "url": "https://github.com/utopia-php/lock" - } - ], "config": { "platform": { }, diff --git a/composer.lock b/composer.lock index ff112ce5b1..66d8f62925 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6bc4a0d273ff6b032b36fe205847d02a", + "content-hash": "035685d1335039f13e16d0532c874b21", "packages": [ { "name": "adhocore/jwt", @@ -4532,28 +4532,7 @@ "Utopia\\Lock\\": "src/" } }, - "autoload-dev": { - "psr-4": { - "Utopia\\Lock\\Tests\\": "tests/" - } - }, - "scripts": { - "test": [ - "vendor/bin/phpunit" - ], - "lint": [ - "vendor/bin/pint --test" - ], - "format": [ - "vendor/bin/pint" - ], - "format:check": [ - "vendor/bin/pint --test" - ], - "analyze": [ - "vendor/bin/phpstan analyse --memory-limit=512M" - ] - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -4565,8 +4544,8 @@ ], "description": "Mutex, semaphore, file and distributed locks for PHP — one interface, four backends.", "support": { - "source": "https://github.com/utopia-php/lock/tree/0.2.0", - "issues": "https://github.com/utopia-php/lock/issues" + "issues": "https://github.com/utopia-php/lock/issues", + "source": "https://github.com/utopia-php/lock/tree/0.2.0" }, "time": "2026-04-24T10:47:56+00:00" }, From 1b945bdeeabee58773f1d1b5cf5046c8f26076b9 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Sun, 17 May 2026 17:11:05 +0100 Subject: [PATCH 25/25] chore: bump utopia-php/cache to ^3.0 Co-Authored-By: Claude Opus 4.7 --- composer.json | 4 ++-- composer.lock | 58 +++++++++++++++++++++++++-------------------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/composer.json b/composer.json index 400e3c1822..d4472c6868 100644 --- a/composer.json +++ b/composer.json @@ -56,14 +56,14 @@ "utopia-php/analytics": "0.15.*", "utopia-php/audit": "2.3.*", "utopia-php/auth": "0.5.*", - "utopia-php/cache": "^2.1", + "utopia-php/cache": "^3.0", "utopia-php/cli": "0.23.*", "utopia-php/compression": "0.1.*", "utopia-php/config": "1.*", "utopia-php/console": "0.1.*", "utopia-php/database": "5.*", "utopia-php/detector": "0.2.*", - "utopia-php/domains": "2.*", + "utopia-php/domains": "^2.1", "utopia-php/emails": "0.7.*", "utopia-php/dns": "1.7.*", "utopia-php/dsn": "0.2.1", diff --git a/composer.lock b/composer.lock index 66d8f62925..1d71d75447 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "035685d1335039f13e16d0532c874b21", + "content-hash": "b645c7da1728536497fe8186b158f9c6", "packages": [ { "name": "adhocore/jwt", @@ -3615,16 +3615,16 @@ }, { "name": "utopia-php/cache", - "version": "2.1.0", + "version": "3.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "fc3b9ae33c4b83e0e2c91ecf60b4f40fb7ee8f8e" + "reference": "ece1f4d11ec2804cd7e05b9717dc7a2bc66e4176" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/fc3b9ae33c4b83e0e2c91ecf60b4f40fb7ee8f8e", - "reference": "fc3b9ae33c4b83e0e2c91ecf60b4f40fb7ee8f8e", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/ece1f4d11ec2804cd7e05b9717dc7a2bc66e4176", + "reference": "ece1f4d11ec2804cd7e05b9717dc7a2bc66e4176", "shasum": "" }, "require": { @@ -3663,9 +3663,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/2.1.0" + "source": "https://github.com/utopia-php/cache/tree/3.0.0" }, - "time": "2026-05-12T15:03:23+00:00" + "time": "2026-05-14T14:13:17+00:00" }, { "name": "utopia-php/circuit-breaker", @@ -3923,16 +3923,16 @@ }, { "name": "utopia-php/database", - "version": "5.8.0", + "version": "5.9.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "3391c97318f0e7f94d2c1ea0f7d09e5ba8aad696" + "reference": "477bae83e27631f78c159f45b0441c0c7dc69050" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/3391c97318f0e7f94d2c1ea0f7d09e5ba8aad696", - "reference": "3391c97318f0e7f94d2c1ea0f7d09e5ba8aad696", + "url": "https://api.github.com/repos/utopia-php/database/zipball/477bae83e27631f78c159f45b0441c0c7dc69050", + "reference": "477bae83e27631f78c159f45b0441c0c7dc69050", "shasum": "" }, "require": { @@ -3941,7 +3941,7 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.4", - "utopia-php/cache": "^2.0", + "utopia-php/cache": "^3.0", "utopia-php/console": "0.1.*", "utopia-php/mongo": "1.*", "utopia-php/pools": "1.*", @@ -3977,9 +3977,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/5.8.0" + "source": "https://github.com/utopia-php/database/tree/5.9.0" }, - "time": "2026-05-12T12:52:44+00:00" + "time": "2026-05-17T15:57:21+00:00" }, { "name": "utopia-php/detector", @@ -4136,21 +4136,21 @@ }, { "name": "utopia-php/domains", - "version": "2.0.0", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/utopia-php/domains.git", - "reference": "7f76390998359ef67fcea168f614cbd63a4001e8" + "reference": "1b1fea8674e8712e0344d3abb5a7acd558dede50" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/domains/zipball/7f76390998359ef67fcea168f614cbd63a4001e8", - "reference": "7f76390998359ef67fcea168f614cbd63a4001e8", + "url": "https://api.github.com/repos/utopia-php/domains/zipball/1b1fea8674e8712e0344d3abb5a7acd558dede50", + "reference": "1b1fea8674e8712e0344d3abb5a7acd558dede50", "shasum": "" }, "require": { - "php": ">=8.2", - "utopia-php/cache": "^2.0", + "php": ">=8.3", + "utopia-php/cache": "^3.0", "utopia-php/validators": "0.*" }, "require-dev": { @@ -4192,9 +4192,9 @@ ], "support": { "issues": "https://github.com/utopia-php/domains/issues", - "source": "https://github.com/utopia-php/domains/tree/2.0.0" + "source": "https://github.com/utopia-php/domains/tree/2.1.0" }, - "time": "2026-05-12T12:52:53+00:00" + "time": "2026-05-14T14:33:46+00:00" }, { "name": "utopia-php/dsn", @@ -5400,22 +5400,22 @@ }, { "name": "utopia-php/vcs", - "version": "4.1.0", + "version": "4.2.0", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "2850dbe975ee69b9466ee6df385fe1679394ce78" + "reference": "49d7751f0ae94634b00057177d9823928f6777c6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/2850dbe975ee69b9466ee6df385fe1679394ce78", - "reference": "2850dbe975ee69b9466ee6df385fe1679394ce78", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/49d7751f0ae94634b00057177d9823928f6777c6", + "reference": "49d7751f0ae94634b00057177d9823928f6777c6", "shasum": "" }, "require": { "adhocore/jwt": "^1.1", "php": ">=8.2", - "utopia-php/cache": "^2.0", + "utopia-php/cache": "^3.0", "utopia-php/fetch": "^1.1" }, "require-dev": { @@ -5443,9 +5443,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/4.1.0" + "source": "https://github.com/utopia-php/vcs/tree/4.2.0" }, - "time": "2026-05-14T10:04:10+00:00" + "time": "2026-05-17T15:58:27+00:00" }, { "name": "utopia-php/websocket",