feat: support out-of-order chunked uploads

- Add APP_LIMIT_UPLOAD_CHUNK_SIZE constant (5MB) matching official SDKs
- Replace dynamic chunk calculation with fixed 5MB chunk math in all upload endpoints
- Remove -1 last-chunk sentinel that broke when last chunk arrived first
- Fix duplicate-retry guards: return existing resource instead of erroring for chunked uploads
- Add out-of-order e2e tests for Storage, Functions, and Sites
- Upgrade utopia-php/storage to 2.0.0 for device-level out-of-order assembly support
This commit is contained in:
Torsten Dittmann
2026-04-27 17:15:00 +04:00
parent 1b8123bf62
commit 49d2db65e6
9 changed files with 510 additions and 145 deletions
+1
View File
@@ -244,6 +244,7 @@ const APP_AUTH_TYPE_KEY = 'Key';
const APP_AUTH_TYPE_ADMIN = 'Admin';
// Response related
const MAX_OUTPUT_CHUNK_SIZE = 10 * 1024 * 1024; // 10MB
const APP_LIMIT_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024; // 5MB
const APP_FUNCTION_LOG_LENGTH_LIMIT = 1000000;
const APP_FUNCTION_ERROR_LENGTH_LIMIT = 1000000;
// Function headers
+1 -1
View File
@@ -81,7 +81,7 @@
"utopia-php/queue": "0.17.*",
"utopia-php/servers": "0.3.*",
"utopia-php/registry": "0.5.*",
"utopia-php/storage": "1.0.*",
"utopia-php/storage": "2.*",
"utopia-php/system": "0.10.*",
"utopia-php/telemetry": "0.2.*",
"utopia-php/vcs": "3.*",
Generated
+91 -90
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "c5ae97637fd0ec0a950044d1c33677ea",
"content-hash": "ba332fbec7c2e7d462ee5bb3fad9775c",
"packages": [
{
"name": "adhocore/jwt",
@@ -1996,16 +1996,16 @@
},
{
"name": "phpseclib/phpseclib",
"version": "3.0.51",
"version": "3.0.52",
"source": {
"type": "git",
"url": "https://github.com/phpseclib/phpseclib.git",
"reference": "d59c94077f9c9915abb51ddb52ce85188ece1748"
"reference": "2adaefc83df2ec548558307690f376dd7d4f4fce"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/phpseclib/phpseclib/zipball/d59c94077f9c9915abb51ddb52ce85188ece1748",
"reference": "d59c94077f9c9915abb51ddb52ce85188ece1748",
"url": "https://api.github.com/repos/phpseclib/phpseclib/zipball/2adaefc83df2ec548558307690f376dd7d4f4fce",
"reference": "2adaefc83df2ec548558307690f376dd7d4f4fce",
"shasum": ""
},
"require": {
@@ -2086,7 +2086,7 @@
],
"support": {
"issues": "https://github.com/phpseclib/phpseclib/issues",
"source": "https://github.com/phpseclib/phpseclib/tree/3.0.51"
"source": "https://github.com/phpseclib/phpseclib/tree/3.0.52"
},
"funding": [
{
@@ -2102,7 +2102,7 @@
"type": "tidelift"
}
],
"time": "2026-04-10T01:33:53+00:00"
"time": "2026-04-27T07:02:15+00:00"
},
{
"name": "psr/clock",
@@ -2887,7 +2887,7 @@
},
{
"name": "symfony/polyfill-mbstring",
"version": "v1.36.0",
"version": "v1.37.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-mbstring.git",
@@ -2948,7 +2948,7 @@
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-mbstring/tree/v1.36.0"
"source": "https://github.com/symfony/polyfill-mbstring/tree/v1.37.0"
},
"funding": [
{
@@ -2972,7 +2972,7 @@
},
{
"name": "symfony/polyfill-php82",
"version": "v1.36.0",
"version": "v1.37.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-php82.git",
@@ -3028,7 +3028,7 @@
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-php82/tree/v1.36.0"
"source": "https://github.com/symfony/polyfill-php82/tree/v1.37.0"
},
"funding": [
{
@@ -3052,7 +3052,7 @@
},
{
"name": "symfony/polyfill-php83",
"version": "v1.36.0",
"version": "v1.37.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-php83.git",
@@ -3108,7 +3108,7 @@
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-php83/tree/v1.36.0"
"source": "https://github.com/symfony/polyfill-php83/tree/v1.37.0"
},
"funding": [
{
@@ -3132,16 +3132,16 @@
},
{
"name": "symfony/polyfill-php85",
"version": "v1.36.0",
"version": "v1.37.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-php85.git",
"reference": "2c408a6bb0313e6001a83628dc5506100474254e"
"reference": "fcfa4973a9917cef23f2e38774da74a2b7d115ee"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/2c408a6bb0313e6001a83628dc5506100474254e",
"reference": "2c408a6bb0313e6001a83628dc5506100474254e",
"url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/fcfa4973a9917cef23f2e38774da74a2b7d115ee",
"reference": "fcfa4973a9917cef23f2e38774da74a2b7d115ee",
"shasum": ""
},
"require": {
@@ -3188,7 +3188,7 @@
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-php85/tree/v1.36.0"
"source": "https://github.com/symfony/polyfill-php85/tree/v1.37.0"
},
"funding": [
{
@@ -3208,7 +3208,7 @@
"type": "tidelift"
}
],
"time": "2026-04-10T16:50:15+00:00"
"time": "2026-04-26T13:10:57+00:00"
},
{
"name": "symfony/service-contracts",
@@ -3658,16 +3658,16 @@
},
{
"name": "utopia-php/cli",
"version": "0.23.1",
"version": "0.23.2",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/cli.git",
"reference": "8d1955b8bc4dc631f45d7c7df689ed7b63f70621"
"reference": "145b91fef827853bcceaa3ab8ca2b1d6faaca2ab"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/cli/zipball/8d1955b8bc4dc631f45d7c7df689ed7b63f70621",
"reference": "8d1955b8bc4dc631f45d7c7df689ed7b63f70621",
"url": "https://api.github.com/repos/utopia-php/cli/zipball/145b91fef827853bcceaa3ab8ca2b1d6faaca2ab",
"reference": "145b91fef827853bcceaa3ab8ca2b1d6faaca2ab",
"shasum": ""
},
"require": {
@@ -3703,9 +3703,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/cli/issues",
"source": "https://github.com/utopia-php/cli/tree/0.23.1"
"source": "https://github.com/utopia-php/cli/tree/0.23.2"
},
"time": "2026-04-05T15:27:35+00:00"
"time": "2026-04-27T09:19:04+00:00"
},
{
"name": "utopia-php/compression",
@@ -4271,21 +4271,20 @@
},
{
"name": "utopia-php/http",
"version": "0.34.21",
"version": "0.34.24",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/http.git",
"reference": "49a6bd3ea0d2966aa19cf707255d442675288a24"
"reference": "d1eced0627c5a9fceddf53992ed97d664b810d33"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/http/zipball/49a6bd3ea0d2966aa19cf707255d442675288a24",
"reference": "49a6bd3ea0d2966aa19cf707255d442675288a24",
"url": "https://api.github.com/repos/utopia-php/http/zipball/d1eced0627c5a9fceddf53992ed97d664b810d33",
"reference": "d1eced0627c5a9fceddf53992ed97d664b810d33",
"shasum": ""
},
"require": {
"ext-swoole": "*",
"php": ">=8.2",
"php": ">=8.3",
"utopia-php/compression": "0.1.*",
"utopia-php/di": "0.3.*",
"utopia-php/servers": "0.3.*",
@@ -4295,11 +4294,14 @@
"require-dev": {
"doctrine/instantiator": "^1.5",
"laravel/pint": "1.*",
"phpbench/phpbench": "^1.2",
"phpstan/phpstan": "1.*",
"phpunit/phpunit": "^9.5.25",
"phpstan/phpstan": "^2.1",
"phpunit/phpunit": "^12.0",
"rector/rector": "^2.4",
"swoole/ide-helper": "4.8.3"
},
"suggest": {
"ext-swoole": "Required to use the Swoole server adapter (\\Utopia\\Http\\Adapter\\Swoole\\Server)."
},
"type": "library",
"autoload": {
"psr-4": {
@@ -4319,9 +4321,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/http/issues",
"source": "https://github.com/utopia-php/http/tree/0.34.21"
"source": "https://github.com/utopia-php/http/tree/0.34.24"
},
"time": "2026-04-19T19:44:04+00:00"
"time": "2026-04-24T12:16:53+00:00"
},
{
"name": "utopia-php/image",
@@ -4528,16 +4530,16 @@
},
{
"name": "utopia-php/migration",
"version": "1.9.1",
"version": "1.9.4",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/migration.git",
"reference": "7a86aeadf182b63a9f4ceba7e137588b31c5d2e2"
"reference": "969dc9477ea962f16da9254facdbd8944cf13477"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/7a86aeadf182b63a9f4ceba7e137588b31c5d2e2",
"reference": "7a86aeadf182b63a9f4ceba7e137588b31c5d2e2",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/969dc9477ea962f16da9254facdbd8944cf13477",
"reference": "969dc9477ea962f16da9254facdbd8944cf13477",
"shasum": ""
},
"require": {
@@ -4548,7 +4550,7 @@
"php": ">=8.1",
"utopia-php/database": "5.*",
"utopia-php/dsn": "0.2.*",
"utopia-php/storage": "1.0.*"
"utopia-php/storage": "2.*"
},
"require-dev": {
"ext-pdo": "*",
@@ -4577,22 +4579,22 @@
],
"support": {
"issues": "https://github.com/utopia-php/migration/issues",
"source": "https://github.com/utopia-php/migration/tree/1.9.1"
"source": "https://github.com/utopia-php/migration/tree/1.9.4"
},
"time": "2026-03-25T07:05:27+00:00"
"time": "2026-04-27T12:42:51+00:00"
},
{
"name": "utopia-php/mongo",
"version": "1.0.2",
"version": "1.1.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/mongo.git",
"reference": "677a21c53f7a1316c528b4b45b3fce886cee7223"
"reference": "73593682deee4696525a04e26524c1c1226e1530"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/mongo/zipball/677a21c53f7a1316c528b4b45b3fce886cee7223",
"reference": "677a21c53f7a1316c528b4b45b3fce886cee7223",
"url": "https://api.github.com/repos/utopia-php/mongo/zipball/73593682deee4696525a04e26524c1c1226e1530",
"reference": "73593682deee4696525a04e26524c1c1226e1530",
"shasum": ""
},
"require": {
@@ -4638,9 +4640,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/mongo/issues",
"source": "https://github.com/utopia-php/mongo/tree/1.0.2"
"source": "https://github.com/utopia-php/mongo/tree/1.1.0"
},
"time": "2026-03-18T02:45:50+00:00"
"time": "2026-04-24T06:15:10+00:00"
},
{
"name": "utopia-php/platform",
@@ -5018,16 +5020,16 @@
},
{
"name": "utopia-php/storage",
"version": "1.0.1",
"version": "2.0.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/storage.git",
"reference": "f014be445f0baa635d0764e1673196f412511618"
"reference": "52d1f89a47165ef0d3deff63043cda182175adfb"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/f014be445f0baa635d0764e1673196f412511618",
"reference": "f014be445f0baa635d0764e1673196f412511618",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/52d1f89a47165ef0d3deff63043cda182175adfb",
"reference": "52d1f89a47165ef0d3deff63043cda182175adfb",
"shasum": ""
},
"require": {
@@ -5041,9 +5043,8 @@
"utopia-php/validators": "0.2.*"
},
"require-dev": {
"laravel/pint": "1.2.*",
"phpunit/phpunit": "^9.3",
"vimeo/psalm": "4.0.1"
"laravel/pint": "^1.21",
"phpunit/phpunit": "^9.3"
},
"type": "library",
"autoload": {
@@ -5065,9 +5066,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/storage/issues",
"source": "https://github.com/utopia-php/storage/tree/1.0.1"
"source": "https://github.com/utopia-php/storage/tree/2.0.0"
},
"time": "2026-02-23T05:59:32+00:00"
"time": "2026-04-27T11:39:32+00:00"
},
{
"name": "utopia-php/system",
@@ -5464,16 +5465,16 @@
"packages-dev": [
{
"name": "appwrite/sdk-generator",
"version": "1.20",
"version": "1.24.0",
"source": {
"type": "git",
"url": "https://github.com/appwrite/sdk-generator.git",
"reference": "525f0630520c95100fcdfb63c9dac859c1d02588"
"reference": "6d4d26659bc7a1c347c1d4d8dae3b77b5562e0cb"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/525f0630520c95100fcdfb63c9dac859c1d02588",
"reference": "525f0630520c95100fcdfb63c9dac859c1d02588",
"url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/6d4d26659bc7a1c347c1d4d8dae3b77b5562e0cb",
"reference": "6d4d26659bc7a1c347c1d4d8dae3b77b5562e0cb",
"shasum": ""
},
"require": {
@@ -5509,9 +5510,9 @@
"description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms",
"support": {
"issues": "https://github.com/appwrite/sdk-generator/issues",
"source": "https://github.com/appwrite/sdk-generator/tree/1.20"
"source": "https://github.com/appwrite/sdk-generator/tree/1.24.0"
},
"time": "2026-04-20T05:45:00+00:00"
"time": "2026-04-24T12:50:05+00:00"
},
{
"name": "brianium/paratest",
@@ -5793,16 +5794,16 @@
},
{
"name": "laravel/pint",
"version": "v1.29.0",
"version": "v1.29.1",
"source": {
"type": "git",
"url": "https://github.com/laravel/pint.git",
"reference": "bdec963f53172c5e36330f3a400604c69bf02d39"
"reference": "0770e9b7fafd50d4586881d456d6eb41c9247a80"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/laravel/pint/zipball/bdec963f53172c5e36330f3a400604c69bf02d39",
"reference": "bdec963f53172c5e36330f3a400604c69bf02d39",
"url": "https://api.github.com/repos/laravel/pint/zipball/0770e9b7fafd50d4586881d456d6eb41c9247a80",
"reference": "0770e9b7fafd50d4586881d456d6eb41c9247a80",
"shasum": ""
},
"require": {
@@ -5813,14 +5814,14 @@
"php": "^8.2.0"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "^3.94.2",
"illuminate/view": "^12.54.1",
"larastan/larastan": "^3.9.3",
"laravel-zero/framework": "^12.0.5",
"friendsofphp/php-cs-fixer": "^3.95.1",
"illuminate/view": "^12.56.0",
"larastan/larastan": "^3.9.6",
"laravel-zero/framework": "^12.1.0",
"mockery/mockery": "^1.6.12",
"nunomaduro/termwind": "^2.4.0",
"pestphp/pest": "^3.8.6",
"shipfastlabs/agent-detector": "^1.1.0"
"shipfastlabs/agent-detector": "^1.1.3"
},
"bin": [
"builds/pint"
@@ -5857,7 +5858,7 @@
"issues": "https://github.com/laravel/pint/issues",
"source": "https://github.com/laravel/pint"
},
"time": "2026-03-12T15:51:39+00:00"
"time": "2026-04-20T15:26:14+00:00"
},
{
"name": "matthiasmullie/minify",
@@ -6220,11 +6221,11 @@
},
{
"name": "phpstan/phpstan",
"version": "2.1.50",
"version": "2.1.51",
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/phpstan/phpstan/zipball/d452086fb4cf648c6b2d8cf3b639351f79e4f3e2",
"reference": "d452086fb4cf648c6b2d8cf3b639351f79e4f3e2",
"url": "https://api.github.com/repos/phpstan/phpstan/zipball/dc3b523c45e714c70de2ac5113b958223b55dc59",
"reference": "dc3b523c45e714c70de2ac5113b958223b55dc59",
"shasum": ""
},
"require": {
@@ -6269,7 +6270,7 @@
"type": "github"
}
],
"time": "2026-04-17T13:10:32+00:00"
"time": "2026-04-21T18:22:01+00:00"
},
{
"name": "phpunit/php-code-coverage",
@@ -7779,7 +7780,7 @@
},
{
"name": "symfony/polyfill-ctype",
"version": "v1.36.0",
"version": "v1.37.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-ctype.git",
@@ -7838,7 +7839,7 @@
"portable"
],
"support": {
"source": "https://github.com/symfony/polyfill-ctype/tree/v1.36.0"
"source": "https://github.com/symfony/polyfill-ctype/tree/v1.37.0"
},
"funding": [
{
@@ -7862,16 +7863,16 @@
},
{
"name": "symfony/polyfill-intl-grapheme",
"version": "v1.36.0",
"version": "v1.37.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-intl-grapheme.git",
"reference": "ad1b7b9092976d6c948b8a187cec9faaea9ec1df"
"reference": "4864388bfbd3001ce88e234fab652acd91fdc57e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/ad1b7b9092976d6c948b8a187cec9faaea9ec1df",
"reference": "ad1b7b9092976d6c948b8a187cec9faaea9ec1df",
"url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/4864388bfbd3001ce88e234fab652acd91fdc57e",
"reference": "4864388bfbd3001ce88e234fab652acd91fdc57e",
"shasum": ""
},
"require": {
@@ -7920,7 +7921,7 @@
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.36.0"
"source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.37.0"
},
"funding": [
{
@@ -7940,11 +7941,11 @@
"type": "tidelift"
}
],
"time": "2026-04-10T16:19:22+00:00"
"time": "2026-04-26T13:13:48+00:00"
},
{
"name": "symfony/polyfill-intl-normalizer",
"version": "v1.36.0",
"version": "v1.37.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-intl-normalizer.git",
@@ -8005,7 +8006,7 @@
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.36.0"
"source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.37.0"
},
"funding": [
{
@@ -8029,7 +8030,7 @@
},
{
"name": "symfony/polyfill-php81",
"version": "v1.36.0",
"version": "v1.37.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-php81.git",
@@ -8085,7 +8086,7 @@
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-php81/tree/v1.36.0"
"source": "https://github.com/symfony/polyfill-php81/tree/v1.37.0"
},
"funding": [
{
@@ -175,15 +175,8 @@ class Create extends Action
throw new Exception(Exception::STORAGE_INVALID_CONTENT_RANGE);
}
// TODO remove the condition that checks `$end === $fileSize` in next breaking version
if ($end === $fileSize - 1 || $end === $fileSize) {
//if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk
$chunks = $chunk = -1;
} else {
// Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart)
$chunks = (int) ceil($fileSize / ($end + 1 - $start));
$chunk = (int) ($start / ($end + 1 - $start)) + 1;
}
$chunks = (int) ceil($fileSize / APP_LIMIT_UPLOAD_CHUNK_SIZE);
$chunk = (int) ($start / APP_LIMIT_UPLOAD_CHUNK_SIZE) + 1;
}
if (!$fileSizeValidator->isValid($fileSize) && $functionSizeLimit !== 0) { // Check if file size is exceeding allowed limit
@@ -202,15 +195,14 @@ class Create extends Action
$metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)];
if (!$deployment->isEmpty()) {
$chunks = $deployment->getAttribute('sourceChunksTotal', 1);
$uploaded = $deployment->getAttribute('sourceChunksUploaded', 0);
$metadata = $deployment->getAttribute('sourceMetadata', []);
if ($chunk === -1) {
$chunk = $chunks;
}
} else {
// Guard against manually setting range header for single chunk upload
if ($chunks === -1) {
$chunks = 1;
$chunk = 1;
if ($uploaded === $chunks) {
$response
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
return;
}
}
@@ -177,15 +177,8 @@ class Create extends Action
throw new Exception(Exception::STORAGE_INVALID_CONTENT_RANGE);
}
// TODO remove the condition that checks `$end === $fileSize` in next breaking version
if ($end === $fileSize - 1 || $end === $fileSize) {
//if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk
$chunks = $chunk = -1;
} else {
// Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart)
$chunks = (int) ceil($fileSize / ($end + 1 - $start));
$chunk = (int) ($start / ($end + 1 - $start)) + 1;
}
$chunks = (int) ceil($fileSize / APP_LIMIT_UPLOAD_CHUNK_SIZE);
$chunk = (int) ($start / APP_LIMIT_UPLOAD_CHUNK_SIZE) + 1;
}
if (!$fileSizeValidator->isValid($fileSize) && $siteSizeLimit !== 0) { // Check if file size is exceeding allowed limit
@@ -204,15 +197,14 @@ class Create extends Action
$metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)];
if (!$deployment->isEmpty()) {
$chunks = $deployment->getAttribute('sourceChunksTotal', 1);
$uploaded = $deployment->getAttribute('sourceChunksUploaded', 0);
$metadata = $deployment->getAttribute('sourceMetadata', []);
if ($chunk === -1) {
$chunk = $chunks;
}
} else {
// Guard against manually setting range header for single chunk upload
if ($chunks === -1) {
$chunks = 1;
$chunk = 1;
if ($uploaded === $chunks) {
$response
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
return;
}
}
@@ -204,15 +204,8 @@ class Create extends Action
throw new Exception(Exception::STORAGE_INVALID_APPWRITE_ID);
}
// TODO remove the condition that checks `$end === $fileSize` in next breaking version
if ($end === $fileSize - 1 || $end === $fileSize) {
//if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to -1 notify it's last chunk
$chunks = $chunk = -1;
} else {
// Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart)
$chunks = (int) ceil($fileSize / ($end + 1 - $start));
$chunk = (int) ($start / ($end + 1 - $start)) + 1;
}
$chunks = (int) ceil($fileSize / APP_LIMIT_UPLOAD_CHUNK_SIZE);
$chunk = (int) ($start / APP_LIMIT_UPLOAD_CHUNK_SIZE) + 1;
}
/**
@@ -249,18 +242,15 @@ class Create extends Action
$uploaded = $file->getAttribute('chunksUploaded', 0);
$metadata = $file->getAttribute('metadata', []);
if ($chunk === -1) {
$chunk = $chunks;
}
if ($uploaded === $chunks) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
}
} else {
// Guard against manually setting range header for single chunk upload
if ($chunks === -1) {
$chunks = 1;
$chunk = 1;
if (empty($contentRange)) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
}
$response
->setStatusCode(Response::STATUS_CODE_OK)
->dynamic($file, Response::MODEL_FILE);
return;
}
}
@@ -1079,6 +1079,118 @@ class FunctionsCustomServerTest extends Scope
}, 120000, 500);
}
public function testCreateDeploymentOutOfOrder(): void
{
$data = $this->setupTestFunction();
$functionId = $data['functionId'];
// Prepare a code file that spans at least 3 chunks
$folder = 'large';
$code = realpath(__DIR__ . '/../../../resources/functions') . "/$folder/code.tar.gz";
Console::execute('cd ' . realpath(__DIR__ . "/../../../resources/functions") . "/$folder && tar --exclude code.tar.gz --exclude node_modules -czf code.tar.gz .", '', $this->stdout, $this->stderr);
$totalSize = filesize($code);
$chunkSize = 5 * 1024 * 1024; // 5MB chunks
$mimeType = 'application/x-gzip';
$chunksTotal = (int) ceil($totalSize / $chunkSize);
// Read all chunks into memory
$handle = fopen($code, "rb");
$this->assertNotFalse($handle, "Could not open test resource: $code");
$chunks = [];
for ($i = 0; $i < $chunksTotal; $i++) {
$start = $i * $chunkSize;
$end = min($start + $chunkSize, $totalSize);
$length = $end - $start;
$data = fread($handle, $length);
$chunks[] = [
'data' => $data,
'start' => $start,
'end' => $end - 1,
'index' => $i,
];
}
fclose($handle);
// We need at least 3 chunks for a meaningful out-of-order test
$this->assertGreaterThanOrEqual(3, count($chunks), 'Test file must span at least 3 chunks');
// Upload chunks in out-of-order sequence: last chunk first, then first, then second
$uploadOrder = [count($chunks) - 1, 0, 1];
$deploymentId = '';
$deployment = null;
foreach ($uploadOrder as $chunkIndex) {
$chunk = $chunks[$chunkIndex];
$curlFile = new \CURLFile(
'data://' . $mimeType . ';base64,' . base64_encode($chunk['data']),
$mimeType,
'large-fx.tar.gz'
);
$headers = [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
'content-range' => 'bytes ' . $chunk['start'] . '-' . $chunk['end'] . '/' . $totalSize,
];
if (!empty($deploymentId)) {
$headers['x-appwrite-id'] = $deploymentId;
}
$deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge($headers, $this->getHeaders()), [
'entrypoint' => 'index.js',
'code' => $curlFile,
'activate' => true,
]);
$this->assertEquals(202, $deployment['headers']['status-code']);
$deploymentId = $deployment['body']['$id'];
}
// Upload remaining chunks in any order to complete the file
$remainingChunks = [];
for ($i = 2; $i < count($chunks) - 1; $i++) {
$remainingChunks[] = $i;
}
shuffle($remainingChunks);
foreach ($remainingChunks as $chunkIndex) {
$chunk = $chunks[$chunkIndex];
$curlFile = new \CURLFile(
'data://' . $mimeType . ';base64,' . base64_encode($chunk['data']),
$mimeType,
'large-fx.tar.gz'
);
$headers = [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
'content-range' => 'bytes ' . $chunk['start'] . '-' . $chunk['end'] . '/' . $totalSize,
'x-appwrite-id' => $deploymentId,
];
$deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge($headers, $this->getHeaders()), [
'entrypoint' => 'index.js',
'code' => $curlFile,
'activate' => true,
]);
$this->assertEquals(202, $deployment['headers']['status-code']);
}
// Verify the final upload response indicates completion
$this->assertEquals($chunksTotal, $deployment['body']['sourceChunksTotal']);
$this->assertEquals($chunksTotal, $deployment['body']['sourceChunksUploaded']);
// Wait for build to complete
$this->assertEventually(function () use ($functionId, $deploymentId) {
$deployment = $this->getDeployment($functionId, $deploymentId);
$this->assertEquals(200, $deployment['headers']['status-code']);
$this->assertEquals('ready', $deployment['body']['status']);
}, 120000, 500);
}
public function testUpdateDeployment(): void
{
$data = $this->setupTestDeployment();
@@ -906,6 +906,136 @@ class SitesCustomServerTest extends Scope
$this->cleanupSite($siteId);
}
public function testCreateDeploymentOutOfOrder(): void
{
$siteId = $this->setupSite([
'buildRuntime' => 'node-22',
'fallbackFile' => '',
'framework' => 'other',
'name' => 'Test Site Out of Order Upload',
'outputDirectory' => './',
'providerBranch' => 'main',
'providerRootDirectory' => './',
'siteId' => ID::unique()
]);
// Create a temporary large site package for chunked upload
$tempDir = sys_get_temp_dir() . '/appwrite-test-site-' . uniqid();
mkdir($tempDir, 0777, true);
file_put_contents($tempDir . '/index.html', '<html><body>Hello World</body></html>');
// Add a large dummy file to make the package span multiple chunks
file_put_contents($tempDir . '/large.bin', str_repeat('X', 12 * 1024 * 1024)); // 12MB
$codePath = $tempDir . '/code.tar.gz';
Console::execute("cd $tempDir && tar --exclude code.tar.gz -czf code.tar.gz .", '', $this->stdout, $this->stderr);
$totalSize = filesize($codePath);
$chunkSize = 5 * 1024 * 1024; // 5MB chunks
$mimeType = 'application/x-gzip';
$chunksTotal = (int) ceil($totalSize / $chunkSize);
$this->assertGreaterThanOrEqual(3, $chunksTotal, 'Test file must span at least 3 chunks');
// Read all chunks into memory
$handle = fopen($codePath, "rb");
$this->assertNotFalse($handle, "Could not open test resource: $codePath");
$chunks = [];
for ($i = 0; $i < $chunksTotal; $i++) {
$start = $i * $chunkSize;
$end = min($start + $chunkSize, $totalSize);
$length = $end - $start;
$data = fread($handle, $length);
$chunks[] = [
'data' => $data,
'start' => $start,
'end' => $end - 1,
'index' => $i,
];
}
fclose($handle);
// Upload chunks in out-of-order sequence: last chunk first, then first, then second
$uploadOrder = [count($chunks) - 1, 0, 1];
$deploymentId = '';
$deployment = null;
foreach ($uploadOrder as $chunkIndex) {
$chunk = $chunks[$chunkIndex];
$curlFile = new \CURLFile(
'data://' . $mimeType . ';base64,' . base64_encode($chunk['data']),
$mimeType,
'code.tar.gz'
);
$headers = [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
'content-range' => 'bytes ' . $chunk['start'] . '-' . $chunk['end'] . '/' . $totalSize,
];
if (!empty($deploymentId)) {
$headers['x-appwrite-id'] = $deploymentId;
}
$deployment = $this->client->call(Client::METHOD_POST, '/sites/' . $siteId . '/deployments', array_merge($headers, $this->getHeaders()), [
'code' => $curlFile,
'activate' => true,
]);
$this->assertEquals(202, $deployment['headers']['status-code']);
$deploymentId = $deployment['body']['$id'];
}
// Upload remaining chunks in any order to complete the file
$remainingChunks = [];
for ($i = 2; $i < count($chunks) - 1; $i++) {
$remainingChunks[] = $i;
}
shuffle($remainingChunks);
foreach ($remainingChunks as $chunkIndex) {
$chunk = $chunks[$chunkIndex];
$curlFile = new \CURLFile(
'data://' . $mimeType . ';base64,' . base64_encode($chunk['data']),
$mimeType,
'code.tar.gz'
);
$headers = [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
'content-range' => 'bytes ' . $chunk['start'] . '-' . $chunk['end'] . '/' . $totalSize,
'x-appwrite-id' => $deploymentId,
];
$deployment = $this->client->call(Client::METHOD_POST, '/sites/' . $siteId . '/deployments', array_merge($headers, $this->getHeaders()), [
'code' => $curlFile,
'activate' => true,
]);
$this->assertEquals(202, $deployment['headers']['status-code']);
}
// Verify the final upload response indicates completion
$this->assertEquals($chunksTotal, $deployment['body']['sourceChunksTotal']);
$this->assertEquals($chunksTotal, $deployment['body']['sourceChunksUploaded']);
// Wait for build to complete
$this->assertEventually(function () use ($siteId, $deploymentId) {
$deployment = $this->getDeployment($siteId, $deploymentId);
$this->assertEquals(200, $deployment['headers']['status-code']);
$this->assertEquals('ready', $deployment['body']['status']);
}, 120000, 500);
// Clean up temp files
unlink($codePath);
unlink($tempDir . '/index.html');
unlink($tempDir . '/large.bin');
rmdir($tempDir);
$this->cleanupSite($siteId);
}
public function testCreateDeployment()
{
$siteId = $this->setupSite([
+147
View File
@@ -1227,6 +1227,153 @@ trait StorageBase
$this->assertEquals(204, $deleteBucketResponse['headers']['status-code']);
}
public function testCreateBucketFileOutOfOrder(): void
{
// Create a bucket for this test
$bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'bucketId' => ID::unique(),
'name' => 'Test Bucket Out of Order Upload',
'fileSecurity' => true,
'permissions' => [
Permission::read(Role::any()),
Permission::create(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
]);
$this->assertEquals(201, $bucket['headers']['status-code']);
$bucketId = $bucket['body']['$id'];
// Prepare a file that spans at least 3 chunks
$source = __DIR__ . "/../../../resources/disk-a/large-file.mp4";
$totalSize = \filesize($source);
$chunkSize = 5 * 1024 * 1024; // 5MB chunks
$mimeType = mime_content_type($source);
$chunksTotal = (int) ceil($totalSize / $chunkSize);
// Read all chunks into memory
$handle = fopen($source, "rb");
$this->assertNotFalse($handle, "Could not open test resource: $source");
$chunks = [];
for ($i = 0; $i < $chunksTotal; $i++) {
$start = $i * $chunkSize;
$end = min($start + $chunkSize, $totalSize);
$length = $end - $start;
$data = fread($handle, $length);
$chunks[] = [
'data' => $data,
'start' => $start,
'end' => $end - 1,
'index' => $i,
];
}
fclose($handle);
// We need at least 3 chunks for a meaningful out-of-order test
$this->assertGreaterThanOrEqual(3, count($chunks), 'Test file must span at least 3 chunks');
// Upload chunks in out-of-order sequence: last chunk first, then first, then middle
$uploadOrder = [count($chunks) - 1, 0, 1]; // last, first, second (for 3+ chunks)
$fileId = ID::unique();
$id = '';
$uploadedFile = null;
foreach ($uploadOrder as $chunkIndex) {
$chunk = $chunks[$chunkIndex];
$curlFile = new \CURLFile(
'data://' . $mimeType . ';base64,' . base64_encode($chunk['data']),
$mimeType,
'large-file.mp4'
);
$headers = [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
'content-range' => 'bytes ' . $chunk['start'] . '-' . $chunk['end'] . '/' . $totalSize,
];
if (!empty($id)) {
$headers['x-appwrite-id'] = $id;
}
$uploadedFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge($headers, $this->getHeaders()), [
'fileId' => $fileId,
'file' => $curlFile,
'permissions' => [
Permission::read(Role::any()),
],
]);
$this->assertEquals(201, $uploadedFile['headers']['status-code']);
$id = $uploadedFile['body']['$id'];
}
// Upload remaining chunks in any order to complete the file
$remainingChunks = [];
for ($i = 2; $i < count($chunks) - 1; $i++) {
$remainingChunks[] = $i;
}
// Shuffle remaining chunks for extra randomness
shuffle($remainingChunks);
foreach ($remainingChunks as $chunkIndex) {
$chunk = $chunks[$chunkIndex];
$curlFile = new \CURLFile(
'data://' . $mimeType . ';base64,' . base64_encode($chunk['data']),
$mimeType,
'large-file.mp4'
);
$headers = [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
'content-range' => 'bytes ' . $chunk['start'] . '-' . $chunk['end'] . '/' . $totalSize,
'x-appwrite-id' => $id,
];
$uploadedFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge($headers, $this->getHeaders()), [
'fileId' => $fileId,
'file' => $curlFile,
'permissions' => [
Permission::read(Role::any()),
],
]);
$this->assertEquals(201, $uploadedFile['headers']['status-code']);
}
// Verify the final upload response indicates completion
$this->assertEquals($chunksTotal, $uploadedFile['body']['chunksTotal']);
$this->assertEquals($chunksTotal, $uploadedFile['body']['chunksUploaded']);
// Verify the file can be downloaded and matches the original
$download = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $id . '/download', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$this->assertEquals(200, $download['headers']['status-code']);
$this->assertEquals($totalSize, strlen($download['body']));
$this->assertEquals(md5_file($source), md5($download['body']));
// Clean up
$this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $id, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]);
}
public function testDeleteBucketFile(): void
{
// Create a fresh file just for deletion testing (not using cache since we delete it)