From 7edb7619b4a6cba583ad2efd00cf1bf7fce38713 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sun, 6 Sep 2026 18:29:04 +0100 Subject: [PATCH 1/3] fix(storage): lock chunk transfers through upload completion --- .../Storage/Http/Buckets/Files/Create.php | 146 +++++++++--------- tests/e2e/Services/Storage/StorageBase.php | 36 ++++- 2 files changed, 107 insertions(+), 75 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index f0b399b8fb..3dc3cd2727 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -265,71 +265,62 @@ class Create extends Action return $merged; }; - try { - $locks($lockKey, 600, function () use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $folder, $path, $permissions, $response, &$completed): void { - $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); - if (!$file->isEmpty()) { - $chunks = $file->getAttribute('chunksTotal', 1); - $uploaded = $file->getAttribute('chunksUploaded', 0); - $metadata = $file->getAttribute('metadata', []); + $prepareUpload = function () use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $folder, $path, $permissions, $response, &$completed): void { + $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + if (!$file->isEmpty()) { + $chunks = $file->getAttribute('chunksTotal', 1); + $uploaded = $file->getAttribute('chunksUploaded', 0); + $metadata = $file->getAttribute('metadata', []); - if ($uploaded === $chunks) { - if (empty($contentRange)) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } + if ($uploaded === $chunks) { + if (empty($contentRange)) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } - $response - ->setStatusCode(Response::STATUS_CODE_OK) - ->dynamic($file, Response::MODEL_FILE); + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->dynamic($file, Response::MODEL_FILE); - $completed = true; - return; + $completed = true; + + return; + } + } + + if ($file->isEmpty()) { + $deviceForFiles->prepare($path, $metadata['content_type'] ?? '', $chunks, $metadata); + + if (!empty($contentRange)) { + $doc = new Document([ + '$id' => ID::custom($fileId), + '$permissions' => $permissions, + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $fileName, + 'folder' => $folder, + 'path' => $path, + 'signature' => '', + 'mimeType' => '', + 'sizeOriginal' => $fileSize, + 'sizeActual' => 0, + 'algorithm' => '', + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => 0, + 'search' => implode(' ', [$fileId, $fileName]), + 'metadata' => $metadata, + ]); + + try { + $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); + } catch (DuplicateException) { + throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); + } catch (NotFoundException) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } } - - if ($file->isEmpty()) { - $deviceForFiles->prepare($path, $metadata['content_type'] ?? '', $chunks, $metadata); - - if (!empty($contentRange)) { - $doc = new Document([ - '$id' => ID::custom($fileId), - '$permissions' => $permissions, - 'bucketId' => $bucket->getId(), - 'bucketInternalId' => $bucket->getSequence(), - 'name' => $fileName, - 'folder' => $folder, - 'path' => $path, - 'signature' => '', - 'mimeType' => '', - 'sizeOriginal' => $fileSize, - 'sizeActual' => 0, - 'algorithm' => '', - 'comment' => '', - 'chunksTotal' => $chunks, - 'chunksUploaded' => 0, - 'search' => implode(' ', [$fileId, $fileName]), - 'metadata' => $metadata, - ]); - - try { - $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc); - } catch (DuplicateException) { - throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS); - } catch (NotFoundException) { - throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - } - } - } - }, timeout: 120.0); - } catch (LockContention) { - $response->addHeader('Retry-After', '5'); - throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'File upload is busy. Try again.'); - } - - if ($completed) { - $queueForEvents->reset(); - return; - } + } + }; $finalizeUpload = function (int $chunksUploaded) use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $mergeUploadMetadata, $folder, $path, $permissions, $queueForEvents, $response): void { $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); @@ -355,14 +346,11 @@ class Create extends Action } } - // Another chunk may have finalized the upload and removed its parts - // before upload() counted them. Check the completed document above first. if (empty($chunksUploaded)) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file'); } - // A local chunk file is visible before its write finishes. Only parts - // recorded after upload() returns are safe to include in finalization. + // Count distinct completed parts, including previously persisted chunks. $chunksUploaded = max($uploaded, isset($metadata['parts']) ? \count($metadata['parts']) : $chunksUploaded); if ($chunksUploaded === $chunks && $uploaded < $chunks) { @@ -525,16 +513,28 @@ class Create extends Action }; try { - $chunksUploaded = $deviceForFiles->upload( - $deviceForLocal->read($fileTmpName), - $path, - $metadata['content_type'] ?? '', - $chunk, - $chunks, - $metadata - ); + // upload() can finalize and remove chunk files itself. Keep preparation, + // transfer and document completion under the same per-file lock. + $locks($lockKey, 600, function () use ($prepareUpload, $finalizeUpload, &$completed, $queueForEvents, $deviceForFiles, $deviceForLocal, $fileTmpName, $path, $chunk, &$chunks, &$metadata): void { + $prepareUpload(); - $locks($lockKey, 600, fn () => $finalizeUpload($chunksUploaded), timeout: 120.0); + if ($completed) { + $queueForEvents->reset(); + + return; + } + + $chunksUploaded = $deviceForFiles->upload( + $deviceForLocal->read($fileTmpName), + $path, + $metadata['content_type'] ?? '', + $chunk, + $chunks, + $metadata + ); + + $finalizeUpload($chunksUploaded); + }, timeout: 120.0); } catch (LockContention) { $response->addHeader('Retry-After', '5'); throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'File upload is busy. Try again.'); diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index ea6390969c..3f39ce9c97 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -4,6 +4,7 @@ namespace Tests\E2E\Services\Storage; use Appwrite\Extend\Exception; use CURLFile; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\Group; use Tests\E2E\Client; use Utopia\Database\Helpers\ID; @@ -1829,8 +1830,18 @@ trait StorageBase ]); } - public function testCreateBucketFileParallelChunksLargeFile(): void + public static function parallelChunksProvider(): array { + return [ + 'distinct chunks' => [false], + 'duplicate chunks' => [true], + ]; + } + + #[DataProvider('parallelChunksProvider')] + public function testCreateBucketFileParallelChunksLargeFile(bool $duplicate): void + { + // Test for SUCCESS $totalSize = 20 * 1024 * 1024; $chunkSize = 5 * 1024 * 1024; $chunksTotal = (int) ceil($totalSize / $chunkSize); @@ -1867,8 +1878,9 @@ trait StorageBase $this->assertNotFalse($handle, 'Could not create test file'); $remaining = $totalSize; - $block = str_repeat(hash('sha256', $fileId, binary: true), 1024); while ($remaining > 0) { + // Distinct blocks expose reordered or duplicated chunks in the hash check. + $block = str_repeat(hash('sha256', $fileId . ':' . $remaining, binary: true), 1024); $bytes = substr($block, 0, min(strlen($block), $remaining)); fwrite($handle, $bytes); $remaining -= strlen($bytes); @@ -1900,6 +1912,10 @@ trait StorageBase } fclose($sourceHandle); + if ($duplicate) { + $requests = array_merge($requests, $requests); + } + $responses = []; $endpoint = parse_url($this->client->getEndpoint()); $scheme = $endpoint['scheme'] ?? 'http'; @@ -1958,6 +1974,7 @@ trait StorageBase ksort($responses); + $this->assertCount(count($requests), $responses); foreach ($responses as $response) { $this->assertSame('', $response['error']); $this->assertContains($response['statusCode'], [200, 201], (string) $response['body']); @@ -1973,6 +1990,21 @@ trait StorageBase $this->assertEquals($chunksTotal, $uploadedFile['body']['chunksTotal']); $this->assertEquals($chunksTotal, $uploadedFile['body']['chunksUploaded']); + // A late retry must return the completed file without writing or finalizing again. + $retry = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge( + $requests[0]['headers'], + ['content-type' => 'multipart/form-data'] + ), [ + 'fileId' => $fileId, + 'file' => new CURLFile($requests[0]['chunkPath'], 'application/octet-stream', 'large-parallel-upload.bin'), + 'permissions' => [Permission::read(Role::any()), Permission::delete(Role::any())], + ]); + + $this->assertEquals(200, $retry['headers']['status-code']); + $this->assertEquals($fileId, $retry['body']['$id']); + $this->assertEquals($chunksTotal, $retry['body']['chunksUploaded']); + $this->assertEquals($uploadedFile['body']['signature'], $retry['body']['signature']); + $download = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], From a86d9b2624795cf3bfa271243940f0090b07fd54 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sun, 6 Sep 2026 18:47:14 +0100 Subject: [PATCH 2/3] fix(storage): refresh upload lock lease around chunk transfer The per-file lock used a fixed 600 second lease measured from acquisition. A slow transfer could outlive it and let a second request finalize the same file concurrently. The lock resource now hands the Distributed lock to its callback so the upload refreshes the lease before the transfer and verifies it still holds the lock before recording completion, failing with the existing 429 instead of finalizing under a lapsed lease. --- app/init/resources.php | 8 ++++++- .../Storage/Http/Buckets/Files/Create.php | 21 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/app/init/resources.php b/app/init/resources.php index 4897062b00..1d2cd54a9d 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -306,7 +306,13 @@ $container->set('redis', function () { }); $container->set('locks', fn (Group $pools) => fn (string $key, int $ttl, callable $callback, float $timeout = 0.0): mixed => $pools->get('lock')->use( - fn (\Redis $redis) => (new Distributed($redis, $key, ttl: $ttl))->withLock($callback, timeout: $timeout) + function (\Redis $redis) use ($key, $ttl, $callback, $timeout): mixed { + // The callback receives the lock so long-running holders can refresh + // the lease and verify it is still theirs before committing work. + $lock = new Distributed($redis, $key, ttl: $ttl); + + return $lock->withLock(fn () => $callback($lock), timeout: $timeout); + } ), ['pools']); $container->set('timelimit', fn (\Redis $redis) => fn (string $key, int $limit, int $time) => new TimeLimitRedis($key, $limit, $time, $redis), ['redis']); diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index 3dc3cd2727..6a6fd76d3d 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -31,6 +31,7 @@ use Utopia\Database\Validator\Authorization\Input; use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; +use Utopia\Lock\Distributed; use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; @@ -47,6 +48,12 @@ class Create extends Action { use HTTP; + /** + * Lease for the per-file upload lock, in seconds. Refreshed before the + * transfer and before completion so a single chunk always gets the full window. + */ + private const LOCK_TTL = 600; + public static function getName() { return 'createFile'; @@ -515,7 +522,7 @@ class Create extends Action try { // upload() can finalize and remove chunk files itself. Keep preparation, // transfer and document completion under the same per-file lock. - $locks($lockKey, 600, function () use ($prepareUpload, $finalizeUpload, &$completed, $queueForEvents, $deviceForFiles, $deviceForLocal, $fileTmpName, $path, $chunk, &$chunks, &$metadata): void { + $locks($lockKey, self::LOCK_TTL, function (Distributed $lock) use ($prepareUpload, $finalizeUpload, &$completed, $queueForEvents, $deviceForFiles, $deviceForLocal, $fileTmpName, $path, $chunk, &$chunks, &$metadata): void { $prepareUpload(); if ($completed) { @@ -524,6 +531,12 @@ class Create extends Action return; } + // Restart the lease so the transfer gets the full window, + // regardless of how long preparation took. + if (!$lock->refresh()) { + throw new LockContention('Upload lease lost before transfer: ' . $lock->token()); + } + $chunksUploaded = $deviceForFiles->upload( $deviceForLocal->read($fileTmpName), $path, @@ -533,6 +546,12 @@ class Create extends Action $metadata ); + // Never record completion under a lapsed lease: another request + // may already own the file and be finalizing it. + if (!$lock->isHeld()) { + throw new LockContention('Upload lease lost after transfer: ' . $lock->token()); + } + $finalizeUpload($chunksUploaded); }, timeout: 120.0); } catch (LockContention) { From 46d50a49484732a4a16bb65490e8dcee4b3003e7 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sun, 6 Sep 2026 19:03:36 +0100 Subject: [PATCH 3/3] test(usage): wait for usage schema readiness --- .../Services/Usage/UsageCustomServerTest.php | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/e2e/Services/Usage/UsageCustomServerTest.php b/tests/e2e/Services/Usage/UsageCustomServerTest.php index 0f6568127c..db33b520d3 100644 --- a/tests/e2e/Services/Usage/UsageCustomServerTest.php +++ b/tests/e2e/Services/Usage/UsageCustomServerTest.php @@ -17,7 +17,7 @@ final class UsageCustomServerTest extends Scope public function testListEventsReturnsRequestedEmptySeries(): void { - $this->skipUnlessUsageStatsEnabled(); + $this->waitForUsageStats(); $response = $this->call('/usage/events', [ 'metrics' => ['test.unknown.event'], @@ -32,7 +32,7 @@ final class UsageCustomServerTest extends Scope public function testListGaugesReturnsEveryRequestedSeries(): void { - $this->skipUnlessUsageStatsEnabled(); + $this->waitForUsageStats(); $response = $this->call('/usage/gauges', [ 'metrics' => ['test.unknown.gauge', 'test.second.gauge'], @@ -51,7 +51,7 @@ final class UsageCustomServerTest extends Scope public function testFlatEventAggregateNormalizesExplicitEndTime(): void { - $this->skipUnlessUsageStatsEnabled(); + $this->waitForUsageStats(); $response = $this->call('/usage/events', [ 'metrics' => ['test.unknown.event'], @@ -64,7 +64,7 @@ final class UsageCustomServerTest extends Scope public function testUnknownMaxGaugeDoesNotFabricateZeroSeries(): void { - $this->skipUnlessUsageStatsEnabled(); + $this->waitForUsageStats(); $flat = $this->call('/usage/gauges', [ 'metrics' => ['test.unknown.max.gauge'], @@ -84,7 +84,7 @@ final class UsageCustomServerTest extends Scope public function testInvalidFilterAttributeIsRejected(): void { - $this->skipUnlessUsageStatsEnabled(); + $this->waitForUsageStats(); $response = $this->call('/usage/events', [ 'metrics' => ['network.requests'], @@ -98,7 +98,7 @@ final class UsageCustomServerTest extends Scope public function testStructuralFilterQueryIsRejected(): void { - $this->skipUnlessUsageStatsEnabled(); + $this->waitForUsageStats(); $response = $this->call('/usage/events', [ 'metrics' => ['network.requests'], @@ -146,11 +146,25 @@ final class UsageCustomServerTest extends Scope $this->assertSame('general_unauthorized_scope', $response['body']['type']); } - private function skipUnlessUsageStatsEnabled(): void + private function waitForUsageStats(): void { if (System::getEnv('_APP_USAGE_STATS', 'enabled') === 'disabled') { $this->markTestSkipped('Usage stats are disabled on this stack'); } + + $project = $this->getProject(); + $key = $this->getNewKey(['health.read']); + + // HTTP health can pass while the ClickHouse usage schema is still initializing. + $this->assertEventually(function () use ($project, $key) { + $response = $this->client->call(Client::METHOD_GET, '/health/usage', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $project['$id'], + 'x-appwrite-key' => $key, + ]); + + $this->assertSame(200, $response['headers']['status-code'], 'Usage storage must be ready before testing usage queries'); + }, 60_000, 500); } /** @param array $parameters */