Merge branch 'main' into fix/usage-e2e-schema-readiness

This commit is contained in:
Chirag Aggarwal authored and GitHub committed 2026-09-06 11:30:16 -07:00
commit a925e9ea06
3 files changed
+133 -76

No files matched your search

+7 -1
View File
@@ -306,7 +306,13 @@ $container->set('redis', function () {
});
$container->set('locks', fn (Group $pools) => fn (string $key, int $ttl, callable $callback, float $timeout = 0.0): mixed => $pools->get('lock')->use(
fn (\Redis $redis) => (new Distributed($redis, $key, ttl: $ttl))->withLock($callback, timeout: $timeout)
function (\Redis $redis) use ($key, $ttl, $callback, $timeout): mixed {
// The callback receives the lock so long-running holders can refresh
// the lease and verify it is still theirs before committing work.
$lock = new Distributed($redis, $key, ttl: $ttl);
return $lock->withLock(fn () => $callback($lock), timeout: $timeout);
}
), ['pools']);
$container->set('timelimit', fn (\Redis $redis) => fn (string $key, int $limit, int $time) => new TimeLimitRedis($key, $limit, $time, $redis), ['redis']);
@@ -31,6 +31,7 @@ use Utopia\Database\Validator\Authorization\Input;
use Utopia\Database\Validator\Permissions;
use Utopia\Database\Validator\UID;
use Utopia\Http\Adapter\Swoole\Request;
use Utopia\Lock\Distributed;
use Utopia\Lock\Exception\Contention as LockContention;
use Utopia\Platform\Action;
use Utopia\Platform\Scope\HTTP;
@@ -47,6 +48,12 @@ class Create extends Action
{
use HTTP;
/**
* Lease for the per-file upload lock, in seconds. Refreshed before the
* transfer and before completion so a single chunk always gets the full window.
*/
private const LOCK_TTL = 600;
public static function getName()
{
return 'createFile';
@@ -265,71 +272,62 @@ class Create extends Action
return $merged;
};
try {
$locks($lockKey, 600, function () use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $folder, $path, $permissions, $response, &$completed): void {
$file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId));
if (!$file->isEmpty()) {
$chunks = $file->getAttribute('chunksTotal', 1);
$uploaded = $file->getAttribute('chunksUploaded', 0);
$metadata = $file->getAttribute('metadata', []);
$prepareUpload = function () use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $folder, $path, $permissions, $response, &$completed): void {
$file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId));
if (!$file->isEmpty()) {
$chunks = $file->getAttribute('chunksTotal', 1);
$uploaded = $file->getAttribute('chunksUploaded', 0);
$metadata = $file->getAttribute('metadata', []);
if ($uploaded === $chunks) {
if (empty($contentRange)) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
}
if ($uploaded === $chunks) {
if (empty($contentRange)) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
}
$response
->setStatusCode(Response::STATUS_CODE_OK)
->dynamic($file, Response::MODEL_FILE);
$response
->setStatusCode(Response::STATUS_CODE_OK)
->dynamic($file, Response::MODEL_FILE);
$completed = true;
return;
$completed = true;
return;
}
}
if ($file->isEmpty()) {
$deviceForFiles->prepare($path, $metadata['content_type'] ?? '', $chunks, $metadata);
if (!empty($contentRange)) {
$doc = new Document([
'$id' => ID::custom($fileId),
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getSequence(),
'name' => $fileName,
'folder' => $folder,
'path' => $path,
'signature' => '',
'mimeType' => '',
'sizeOriginal' => $fileSize,
'sizeActual' => 0,
'algorithm' => '',
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => 0,
'search' => implode(' ', [$fileId, $fileName]),
'metadata' => $metadata,
]);
try {
$dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc);
} catch (DuplicateException) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
} catch (NotFoundException) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
}
if ($file->isEmpty()) {
$deviceForFiles->prepare($path, $metadata['content_type'] ?? '', $chunks, $metadata);
if (!empty($contentRange)) {
$doc = new Document([
'$id' => ID::custom($fileId),
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getSequence(),
'name' => $fileName,
'folder' => $folder,
'path' => $path,
'signature' => '',
'mimeType' => '',
'sizeOriginal' => $fileSize,
'sizeActual' => 0,
'algorithm' => '',
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => 0,
'search' => implode(' ', [$fileId, $fileName]),
'metadata' => $metadata,
]);
try {
$dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc);
} catch (DuplicateException) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
} catch (NotFoundException) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
}
}
}, timeout: 120.0);
} catch (LockContention) {
$response->addHeader('Retry-After', '5');
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'File upload is busy. Try again.');
}
if ($completed) {
$queueForEvents->reset();
return;
}
}
};
$finalizeUpload = function (int $chunksUploaded) use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $mergeUploadMetadata, $folder, $path, $permissions, $queueForEvents, $response): void {
$file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId));
@@ -355,14 +353,11 @@ class Create extends Action
}
}
// Another chunk may have finalized the upload and removed its parts
// before upload() counted them. Check the completed document above first.
if (empty($chunksUploaded)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file');
}
// A local chunk file is visible before its write finishes. Only parts
// recorded after upload() returns are safe to include in finalization.
// Count distinct completed parts, including previously persisted chunks.
$chunksUploaded = max($uploaded, isset($metadata['parts']) ? \count($metadata['parts']) : $chunksUploaded);
if ($chunksUploaded === $chunks && $uploaded < $chunks) {
@@ -525,16 +520,40 @@ class Create extends Action
};
try {
$chunksUploaded = $deviceForFiles->upload(
$deviceForLocal->read($fileTmpName),
$path,
$metadata['content_type'] ?? '',
$chunk,
$chunks,
$metadata
);
// upload() can finalize and remove chunk files itself. Keep preparation,
// transfer and document completion under the same per-file lock.
$locks($lockKey, self::LOCK_TTL, function (Distributed $lock) use ($prepareUpload, $finalizeUpload, &$completed, $queueForEvents, $deviceForFiles, $deviceForLocal, $fileTmpName, $path, $chunk, &$chunks, &$metadata): void {
$prepareUpload();
$locks($lockKey, 600, fn () => $finalizeUpload($chunksUploaded), timeout: 120.0);
if ($completed) {
$queueForEvents->reset();
return;
}
// Restart the lease so the transfer gets the full window,
// regardless of how long preparation took.
if (!$lock->refresh()) {
throw new LockContention('Upload lease lost before transfer: ' . $lock->token());
}
$chunksUploaded = $deviceForFiles->upload(
$deviceForLocal->read($fileTmpName),
$path,
$metadata['content_type'] ?? '',
$chunk,
$chunks,
$metadata
);
// Never record completion under a lapsed lease: another request
// may already own the file and be finalizing it.
if (!$lock->isHeld()) {
throw new LockContention('Upload lease lost after transfer: ' . $lock->token());
}
$finalizeUpload($chunksUploaded);
}, timeout: 120.0);
} catch (LockContention) {
$response->addHeader('Retry-After', '5');
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'File upload is busy. Try again.');
+34 -2
View File
@@ -4,6 +4,7 @@ namespace Tests\E2E\Services\Storage;
use Appwrite\Extend\Exception;
use CURLFile;
use PHPUnit\Framework\Attributes\DataProvider;
use PHPUnit\Framework\Attributes\Group;
use Tests\E2E\Client;
use Utopia\Database\Helpers\ID;
@@ -1829,8 +1830,18 @@ trait StorageBase
]);
}
public function testCreateBucketFileParallelChunksLargeFile(): void
public static function parallelChunksProvider(): array
{
return [
'distinct chunks' => [false],
'duplicate chunks' => [true],
];
}
#[DataProvider('parallelChunksProvider')]
public function testCreateBucketFileParallelChunksLargeFile(bool $duplicate): void
{
// Test for SUCCESS
$totalSize = 20 * 1024 * 1024;
$chunkSize = 5 * 1024 * 1024;
$chunksTotal = (int) ceil($totalSize / $chunkSize);
@@ -1867,8 +1878,9 @@ trait StorageBase
$this->assertNotFalse($handle, 'Could not create test file');
$remaining = $totalSize;
$block = str_repeat(hash('sha256', $fileId, binary: true), 1024);
while ($remaining > 0) {
// Distinct blocks expose reordered or duplicated chunks in the hash check.
$block = str_repeat(hash('sha256', $fileId . ':' . $remaining, binary: true), 1024);
$bytes = substr($block, 0, min(strlen($block), $remaining));
fwrite($handle, $bytes);
$remaining -= strlen($bytes);
@@ -1900,6 +1912,10 @@ trait StorageBase
}
fclose($sourceHandle);
if ($duplicate) {
$requests = array_merge($requests, $requests);
}
$responses = [];
$endpoint = parse_url($this->client->getEndpoint());
$scheme = $endpoint['scheme'] ?? 'http';
@@ -1958,6 +1974,7 @@ trait StorageBase
ksort($responses);
$this->assertCount(count($requests), $responses);
foreach ($responses as $response) {
$this->assertSame('', $response['error']);
$this->assertContains($response['statusCode'], [200, 201], (string) $response['body']);
@@ -1973,6 +1990,21 @@ trait StorageBase
$this->assertEquals($chunksTotal, $uploadedFile['body']['chunksTotal']);
$this->assertEquals($chunksTotal, $uploadedFile['body']['chunksUploaded']);
// A late retry must return the completed file without writing or finalizing again.
$retry = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge(
$requests[0]['headers'],
['content-type' => 'multipart/form-data']
), [
'fileId' => $fileId,
'file' => new CURLFile($requests[0]['chunkPath'], 'application/octet-stream', 'large-parallel-upload.bin'),
'permissions' => [Permission::read(Role::any()), Permission::delete(Role::any())],
]);
$this->assertEquals(200, $retry['headers']['status-code']);
$this->assertEquals($fileId, $retry['body']['$id']);
$this->assertEquals($chunksTotal, $retry['body']['chunksUploaded']);
$this->assertEquals($uploadedFile['body']['signature'], $retry['body']['signature']);
$download = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],