mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge branch 'main' into fix/usage-e2e-schema-readiness
This commit is contained in:
3 files changed
+133
-76
No files matched your search
@@ -306,7 +306,13 @@ $container->set('redis', function () {
|
||||
});
|
||||
|
||||
$container->set('locks', fn (Group $pools) => fn (string $key, int $ttl, callable $callback, float $timeout = 0.0): mixed => $pools->get('lock')->use(
|
||||
fn (\Redis $redis) => (new Distributed($redis, $key, ttl: $ttl))->withLock($callback, timeout: $timeout)
|
||||
function (\Redis $redis) use ($key, $ttl, $callback, $timeout): mixed {
|
||||
// The callback receives the lock so long-running holders can refresh
|
||||
// the lease and verify it is still theirs before committing work.
|
||||
$lock = new Distributed($redis, $key, ttl: $ttl);
|
||||
|
||||
return $lock->withLock(fn () => $callback($lock), timeout: $timeout);
|
||||
}
|
||||
), ['pools']);
|
||||
|
||||
$container->set('timelimit', fn (\Redis $redis) => fn (string $key, int $limit, int $time) => new TimeLimitRedis($key, $limit, $time, $redis), ['redis']);
|
||||
|
||||
@@ -31,6 +31,7 @@ use Utopia\Database\Validator\Authorization\Input;
|
||||
use Utopia\Database\Validator\Permissions;
|
||||
use Utopia\Database\Validator\UID;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Lock\Distributed;
|
||||
use Utopia\Lock\Exception\Contention as LockContention;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
@@ -47,6 +48,12 @@ class Create extends Action
|
||||
{
|
||||
use HTTP;
|
||||
|
||||
/**
|
||||
* Lease for the per-file upload lock, in seconds. Refreshed before the
|
||||
* transfer and before completion so a single chunk always gets the full window.
|
||||
*/
|
||||
private const LOCK_TTL = 600;
|
||||
|
||||
public static function getName()
|
||||
{
|
||||
return 'createFile';
|
||||
@@ -265,71 +272,62 @@ class Create extends Action
|
||||
return $merged;
|
||||
};
|
||||
|
||||
try {
|
||||
$locks($lockKey, 600, function () use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $folder, $path, $permissions, $response, &$completed): void {
|
||||
$file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId));
|
||||
if (!$file->isEmpty()) {
|
||||
$chunks = $file->getAttribute('chunksTotal', 1);
|
||||
$uploaded = $file->getAttribute('chunksUploaded', 0);
|
||||
$metadata = $file->getAttribute('metadata', []);
|
||||
$prepareUpload = function () use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $folder, $path, $permissions, $response, &$completed): void {
|
||||
$file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId));
|
||||
if (!$file->isEmpty()) {
|
||||
$chunks = $file->getAttribute('chunksTotal', 1);
|
||||
$uploaded = $file->getAttribute('chunksUploaded', 0);
|
||||
$metadata = $file->getAttribute('metadata', []);
|
||||
|
||||
if ($uploaded === $chunks) {
|
||||
if (empty($contentRange)) {
|
||||
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
|
||||
}
|
||||
if ($uploaded === $chunks) {
|
||||
if (empty($contentRange)) {
|
||||
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_OK)
|
||||
->dynamic($file, Response::MODEL_FILE);
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_OK)
|
||||
->dynamic($file, Response::MODEL_FILE);
|
||||
|
||||
$completed = true;
|
||||
return;
|
||||
$completed = true;
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if ($file->isEmpty()) {
|
||||
$deviceForFiles->prepare($path, $metadata['content_type'] ?? '', $chunks, $metadata);
|
||||
|
||||
if (!empty($contentRange)) {
|
||||
$doc = new Document([
|
||||
'$id' => ID::custom($fileId),
|
||||
'$permissions' => $permissions,
|
||||
'bucketId' => $bucket->getId(),
|
||||
'bucketInternalId' => $bucket->getSequence(),
|
||||
'name' => $fileName,
|
||||
'folder' => $folder,
|
||||
'path' => $path,
|
||||
'signature' => '',
|
||||
'mimeType' => '',
|
||||
'sizeOriginal' => $fileSize,
|
||||
'sizeActual' => 0,
|
||||
'algorithm' => '',
|
||||
'comment' => '',
|
||||
'chunksTotal' => $chunks,
|
||||
'chunksUploaded' => 0,
|
||||
'search' => implode(' ', [$fileId, $fileName]),
|
||||
'metadata' => $metadata,
|
||||
]);
|
||||
|
||||
try {
|
||||
$dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc);
|
||||
} catch (DuplicateException) {
|
||||
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
|
||||
} catch (NotFoundException) {
|
||||
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
|
||||
}
|
||||
}
|
||||
|
||||
if ($file->isEmpty()) {
|
||||
$deviceForFiles->prepare($path, $metadata['content_type'] ?? '', $chunks, $metadata);
|
||||
|
||||
if (!empty($contentRange)) {
|
||||
$doc = new Document([
|
||||
'$id' => ID::custom($fileId),
|
||||
'$permissions' => $permissions,
|
||||
'bucketId' => $bucket->getId(),
|
||||
'bucketInternalId' => $bucket->getSequence(),
|
||||
'name' => $fileName,
|
||||
'folder' => $folder,
|
||||
'path' => $path,
|
||||
'signature' => '',
|
||||
'mimeType' => '',
|
||||
'sizeOriginal' => $fileSize,
|
||||
'sizeActual' => 0,
|
||||
'algorithm' => '',
|
||||
'comment' => '',
|
||||
'chunksTotal' => $chunks,
|
||||
'chunksUploaded' => 0,
|
||||
'search' => implode(' ', [$fileId, $fileName]),
|
||||
'metadata' => $metadata,
|
||||
]);
|
||||
|
||||
try {
|
||||
$dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc);
|
||||
} catch (DuplicateException) {
|
||||
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
|
||||
} catch (NotFoundException) {
|
||||
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
|
||||
}
|
||||
}
|
||||
}
|
||||
}, timeout: 120.0);
|
||||
} catch (LockContention) {
|
||||
$response->addHeader('Retry-After', '5');
|
||||
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'File upload is busy. Try again.');
|
||||
}
|
||||
|
||||
if ($completed) {
|
||||
$queueForEvents->reset();
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
$finalizeUpload = function (int $chunksUploaded) use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $mergeUploadMetadata, $folder, $path, $permissions, $queueForEvents, $response): void {
|
||||
$file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId));
|
||||
@@ -355,14 +353,11 @@ class Create extends Action
|
||||
}
|
||||
}
|
||||
|
||||
// Another chunk may have finalized the upload and removed its parts
|
||||
// before upload() counted them. Check the completed document above first.
|
||||
if (empty($chunksUploaded)) {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file');
|
||||
}
|
||||
|
||||
// A local chunk file is visible before its write finishes. Only parts
|
||||
// recorded after upload() returns are safe to include in finalization.
|
||||
// Count distinct completed parts, including previously persisted chunks.
|
||||
$chunksUploaded = max($uploaded, isset($metadata['parts']) ? \count($metadata['parts']) : $chunksUploaded);
|
||||
|
||||
if ($chunksUploaded === $chunks && $uploaded < $chunks) {
|
||||
@@ -525,16 +520,40 @@ class Create extends Action
|
||||
};
|
||||
|
||||
try {
|
||||
$chunksUploaded = $deviceForFiles->upload(
|
||||
$deviceForLocal->read($fileTmpName),
|
||||
$path,
|
||||
$metadata['content_type'] ?? '',
|
||||
$chunk,
|
||||
$chunks,
|
||||
$metadata
|
||||
);
|
||||
// upload() can finalize and remove chunk files itself. Keep preparation,
|
||||
// transfer and document completion under the same per-file lock.
|
||||
$locks($lockKey, self::LOCK_TTL, function (Distributed $lock) use ($prepareUpload, $finalizeUpload, &$completed, $queueForEvents, $deviceForFiles, $deviceForLocal, $fileTmpName, $path, $chunk, &$chunks, &$metadata): void {
|
||||
$prepareUpload();
|
||||
|
||||
$locks($lockKey, 600, fn () => $finalizeUpload($chunksUploaded), timeout: 120.0);
|
||||
if ($completed) {
|
||||
$queueForEvents->reset();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Restart the lease so the transfer gets the full window,
|
||||
// regardless of how long preparation took.
|
||||
if (!$lock->refresh()) {
|
||||
throw new LockContention('Upload lease lost before transfer: ' . $lock->token());
|
||||
}
|
||||
|
||||
$chunksUploaded = $deviceForFiles->upload(
|
||||
$deviceForLocal->read($fileTmpName),
|
||||
$path,
|
||||
$metadata['content_type'] ?? '',
|
||||
$chunk,
|
||||
$chunks,
|
||||
$metadata
|
||||
);
|
||||
|
||||
// Never record completion under a lapsed lease: another request
|
||||
// may already own the file and be finalizing it.
|
||||
if (!$lock->isHeld()) {
|
||||
throw new LockContention('Upload lease lost after transfer: ' . $lock->token());
|
||||
}
|
||||
|
||||
$finalizeUpload($chunksUploaded);
|
||||
}, timeout: 120.0);
|
||||
} catch (LockContention) {
|
||||
$response->addHeader('Retry-After', '5');
|
||||
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'File upload is busy. Try again.');
|
||||
|
||||
@@ -4,6 +4,7 @@ namespace Tests\E2E\Services\Storage;
|
||||
|
||||
use Appwrite\Extend\Exception;
|
||||
use CURLFile;
|
||||
use PHPUnit\Framework\Attributes\DataProvider;
|
||||
use PHPUnit\Framework\Attributes\Group;
|
||||
use Tests\E2E\Client;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
@@ -1829,8 +1830,18 @@ trait StorageBase
|
||||
]);
|
||||
}
|
||||
|
||||
public function testCreateBucketFileParallelChunksLargeFile(): void
|
||||
public static function parallelChunksProvider(): array
|
||||
{
|
||||
return [
|
||||
'distinct chunks' => [false],
|
||||
'duplicate chunks' => [true],
|
||||
];
|
||||
}
|
||||
|
||||
#[DataProvider('parallelChunksProvider')]
|
||||
public function testCreateBucketFileParallelChunksLargeFile(bool $duplicate): void
|
||||
{
|
||||
// Test for SUCCESS
|
||||
$totalSize = 20 * 1024 * 1024;
|
||||
$chunkSize = 5 * 1024 * 1024;
|
||||
$chunksTotal = (int) ceil($totalSize / $chunkSize);
|
||||
@@ -1867,8 +1878,9 @@ trait StorageBase
|
||||
$this->assertNotFalse($handle, 'Could not create test file');
|
||||
|
||||
$remaining = $totalSize;
|
||||
$block = str_repeat(hash('sha256', $fileId, binary: true), 1024);
|
||||
while ($remaining > 0) {
|
||||
// Distinct blocks expose reordered or duplicated chunks in the hash check.
|
||||
$block = str_repeat(hash('sha256', $fileId . ':' . $remaining, binary: true), 1024);
|
||||
$bytes = substr($block, 0, min(strlen($block), $remaining));
|
||||
fwrite($handle, $bytes);
|
||||
$remaining -= strlen($bytes);
|
||||
@@ -1900,6 +1912,10 @@ trait StorageBase
|
||||
}
|
||||
fclose($sourceHandle);
|
||||
|
||||
if ($duplicate) {
|
||||
$requests = array_merge($requests, $requests);
|
||||
}
|
||||
|
||||
$responses = [];
|
||||
$endpoint = parse_url($this->client->getEndpoint());
|
||||
$scheme = $endpoint['scheme'] ?? 'http';
|
||||
@@ -1958,6 +1974,7 @@ trait StorageBase
|
||||
|
||||
ksort($responses);
|
||||
|
||||
$this->assertCount(count($requests), $responses);
|
||||
foreach ($responses as $response) {
|
||||
$this->assertSame('', $response['error']);
|
||||
$this->assertContains($response['statusCode'], [200, 201], (string) $response['body']);
|
||||
@@ -1973,6 +1990,21 @@ trait StorageBase
|
||||
$this->assertEquals($chunksTotal, $uploadedFile['body']['chunksTotal']);
|
||||
$this->assertEquals($chunksTotal, $uploadedFile['body']['chunksUploaded']);
|
||||
|
||||
// A late retry must return the completed file without writing or finalizing again.
|
||||
$retry = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge(
|
||||
$requests[0]['headers'],
|
||||
['content-type' => 'multipart/form-data']
|
||||
), [
|
||||
'fileId' => $fileId,
|
||||
'file' => new CURLFile($requests[0]['chunkPath'], 'application/octet-stream', 'large-parallel-upload.bin'),
|
||||
'permissions' => [Permission::read(Role::any()), Permission::delete(Role::any())],
|
||||
]);
|
||||
|
||||
$this->assertEquals(200, $retry['headers']['status-code']);
|
||||
$this->assertEquals($fileId, $retry['body']['$id']);
|
||||
$this->assertEquals($chunksTotal, $retry['body']['chunksUploaded']);
|
||||
$this->assertEquals($uploadedFile['body']['signature'], $retry['body']['signature']);
|
||||
|
||||
$download = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
|
||||
Reference in new issue
Block a user