Fix parallel storage chunk upload state

This commit is contained in:
Torsten Dittmann committed 2026-05-04 14:15:01 +04:00
1 parent 554fdf5472
commit 61ca4e3969
4 files changed
+471 -189

No files matched your search

@@ -29,6 +29,8 @@ use Utopia\Database\Validator\Authorization\Input;
use Utopia\Database\Validator\Permissions;
use Utopia\Database\Validator\UID;
use Utopia\Http\Adapter\Swoole\Request;
use Utopia\Lock\Distributed;
use Utopia\Lock\Exception\Contention as LockContention;
use Utopia\Platform\Action;
use Utopia\Platform\Scope\HTTP;
use Utopia\Storage\Device;
@@ -86,12 +88,14 @@ class Create extends Action
->inject('request')
->inject('response')
->inject('dbForProject')
->inject('project')
->inject('user')
->inject('queueForEvents')
->inject('mode')
->inject('deviceForFiles')
->inject('deviceForLocal')
->inject('authorization')
->inject('redis')
->callback($this->action(...));
}
@@ -103,12 +107,14 @@ class Create extends Action
Request $request,
Response $response,
Database $dbForProject,
Document $project,
User $user,
Event $queueForEvents,
string $mode,
Device $deviceForFiles,
Device $deviceForLocal,
Authorization $authorization
Authorization $authorization,
\Redis $redis
) {
$bucket = $authorization->skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
@@ -234,24 +240,43 @@ class Create extends Action
$path = $deviceForFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
$path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root
$file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId);
$lock = new Distributed(
$redis,
'storage:file:' . $project->getId() . ':' . $bucket->getId() . ':' . $fileId,
ttl: 600,
);
$metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)];
if (!$file->isEmpty()) {
$chunks = $file->getAttribute('chunksTotal', 1);
$uploaded = $file->getAttribute('chunksUploaded', 0);
$metadata = $file->getAttribute('metadata', []);
$completed = false;
if ($uploaded === $chunks) {
if (empty($contentRange)) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
try {
$lock->withLock(function () use ($bucket, &$chunks, $contentRange, $dbForProject, $fileId, &$metadata, &$completed, $response): void {
$file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId);
if (!$file->isEmpty()) {
$chunks = $file->getAttribute('chunksTotal', 1);
$uploaded = $file->getAttribute('chunksUploaded', 0);
$metadata = $file->getAttribute('metadata', []);
if ($uploaded === $chunks) {
if (empty($contentRange)) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
}
$response
->setStatusCode(Response::STATUS_CODE_OK)
->dynamic($file, Response::MODEL_FILE);
$completed = true;
return;
}
}
}, timeout: 120.0);
} catch (LockContention) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'File upload is busy. Try again.');
}
$response
->setStatusCode(Response::STATUS_CODE_OK)
->dynamic($file, Response::MODEL_FILE);
return;
}
if ($completed) {
return;
}
$chunksUploaded = $deviceForFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata);
@@ -260,187 +285,211 @@ class Create extends Action
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file');
}
if ($chunksUploaded === $chunks) {
if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) {
$antivirus = new Network(
System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
(int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)
);
try {
$lock->withLock(function () use ($authorization, $bucket, &$chunks, $chunksUploaded, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $queueForEvents, $response): void {
$file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId);
if (!$antivirus->fileScan($path)) {
$deviceForFiles->delete($path);
throw new Exception(Exception::STORAGE_INVALID_FILE);
if (!$file->isEmpty()) {
$chunks = $file->getAttribute('chunksTotal', 1);
$uploaded = $file->getAttribute('chunksUploaded', 0);
$metadata = \array_merge($file->getAttribute('metadata', []), $metadata);
if ($uploaded === $chunks) {
if (empty($contentRange)) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
}
$response
->setStatusCode(Response::STATUS_CODE_OK)
->dynamic($file, Response::MODEL_FILE);
return;
}
}
}
$mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption
$fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption
$data = '';
$iv = '';
$tag = null;
// Compression
$algorithm = $bucket->getAttribute('compression', Compression::NONE);
if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) {
$data = $deviceForFiles->read($path);
switch ($algorithm) {
case Compression::ZSTD:
$compressor = new Zstd();
break;
case Compression::GZIP:
default:
$compressor = new GZIP();
break;
if ($chunksUploaded === $chunks) {
if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) {
$antivirus = new Network(
System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
(int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)
);
if (!$antivirus->fileScan($path)) {
$deviceForFiles->delete($path);
throw new Exception(Exception::STORAGE_INVALID_FILE);
}
}
$mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption
$fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption
$data = '';
$iv = '';
$tag = null;
// Compression
$algorithm = $bucket->getAttribute('compression', Compression::NONE);
if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) {
$data = $deviceForFiles->read($path);
switch ($algorithm) {
case Compression::ZSTD:
$compressor = new Zstd();
break;
case Compression::GZIP:
default:
$compressor = new GZIP();
break;
}
$data = $compressor->compress($data);
} else {
// reset the algorithm to none as we do not compress the file
// if file size exceedes the APP_STORAGE_READ_BUFFER
// regardless the bucket compression algoorithm
$algorithm = Compression::NONE;
}
if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) {
if (empty($data)) {
$data = $deviceForFiles->read($path);
}
$key = System::getEnv('_APP_OPENSSL_KEY_V1');
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
$data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag);
}
if (!empty($data)) {
if (!$deviceForFiles->write($path, $data, $mimeType)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file');
}
}
$sizeActual = $deviceForFiles->getFileSize($path);
$openSSLVersion = null;
$openSSLCipher = null;
$openSSLTag = null;
$openSSLIV = null;
if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) {
$openSSLVersion = '1';
$openSSLCipher = OpenSSL::CIPHER_AES_128_GCM;
$openSSLTag = \bin2hex($tag);
$openSSLIV = \bin2hex($iv);
}
if ($file->isEmpty()) {
$doc = new Document([
'$id' => $fileId,
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getSequence(),
'name' => $fileName,
'path' => $path,
'signature' => $fileHash,
'mimeType' => $mimeType,
'sizeOriginal' => $fileSize,
'sizeActual' => $sizeActual,
'algorithm' => $algorithm,
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'openSSLVersion' => $openSSLVersion,
'openSSLCipher' => $openSSLCipher,
'openSSLTag' => $openSSLTag,
'openSSLIV' => $openSSLIV,
'search' => implode(' ', [$fileId, $fileName]),
'metadata' => $metadata,
]);
try {
$file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc);
} catch (DuplicateException) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
} catch (NotFoundException) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
} else {
/**
* Skip authorization in updateDocument.
* Without this, the file creation will fail when user doesn't have update permission.
* However as with chunk upload even if we are updating, we are essentially creating a file
* adding it's new chunk so we rely on the create-permission check performed earlier.
*/
$file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([
'$permissions' => $permissions,
'signature' => $fileHash,
'mimeType' => $mimeType,
'sizeActual' => $sizeActual,
'algorithm' => $algorithm,
'openSSLVersion' => $openSSLVersion,
'openSSLCipher' => $openSSLCipher,
'openSSLTag' => $openSSLTag,
'openSSLIV' => $openSSLIV,
'metadata' => $metadata,
'chunksUploaded' => $chunksUploaded,
])));
}
// Trigger after create success hook
$this->afterCreateSuccess($file);
} else {
if ($file->isEmpty()) {
$doc = new Document([
'$id' => ID::custom($fileId),
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getSequence(),
'name' => $fileName,
'path' => $path,
'signature' => '',
'mimeType' => '',
'sizeOriginal' => $fileSize,
'sizeActual' => 0,
'algorithm' => '',
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'search' => implode(' ', [$fileId, $fileName]),
'metadata' => $metadata,
]);
try {
$file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc);
} catch (DuplicateException) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
} catch (NotFoundException) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
} else {
/**
* Skip authorization in updateDocument.
* Without this, the file creation will fail when user doesn't have update permission.
* However as with chunk upload even if we are updating, we are essentially creating a file
* adding it's new chunk so we rely on the create-permission check performed earlier.
*/
try {
$file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([
'chunksUploaded' => $chunksUploaded,
'metadata' => $metadata,
])));
} catch (NotFoundException) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
}
}
$data = $compressor->compress($data);
} else {
// reset the algorithm to none as we do not compress the file
// if file size exceedes the APP_STORAGE_READ_BUFFER
// regardless the bucket compression algoorithm
$algorithm = Compression::NONE;
}
if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) {
if (empty($data)) {
$data = $deviceForFiles->read($path);
}
$key = System::getEnv('_APP_OPENSSL_KEY_V1');
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
$data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag);
}
$queueForEvents
->setParam('bucketId', $bucket->getId())
->setParam('fileId', $file->getId())
->setContext('bucket', $bucket);
if (!empty($data)) {
if (!$deviceForFiles->write($path, $data, $mimeType)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file');
}
}
$metadata = null; // was causing leaks as it was passed by reference
$sizeActual = $deviceForFiles->getFileSize($path);
$openSSLVersion = null;
$openSSLCipher = null;
$openSSLTag = null;
$openSSLIV = null;
if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) {
$openSSLVersion = '1';
$openSSLCipher = OpenSSL::CIPHER_AES_128_GCM;
$openSSLTag = \bin2hex($tag);
$openSSLIV = \bin2hex($iv);
}
if ($file->isEmpty()) {
$doc = new Document([
'$id' => $fileId,
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getSequence(),
'name' => $fileName,
'path' => $path,
'signature' => $fileHash,
'mimeType' => $mimeType,
'sizeOriginal' => $fileSize,
'sizeActual' => $sizeActual,
'algorithm' => $algorithm,
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'openSSLVersion' => $openSSLVersion,
'openSSLCipher' => $openSSLCipher,
'openSSLTag' => $openSSLTag,
'openSSLIV' => $openSSLIV,
'search' => implode(' ', [$fileId, $fileName]),
'metadata' => $metadata,
]);
try {
$file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc);
} catch (DuplicateException) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
} catch (NotFoundException) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
} else {
$file = $file
->setAttribute('$permissions', $permissions)
->setAttribute('signature', $fileHash)
->setAttribute('mimeType', $mimeType)
->setAttribute('sizeActual', $sizeActual)
->setAttribute('algorithm', $algorithm)
->setAttribute('openSSLVersion', $openSSLVersion)
->setAttribute('openSSLCipher', $openSSLCipher)
->setAttribute('openSSLTag', $openSSLTag)
->setAttribute('openSSLIV', $openSSLIV)
->setAttribute('metadata', $metadata)
->setAttribute('chunksUploaded', $chunksUploaded);
/**
* Skip authorization in updateDocument.
* Without this, the file creation will fail when user doesn't have update permission.
* However as with chunk upload even if we are updating, we are essentially creating a file
* adding it's new chunk so we rely on the create-permission check performed earlier.
*/
$file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file));
}
// Trigger after create success hook
$this->afterCreateSuccess($file);
} else {
if ($file->isEmpty()) {
$doc = new Document([
'$id' => ID::custom($fileId),
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getSequence(),
'name' => $fileName,
'path' => $path,
'signature' => '',
'mimeType' => '',
'sizeOriginal' => $fileSize,
'sizeActual' => 0,
'algorithm' => '',
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'search' => implode(' ', [$fileId, $fileName]),
'metadata' => $metadata,
]);
try {
$file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc);
} catch (DuplicateException) {
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
} catch (NotFoundException) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
} else {
$file = $file
->setAttribute('chunksUploaded', $chunksUploaded)
->setAttribute('metadata', $metadata);
/**
* Skip authorization in updateDocument.
* Without this, the file creation will fail when user doesn't have update permission.
* However as with chunk upload even if we are updating, we are essentially creating a file
* adding it's new chunk so we rely on the create-permission check performed earlier.
*/
try {
$file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file));
} catch (NotFoundException) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
}
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($file, Response::MODEL_FILE);
}, timeout: 120.0);
} catch (LockContention) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'File upload is busy. Try again.');
}
$queueForEvents
->setParam('bucketId', $bucket->getId())
->setParam('fileId', $file->getId())
->setContext('bucket', $bucket);
$metadata = null; // was causing leaks as it was passed by reference
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($file, Response::MODEL_FILE);
}
/**