mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Fix parallel storage chunk upload state
This commit is contained in:
1 parent
554fdf5472
commit
61ca4e3969
4 files changed
+471
-189
No files matched your search
@@ -29,6 +29,8 @@ use Utopia\Database\Validator\Authorization\Input;
|
||||
use Utopia\Database\Validator\Permissions;
|
||||
use Utopia\Database\Validator\UID;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Lock\Distributed;
|
||||
use Utopia\Lock\Exception\Contention as LockContention;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\Storage\Device;
|
||||
@@ -86,12 +88,14 @@ class Create extends Action
|
||||
->inject('request')
|
||||
->inject('response')
|
||||
->inject('dbForProject')
|
||||
->inject('project')
|
||||
->inject('user')
|
||||
->inject('queueForEvents')
|
||||
->inject('mode')
|
||||
->inject('deviceForFiles')
|
||||
->inject('deviceForLocal')
|
||||
->inject('authorization')
|
||||
->inject('redis')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
@@ -103,12 +107,14 @@ class Create extends Action
|
||||
Request $request,
|
||||
Response $response,
|
||||
Database $dbForProject,
|
||||
Document $project,
|
||||
User $user,
|
||||
Event $queueForEvents,
|
||||
string $mode,
|
||||
Device $deviceForFiles,
|
||||
Device $deviceForLocal,
|
||||
Authorization $authorization
|
||||
Authorization $authorization,
|
||||
\Redis $redis
|
||||
) {
|
||||
$bucket = $authorization->skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
|
||||
|
||||
@@ -234,24 +240,43 @@ class Create extends Action
|
||||
$path = $deviceForFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
|
||||
$path = str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root
|
||||
|
||||
$file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId);
|
||||
$lock = new Distributed(
|
||||
$redis,
|
||||
'storage:file:' . $project->getId() . ':' . $bucket->getId() . ':' . $fileId,
|
||||
ttl: 600,
|
||||
);
|
||||
|
||||
$metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)];
|
||||
if (!$file->isEmpty()) {
|
||||
$chunks = $file->getAttribute('chunksTotal', 1);
|
||||
$uploaded = $file->getAttribute('chunksUploaded', 0);
|
||||
$metadata = $file->getAttribute('metadata', []);
|
||||
$completed = false;
|
||||
|
||||
if ($uploaded === $chunks) {
|
||||
if (empty($contentRange)) {
|
||||
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
|
||||
try {
|
||||
$lock->withLock(function () use ($bucket, &$chunks, $contentRange, $dbForProject, $fileId, &$metadata, &$completed, $response): void {
|
||||
$file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId);
|
||||
if (!$file->isEmpty()) {
|
||||
$chunks = $file->getAttribute('chunksTotal', 1);
|
||||
$uploaded = $file->getAttribute('chunksUploaded', 0);
|
||||
$metadata = $file->getAttribute('metadata', []);
|
||||
|
||||
if ($uploaded === $chunks) {
|
||||
if (empty($contentRange)) {
|
||||
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_OK)
|
||||
->dynamic($file, Response::MODEL_FILE);
|
||||
|
||||
$completed = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}, timeout: 120.0);
|
||||
} catch (LockContention) {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'File upload is busy. Try again.');
|
||||
}
|
||||
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_OK)
|
||||
->dynamic($file, Response::MODEL_FILE);
|
||||
return;
|
||||
}
|
||||
if ($completed) {
|
||||
return;
|
||||
}
|
||||
|
||||
$chunksUploaded = $deviceForFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata);
|
||||
@@ -260,187 +285,211 @@ class Create extends Action
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file');
|
||||
}
|
||||
|
||||
if ($chunksUploaded === $chunks) {
|
||||
if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) {
|
||||
$antivirus = new Network(
|
||||
System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
|
||||
(int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)
|
||||
);
|
||||
try {
|
||||
$lock->withLock(function () use ($authorization, $bucket, &$chunks, $chunksUploaded, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $queueForEvents, $response): void {
|
||||
$file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId);
|
||||
|
||||
if (!$antivirus->fileScan($path)) {
|
||||
$deviceForFiles->delete($path);
|
||||
throw new Exception(Exception::STORAGE_INVALID_FILE);
|
||||
if (!$file->isEmpty()) {
|
||||
$chunks = $file->getAttribute('chunksTotal', 1);
|
||||
$uploaded = $file->getAttribute('chunksUploaded', 0);
|
||||
$metadata = \array_merge($file->getAttribute('metadata', []), $metadata);
|
||||
|
||||
if ($uploaded === $chunks) {
|
||||
if (empty($contentRange)) {
|
||||
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_OK)
|
||||
->dynamic($file, Response::MODEL_FILE);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption
|
||||
$fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption
|
||||
$data = '';
|
||||
$iv = '';
|
||||
$tag = null;
|
||||
// Compression
|
||||
$algorithm = $bucket->getAttribute('compression', Compression::NONE);
|
||||
if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) {
|
||||
$data = $deviceForFiles->read($path);
|
||||
switch ($algorithm) {
|
||||
case Compression::ZSTD:
|
||||
$compressor = new Zstd();
|
||||
break;
|
||||
case Compression::GZIP:
|
||||
default:
|
||||
$compressor = new GZIP();
|
||||
break;
|
||||
if ($chunksUploaded === $chunks) {
|
||||
if (System::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceForFiles->getType() === Storage::DEVICE_LOCAL) {
|
||||
$antivirus = new Network(
|
||||
System::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
|
||||
(int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)
|
||||
);
|
||||
|
||||
if (!$antivirus->fileScan($path)) {
|
||||
$deviceForFiles->delete($path);
|
||||
throw new Exception(Exception::STORAGE_INVALID_FILE);
|
||||
}
|
||||
}
|
||||
|
||||
$mimeType = $deviceForFiles->getFileMimeType($path); // Get mime-type before compression and encryption
|
||||
$fileHash = $deviceForFiles->getFileHash($path); // Get file hash before compression and encryption
|
||||
$data = '';
|
||||
$iv = '';
|
||||
$tag = null;
|
||||
// Compression
|
||||
$algorithm = $bucket->getAttribute('compression', Compression::NONE);
|
||||
if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) {
|
||||
$data = $deviceForFiles->read($path);
|
||||
switch ($algorithm) {
|
||||
case Compression::ZSTD:
|
||||
$compressor = new Zstd();
|
||||
break;
|
||||
case Compression::GZIP:
|
||||
default:
|
||||
$compressor = new GZIP();
|
||||
break;
|
||||
}
|
||||
$data = $compressor->compress($data);
|
||||
} else {
|
||||
// reset the algorithm to none as we do not compress the file
|
||||
// if file size exceedes the APP_STORAGE_READ_BUFFER
|
||||
// regardless the bucket compression algoorithm
|
||||
$algorithm = Compression::NONE;
|
||||
}
|
||||
|
||||
if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) {
|
||||
if (empty($data)) {
|
||||
$data = $deviceForFiles->read($path);
|
||||
}
|
||||
$key = System::getEnv('_APP_OPENSSL_KEY_V1');
|
||||
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
|
||||
$data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag);
|
||||
}
|
||||
|
||||
if (!empty($data)) {
|
||||
if (!$deviceForFiles->write($path, $data, $mimeType)) {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file');
|
||||
}
|
||||
}
|
||||
|
||||
$sizeActual = $deviceForFiles->getFileSize($path);
|
||||
|
||||
$openSSLVersion = null;
|
||||
$openSSLCipher = null;
|
||||
$openSSLTag = null;
|
||||
$openSSLIV = null;
|
||||
|
||||
if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) {
|
||||
$openSSLVersion = '1';
|
||||
$openSSLCipher = OpenSSL::CIPHER_AES_128_GCM;
|
||||
$openSSLTag = \bin2hex($tag);
|
||||
$openSSLIV = \bin2hex($iv);
|
||||
}
|
||||
|
||||
if ($file->isEmpty()) {
|
||||
$doc = new Document([
|
||||
'$id' => $fileId,
|
||||
'$permissions' => $permissions,
|
||||
'bucketId' => $bucket->getId(),
|
||||
'bucketInternalId' => $bucket->getSequence(),
|
||||
'name' => $fileName,
|
||||
'path' => $path,
|
||||
'signature' => $fileHash,
|
||||
'mimeType' => $mimeType,
|
||||
'sizeOriginal' => $fileSize,
|
||||
'sizeActual' => $sizeActual,
|
||||
'algorithm' => $algorithm,
|
||||
'comment' => '',
|
||||
'chunksTotal' => $chunks,
|
||||
'chunksUploaded' => $chunksUploaded,
|
||||
'openSSLVersion' => $openSSLVersion,
|
||||
'openSSLCipher' => $openSSLCipher,
|
||||
'openSSLTag' => $openSSLTag,
|
||||
'openSSLIV' => $openSSLIV,
|
||||
'search' => implode(' ', [$fileId, $fileName]),
|
||||
'metadata' => $metadata,
|
||||
]);
|
||||
|
||||
try {
|
||||
$file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc);
|
||||
} catch (DuplicateException) {
|
||||
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
|
||||
} catch (NotFoundException) {
|
||||
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
|
||||
}
|
||||
} else {
|
||||
/**
|
||||
* Skip authorization in updateDocument.
|
||||
* Without this, the file creation will fail when user doesn't have update permission.
|
||||
* However as with chunk upload even if we are updating, we are essentially creating a file
|
||||
* adding it's new chunk so we rely on the create-permission check performed earlier.
|
||||
*/
|
||||
$file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([
|
||||
'$permissions' => $permissions,
|
||||
'signature' => $fileHash,
|
||||
'mimeType' => $mimeType,
|
||||
'sizeActual' => $sizeActual,
|
||||
'algorithm' => $algorithm,
|
||||
'openSSLVersion' => $openSSLVersion,
|
||||
'openSSLCipher' => $openSSLCipher,
|
||||
'openSSLTag' => $openSSLTag,
|
||||
'openSSLIV' => $openSSLIV,
|
||||
'metadata' => $metadata,
|
||||
'chunksUploaded' => $chunksUploaded,
|
||||
])));
|
||||
}
|
||||
|
||||
// Trigger after create success hook
|
||||
$this->afterCreateSuccess($file);
|
||||
} else {
|
||||
if ($file->isEmpty()) {
|
||||
$doc = new Document([
|
||||
'$id' => ID::custom($fileId),
|
||||
'$permissions' => $permissions,
|
||||
'bucketId' => $bucket->getId(),
|
||||
'bucketInternalId' => $bucket->getSequence(),
|
||||
'name' => $fileName,
|
||||
'path' => $path,
|
||||
'signature' => '',
|
||||
'mimeType' => '',
|
||||
'sizeOriginal' => $fileSize,
|
||||
'sizeActual' => 0,
|
||||
'algorithm' => '',
|
||||
'comment' => '',
|
||||
'chunksTotal' => $chunks,
|
||||
'chunksUploaded' => $chunksUploaded,
|
||||
'search' => implode(' ', [$fileId, $fileName]),
|
||||
'metadata' => $metadata,
|
||||
]);
|
||||
|
||||
try {
|
||||
$file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc);
|
||||
} catch (DuplicateException) {
|
||||
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
|
||||
} catch (NotFoundException) {
|
||||
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
|
||||
}
|
||||
} else {
|
||||
/**
|
||||
* Skip authorization in updateDocument.
|
||||
* Without this, the file creation will fail when user doesn't have update permission.
|
||||
* However as with chunk upload even if we are updating, we are essentially creating a file
|
||||
* adding it's new chunk so we rely on the create-permission check performed earlier.
|
||||
*/
|
||||
try {
|
||||
$file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([
|
||||
'chunksUploaded' => $chunksUploaded,
|
||||
'metadata' => $metadata,
|
||||
])));
|
||||
} catch (NotFoundException) {
|
||||
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
|
||||
}
|
||||
}
|
||||
}
|
||||
$data = $compressor->compress($data);
|
||||
} else {
|
||||
// reset the algorithm to none as we do not compress the file
|
||||
// if file size exceedes the APP_STORAGE_READ_BUFFER
|
||||
// regardless the bucket compression algoorithm
|
||||
$algorithm = Compression::NONE;
|
||||
}
|
||||
|
||||
if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) {
|
||||
if (empty($data)) {
|
||||
$data = $deviceForFiles->read($path);
|
||||
}
|
||||
$key = System::getEnv('_APP_OPENSSL_KEY_V1');
|
||||
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
|
||||
$data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag);
|
||||
}
|
||||
$queueForEvents
|
||||
->setParam('bucketId', $bucket->getId())
|
||||
->setParam('fileId', $file->getId())
|
||||
->setContext('bucket', $bucket);
|
||||
|
||||
if (!empty($data)) {
|
||||
if (!$deviceForFiles->write($path, $data, $mimeType)) {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file');
|
||||
}
|
||||
}
|
||||
$metadata = null; // was causing leaks as it was passed by reference
|
||||
|
||||
$sizeActual = $deviceForFiles->getFileSize($path);
|
||||
|
||||
$openSSLVersion = null;
|
||||
$openSSLCipher = null;
|
||||
$openSSLTag = null;
|
||||
$openSSLIV = null;
|
||||
|
||||
if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) {
|
||||
$openSSLVersion = '1';
|
||||
$openSSLCipher = OpenSSL::CIPHER_AES_128_GCM;
|
||||
$openSSLTag = \bin2hex($tag);
|
||||
$openSSLIV = \bin2hex($iv);
|
||||
}
|
||||
|
||||
if ($file->isEmpty()) {
|
||||
$doc = new Document([
|
||||
'$id' => $fileId,
|
||||
'$permissions' => $permissions,
|
||||
'bucketId' => $bucket->getId(),
|
||||
'bucketInternalId' => $bucket->getSequence(),
|
||||
'name' => $fileName,
|
||||
'path' => $path,
|
||||
'signature' => $fileHash,
|
||||
'mimeType' => $mimeType,
|
||||
'sizeOriginal' => $fileSize,
|
||||
'sizeActual' => $sizeActual,
|
||||
'algorithm' => $algorithm,
|
||||
'comment' => '',
|
||||
'chunksTotal' => $chunks,
|
||||
'chunksUploaded' => $chunksUploaded,
|
||||
'openSSLVersion' => $openSSLVersion,
|
||||
'openSSLCipher' => $openSSLCipher,
|
||||
'openSSLTag' => $openSSLTag,
|
||||
'openSSLIV' => $openSSLIV,
|
||||
'search' => implode(' ', [$fileId, $fileName]),
|
||||
'metadata' => $metadata,
|
||||
]);
|
||||
|
||||
try {
|
||||
$file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc);
|
||||
} catch (DuplicateException) {
|
||||
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
|
||||
} catch (NotFoundException) {
|
||||
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
|
||||
}
|
||||
} else {
|
||||
$file = $file
|
||||
->setAttribute('$permissions', $permissions)
|
||||
->setAttribute('signature', $fileHash)
|
||||
->setAttribute('mimeType', $mimeType)
|
||||
->setAttribute('sizeActual', $sizeActual)
|
||||
->setAttribute('algorithm', $algorithm)
|
||||
->setAttribute('openSSLVersion', $openSSLVersion)
|
||||
->setAttribute('openSSLCipher', $openSSLCipher)
|
||||
->setAttribute('openSSLTag', $openSSLTag)
|
||||
->setAttribute('openSSLIV', $openSSLIV)
|
||||
->setAttribute('metadata', $metadata)
|
||||
->setAttribute('chunksUploaded', $chunksUploaded);
|
||||
|
||||
/**
|
||||
* Skip authorization in updateDocument.
|
||||
* Without this, the file creation will fail when user doesn't have update permission.
|
||||
* However as with chunk upload even if we are updating, we are essentially creating a file
|
||||
* adding it's new chunk so we rely on the create-permission check performed earlier.
|
||||
*/
|
||||
$file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file));
|
||||
}
|
||||
|
||||
// Trigger after create success hook
|
||||
$this->afterCreateSuccess($file);
|
||||
} else {
|
||||
if ($file->isEmpty()) {
|
||||
$doc = new Document([
|
||||
'$id' => ID::custom($fileId),
|
||||
'$permissions' => $permissions,
|
||||
'bucketId' => $bucket->getId(),
|
||||
'bucketInternalId' => $bucket->getSequence(),
|
||||
'name' => $fileName,
|
||||
'path' => $path,
|
||||
'signature' => '',
|
||||
'mimeType' => '',
|
||||
'sizeOriginal' => $fileSize,
|
||||
'sizeActual' => 0,
|
||||
'algorithm' => '',
|
||||
'comment' => '',
|
||||
'chunksTotal' => $chunks,
|
||||
'chunksUploaded' => $chunksUploaded,
|
||||
'search' => implode(' ', [$fileId, $fileName]),
|
||||
'metadata' => $metadata,
|
||||
]);
|
||||
|
||||
try {
|
||||
$file = $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $doc);
|
||||
} catch (DuplicateException) {
|
||||
throw new Exception(Exception::STORAGE_FILE_ALREADY_EXISTS);
|
||||
} catch (NotFoundException) {
|
||||
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
|
||||
}
|
||||
} else {
|
||||
$file = $file
|
||||
->setAttribute('chunksUploaded', $chunksUploaded)
|
||||
->setAttribute('metadata', $metadata);
|
||||
|
||||
/**
|
||||
* Skip authorization in updateDocument.
|
||||
* Without this, the file creation will fail when user doesn't have update permission.
|
||||
* However as with chunk upload even if we are updating, we are essentially creating a file
|
||||
* adding it's new chunk so we rely on the create-permission check performed earlier.
|
||||
*/
|
||||
try {
|
||||
$file = $authorization->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, $file));
|
||||
} catch (NotFoundException) {
|
||||
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
|
||||
}
|
||||
}
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_CREATED)
|
||||
->dynamic($file, Response::MODEL_FILE);
|
||||
}, timeout: 120.0);
|
||||
} catch (LockContention) {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'File upload is busy. Try again.');
|
||||
}
|
||||
|
||||
$queueForEvents
|
||||
->setParam('bucketId', $bucket->getId())
|
||||
->setParam('fileId', $file->getId())
|
||||
->setContext('bucket', $bucket);
|
||||
|
||||
$metadata = null; // was causing leaks as it was passed by reference
|
||||
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_CREATED)
|
||||
->dynamic($file, Response::MODEL_FILE);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in new issue
Block a user