diff --git a/app/config/collections2.php b/app/config/collections2.php index 91f8e9565f..9ee235ce56 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -1758,6 +1758,28 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'chunksTotal', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'chunksUploaded', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => 'search', 'type' => Database::VAR_STRING, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 926a22394c..aad914a437 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -431,42 +431,102 @@ App::post('/v1/functions/:functionId/tags') } // Make sure we handle a single file and multiple files the same way - $file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; - $file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; - $file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; + $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; + $size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; if (!$fileExt->isValid($file['name'])) { // Check if file type is allowed throw new Exception('File type not allowed', 400); } - if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit + $contentRange = $request->getHeader('content-range'); + $tagId = $dbForInternal->getId(); + $chunk = 1; + $chunks = 1; + + if (!empty($contentRange)) { + $start = $request->getContentRangeStart(); + $end = $request->getContentRangeEnd(); + $size = $request->getContentRangeSize(); + $tagId = $request->getHeader('x-appwrite-id', $tagId); + if(is_null($start) || is_null($end) || is_null($size)) { + throw new Exception('Invalid content-range header', 400); + } + + if ($end == $size) { + //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk + $chunks = $chunk = -1; + } else { + // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) + $chunks = (int) ceil($size / ($end + 1 - $start)); + $chunk = (int) ($start / ($end + 1 - $start)); + } + } + + if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit throw new Exception('File size not allowed', 400); } - if (!$upload->isValid($file['tmp_name'])) { + if (!$upload->isValid($fileTmpName)) { throw new Exception('Invalid file', 403); } // Save to storage - $size = $device->getFileSize($file['tmp_name']); - $path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION)); + $size ??= $device->getFileSize($fileTmpName); + $path = $device->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION)); - if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move' + $tag = $dbForInternal->getDocument('tags', $tagId); + + if(!$tag->isEmpty()) { + $chunks = $tag->getAttribute('chunksTotal', 1); + if($chunk == -1) { + $chunk = $chunks - 1; + } + } + + $chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks); + + if (empty($chunksUploaded)) { throw new Exception('Failed moving file', 500); } - $tagId = $dbForInternal->getId(); - $tag = $dbForInternal->createDocument('tags', new Document([ - '$id' => $tagId, - '$read' => [], - '$write' => [], - 'functionId' => $function->getId(), - 'dateCreated' => time(), - 'command' => $command, - 'path' => $path, - 'size' => $size, - 'search' => implode(' ', [$tagId, $command]), - ])); + if($chunksUploaded == $chunks) { + $size = $device->getFileSize($path); + + if ($tag->isEmpty()) { + $tag = $dbForInternal->createDocument('tags', new Document([ + '$id' => $tagId, + '$read' => [], + '$write' => [], + 'functionId' => $function->getId(), + 'dateCreated' => time(), + 'command' => $command, + 'path' => $path, + 'size' => $size, + 'search' => implode(' ', [$tagId, $command]), + ])); + } else { + $tag = $dbForInternal->updateDocument('tags', $tagId, $tag->setAttribute('size', $size)); + } + } else { + if($tag->isEmpty()) { + $tag = $dbForInternal->createDocument('tags', new Document([ + '$id' => $tagId, + '$read' => [], + '$write' => [], + 'functionId' => $function->getId(), + 'dateCreated' => time(), + 'command' => $command, + 'path' => $path, + 'size' => 0, + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'search' => implode(' ', [$tagId, $command]), + ])); + } else { + $tag = $dbForInternal->updateDocument('tags', $tagId, $tag->setAttribute('chunksUploaded', $chunksUploaded)); + } + } $usage ->setParam('storage', $tag->getAttribute('size', 0)) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 34272bfe49..1de53e1ab0 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1,19 +1,13 @@ param('allowedFileExtensions', [], new ArrayList(new Text(64)), 'Allowed file extensions', true) ->param('enabled', true, new Boolean(), 'Is bucket enabled?', true) ->param('adapter', 'local', new WhiteList(['local']), 'Storage adapter.', true) - ->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_LIMIT_ENCRYPTION) . ' encryption is skipped even if it\'s enabled', true) + ->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER) . ' encryption is skipped even if it\'s enabled', true) ->param('antiVirus', true, new Boolean(), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS) . ' AntiVirus scanning is skipped even if it\'s enabled', true) ->inject('response') ->inject('dbForInternal') @@ -226,6 +226,28 @@ App::post('/v1/storage/buckets') 'array' => false, 'filters' => [], ]), + new Document([ + '$id' => 'chunksTotal', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ]), + new Document([ + '$id' => 'chunksUploaded', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ]), new Document([ '$id' => 'search', 'type' => Database::VAR_STRING, @@ -238,6 +260,13 @@ App::post('/v1/storage/buckets') 'filters' => [], ]), ], [ + new Document([ + '$id' => '_key_search', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [2048], + 'orders' => [Database::ORDER_ASC], + ]), new Document([ '$id' => '_key_bucket', 'type' => Database::INDEX_KEY, @@ -245,13 +274,6 @@ App::post('/v1/storage/buckets') 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ]), - new Document([ - '$id' => '_key_search', - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['search'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ],), ]); $bucket = $dbForInternal->createDocument('buckets', new Document([ @@ -380,7 +402,7 @@ App::put('/v1/storage/buckets/:bucketId') ->param('maximumFileSize', null, new Integer(), 'Maximum file size allowed in bytes. Maximum allowed value is ' . App::getEnv('_APP_STORAGE_LIMIT', 0) . '. For self hosted version you can change the limit by changing _APP_STORAGE_LIMIT environment variable. [Learn more about storage environment variables](docs/environment-variables#storage)', true) ->param('allowedFileExtensions', [], new ArrayList(new Text(64)), 'Allowed file extensions', true) ->param('enabled', true, new Boolean(), 'Is bucket enabled?', true) - ->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_LIMIT_ENCRYPTION) . ' encryption is skipped even if it\'s enabled', true) + ->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER) . ' encryption is skipped even if it\'s enabled', true) ->param('antiVirus', true, new Boolean(), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS) . ' AntiVirus scanning is skipped even if it\'s enabled', true) ->inject('response') ->inject('dbForInternal') @@ -398,8 +420,6 @@ App::put('/v1/storage/buckets/:bucketId') throw new Exception('Bucket not found', 404); } - $read ??= $bucket->getAttribute('$read', []); // By default inherit read permissions - $write ??= $bucket->getAttribute('$write',[]); // By default inherit write permissions $read ??= $bucket->getAttribute('$read', []); // By default inherit read permissions $write ??= $bucket->getAttribute('$write', []); // By default inherit write permissions $maximumFileSize ??= $bucket->getAttribute('maximumFileSize', (int)App::getEnv('_APP_STORAGE_LIMIT', 0)); @@ -409,14 +429,14 @@ App::put('/v1/storage/buckets/:bucketId') $antiVirus ??= $bucket->getAttribute('antiVirus', true); $bucket = $dbForInternal->updateDocument('buckets', $bucket->getId(), $bucket - ->setAttribute('name',$name) - ->setAttribute('$read',$read) - ->setAttribute('$write',$write) - ->setAttribute('maximumFileSize',$maximumFileSize) - ->setAttribute('allowedFileExtensions',$allowedFileExtensions) - ->setAttribute('enabled',$enabled) - ->setAttribute('encryption',$encryption) - ->setAttribute('antiVirus',$antiVirus) + ->setAttribute('name', $name) + ->setAttribute('$read', $read) + ->setAttribute('$write', $write) + ->setAttribute('maximumFileSize', $maximumFileSize) + ->setAttribute('allowedFileExtensions', $allowedFileExtensions) + ->setAttribute('enabled', $enabled) + ->setAttribute('encryption', $encryption) + ->setAttribute('antiVirus', $antiVirus) ); $audits @@ -487,7 +507,7 @@ App::delete('/v1/storage/buckets/:bucketId') }); App::post('/v1/storage/buckets/:bucketId/files') - ->alias('/v1/storage/files',['bucketId' => 'default']) + ->alias('/v1/storage/files', ['bucketId' => 'default']) ->desc('Create File') ->groups(['api', 'storage']) ->label('scope', 'files.write') @@ -524,12 +544,13 @@ App::post('/v1/storage/buckets/:bucketId/files') $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } // Check bucket permissions when enforced - if ($bucket->getAttribute('permission') === 'bucket') { + $permissionBucket = $bucket->getAttribute('permission') === 'bucket'; + if ($permissionBucket) { $validator = new Authorization('write'); if (!$validator->isValid($bucket->getWrite())) { throw new Exception('Unauthorized permissions', 401); @@ -538,14 +559,14 @@ App::post('/v1/storage/buckets/:bucketId/files') $file = $request->getFiles('file'); - /* + /** * Validators */ $allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []); $fileExt = new FileExt($allowedFileExtensions); $maximumFileSize = $bucket->getAttribute('maximumFileSize', 0); - if($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT',0)) { + if ($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT', 0)) { throw new Exception('Error bucket maximum file size is larger than _APP_STORAGE_LIMIT', 500); } @@ -557,114 +578,233 @@ App::post('/v1/storage/buckets/:bucketId/files') } // Make sure we handle a single file and multiple files the same way - $file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; - $file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; - $file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; + $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; + $size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + + $contentRange = $request->getHeader('content-range'); + $fileId = $fileId == 'unique()' ? $dbForInternal->getId() : $fileId; + $chunk = 1; + $chunks = 1; + + if (!empty($contentRange)) { + $start = $request->getContentRangeStart(); + $end = $request->getContentRangeEnd(); + $size = $request->getContentRangeSize(); + $fileId = $request->getHeader('x-appwrite-id', $fileId); + if(is_null($start) || is_null($end) || is_null($size)) { + throw new Exception('Invalid content-range header', 400); + } + + if ($end == $size) { + //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk + $chunks = $chunk = -1; + } else { + // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) + $chunks = (int) ceil($size / ($end + 1 - $start)); + $chunk = (int) ($start / ($end + 1 - $start)); + } + } // Check if file type is allowed (feature for project settings?) - if (!empty($allowedFileExtensions) && !$fileExt->isValid($file['name'])) { + if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) { throw new Exception('File extension not allowed', 400); } - if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit + if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit throw new Exception('File size not allowed', 400); } $device = Storage::getDevice('files'); - if (!$upload->isValid($file['tmp_name'])) { + if (!$upload->isValid($fileTmpName)) { throw new Exception('Invalid file', 403); } // Save to storage - $size = $device->getFileSize($file['tmp_name']); - $path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION)); - $path = $bucket->getId() . '/' . $path; - - if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move' - throw new Exception('Failed moving file', 500); + $size ??= $device->getFileSize($fileTmpName); + $path = $device->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); + $path = str_ireplace($device->getRoot(), $device->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); + + $file = $dbForInternal->getDocument('bucket_' . $bucketId, $fileId); + + if (!$file->isEmpty()) { + $chunks = $file->getAttribute('chunksTotal', 1); + if ($chunk == -1) { + $chunk = $chunks - 1; + } } - $mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption + $chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks); + if (empty($chunksUploaded)) { + throw new Exception('Failed uploading file', 500); + } - if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) { - $antiVirus = new Network(App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), + $read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; + $write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; + if ($chunksUploaded == $chunks) { + if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) { + $antiVirus = new Network(App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'), (int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)); - - if (!$antiVirus->fileScan($path)) { - $device->delete($path); - throw new Exception('Invalid file', 403); - } - } - - // Compression - $data = $device->read($path); - if($size <= APP_LIMIT_COMPRESSION) { - $compressor = new GZIP(); - $data = $compressor->compress($data); - } - - if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) { - $key = App::getEnv('_APP_OPENSSL_KEY_V1'); - $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); - $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); - } - - if (!$device->write($path, $data, $mimeType)) { - throw new Exception('Failed to save file', 500); - } - - $sizeActual = $device->getFileSize($path); - - $data = [ - '$id' => $fileId === 'unique()' ? $dbForInternal->getId() : $fileId, - '$read' => (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? [], // By default set read permissions for user - '$write' => (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? [], // By default set write permissions for user - 'dateCreated' => \time(), - 'bucketId' => $bucket->getId(), - 'name' => $file['name'], - 'path' => $path, - 'signature' => $device->getFileHash($path), - 'mimeType' => $mimeType, - 'sizeOriginal' => $size, - 'sizeActual' => $sizeActual, - 'algorithm' => empty($compressor) ? '' : $compressor->getName(), - 'comment' => '', - 'search' => implode(' ', [$fileId, $file['name'] ?? '',]), - ]; - - if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) { - $data['openSSLVersion'] = '1'; - $data['openSSLCipher'] = OpenSSL::CIPHER_AES_128_GCM; - $data['openSSLTag'] = \bin2hex($tag); - $data['openSSLIV'] = \bin2hex($iv); - } - - try { - if($bucket->getAttribute('permission') === 'bucket') { - $file = Authorization::skip(function() use ($dbForExternal, $bucket, $data) { - return $dbForExternal->createDocument('bucket_' . $bucket->getId(), new Document($data)); - }); - } else { - $file = $dbForExternal->createDocument('bucket_' . $bucket->getId(), new Document($data)); + + if (!$antiVirus->fileScan($path)) { + $device->delete($path); + throw new Exception('Invalid file', 403); + } } - } - catch (StructureException $exception) { - throw new Exception($exception->getMessage(), 400); - } - catch (DuplicateException $exception) { - throw new Exception('Document already exists', 409); - } + $mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption + $data = ''; + // Compression + if ($size <= APP_STORAGE_READ_BUFFER) { + $data = $device->read($path); + $compressor = new GZIP(); + $data = $compressor->compress($data); + } + if ($bucket->getAttribute('encryption', true) && $size <= APP_STORAGE_READ_BUFFER) { + if(empty($data)) { + $data = $device->read($path); + } + $key = App::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag); + } + + if(!empty($data)) { + if (!$device->write($path, $data, $mimeType)) { + throw new Exception('Failed to save file', 500); + } + } + + $sizeActual = $device->getFileSize($path); + + $algorithm = empty($compressor) ? '' : $compressor->getName(); + $fileHash = $device->getFileHash($path); + + if ($bucket->getAttribute('encryption', true) && $size <= APP_STORAGE_READ_BUFFER) { + $openSSLVersion = '1'; + $openSSLCipher = OpenSSL::CIPHER_AES_128_GCM; + $openSSLTag = \bin2hex($tag); + $openSSLIV = \bin2hex($iv); + } + + try { + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => $fileId, + '$read' => $read, + '$write' => $write, + 'dateCreated' => \time(), + 'bucketId' => $bucket->getId(), + 'name' => $fileName, + 'path' => $path, + 'signature' => $fileHash, + 'mimeType' => $mimeType, + 'sizeOriginal' => $size, + 'sizeActual' => $sizeActual, + 'algorithm' => $algorithm, + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'openSSLVersion' => $openSSLVersion, + 'openSSLCipher' => $openSSLCipher, + 'openSSLTag' => $openSSLTag, + 'openSSLIV' => $openSSLIV, + 'search' => implode(' ', [$fileId, $fileName,]), + ]); + if($permissionBucket) { + $file = Authorization::skip(function() use ($dbForInternal, $bucketId, $doc) { + return $dbForInternal->createDocument('bucket_' . $bucketId, $doc); + }); + } else { + $file = $dbForInternal->createDocument('bucket_' . $bucketId, $doc); + } + } else { + $file = $file + ->setAttribute('$read', $read) + ->setAttribute('$write', $write) + ->setAttribute('signature', $fileHash) + ->setAttribute('mimeType', $mimeType) + ->setAttribute('sizeActual', $sizeActual) + ->setAttribute('algorithm', $algorithm) + ->setAttribute('openSSLVersion', $openSSLVersion) + ->setAttribute('openSSLCipher', $openSSLCipher) + ->setAttribute('openSSLTag', $openSSLTag) + ->setAttribute('openSSLIV', $openSSLIV); + + if($permissionBucket) { + $file = Authorization::skip(function() use ($dbForInternal, $bucketId, $fileId, $file) { + return $dbForInternal->updateDocument('bucket_' . $bucketId, $fileId, $file); + }); + } else { + $file = $dbForInternal->updateDocument('bucket_' . $bucketId, $fileId, $file); + } + + } + } + catch (StructureException $exception) { + throw new Exception($exception->getMessage(), 400); + } + catch (DuplicateException $exception) { + throw new Exception('Document already exists', 409); + } + } else { + try { + if ($file->isEmpty()) { + $doc = new Document([ + '$id' => $fileId, + '$read' => $read, + '$write' => $write, + 'dateCreated' => \time(), + 'bucketId' => $bucket->getId(), + 'name' => $fileName, + 'path' => $path, + 'signature' => '', + 'mimeType' => '', + 'sizeOriginal' => $size, + 'sizeActual' => 0, + 'algorithm' => '', + 'comment' => '', + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'search' => implode(' ', [$fileId, $fileName,]), + ]); + if($permissionBucket) { + $file = Authorization::skip(function() use ($dbForInternal, $bucketId, $doc) { + return $dbForInternal->createDocument('bucket_' . $bucketId, $doc); + }); + } else { + $file = $dbForInternal->createDocument('bucket_' . $bucketId, $doc); + } + } else { + $file = $file + ->setAttribute('chunksUploaded', $chunksUploaded); + + if($permissionBucket) { + $file = Authorization::skip(function() use ($dbForInternal, $bucketId, $fileId, $file) { + return $dbForInternal->updateDocument('bucket_' . $bucketId, $fileId, $file); + }); + } else { + $file = $dbForInternal->updateDocument('bucket_' . $bucketId, $fileId, $file); + } + } + } + catch (StructureException $exception) { + throw new Exception($exception->getMessage(), 400); + } + catch (DuplicateException $exception) { + throw new Exception('Document already exists', 409); + } + } $audits ->setParam('event', 'storage.files.create') - ->setParam('resource', 'file/'.$file->getId()) + ->setParam('resource', 'storage/files/' . $file->getId()) ; $usage - ->setParam('storage', $sizeActual) + ->setParam('storage', $sizeActual ?? 0) ->setParam('storage.files.create', 1) ->setParam('bucketId', $bucketId) ; @@ -705,7 +845,7 @@ App::get('/v1/storage/buckets/:bucketId/files') $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } @@ -719,7 +859,7 @@ App::get('/v1/storage/buckets/:bucketId/files') $queries = [new Query('bucketId', Query::TYPE_EQUAL, [$bucketId])]; - if($search) { + if ($search) { $queries[] = [new Query('name', Query::TYPE_SEARCH, [$search])]; } @@ -788,7 +928,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId') $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } @@ -839,8 +979,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->param('borderWidth', 0, new Range(0, 100), 'Preview image border in pixels. Pass an integer between 0 to 100. Defaults to 0.', true) ->param('borderColor', '', new HexColor(), 'Preview image border color. Use a valid HEX color, no # is needed for prefix.', true) ->param('borderRadius', 0, new Range(0, 4000), 'Preview image border radius in pixels. Pass an integer between 0 to 4000.', true) - ->param('opacity', 1, new Range(0,1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true) - ->param('rotation', 0, new Range(0,360), 'Preview image rotation in degrees. Pass an integer between 0 and 360.', true) + ->param('opacity', 1, new Range(0, 1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true) + ->param('rotation', 0, new Range(0, 360), 'Preview image rotation in degrees. Pass an integer between 0 and 360.', true) ->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true) ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true) ->inject('request') @@ -868,7 +1008,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') } $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } @@ -926,8 +1066,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') throw new Exception('File not found', 404); } - $cache = new Cache(new Filesystem(APP_STORAGE_CACHE.'/app-'.$project->getId())); // Limit file number or size - $data = $cache->load($key, 60 * 60 * 24 * 30 * 3 /* 3 months */); + $cache = new Cache(new Filesystem(APP_STORAGE_CACHE . '/app-' . $project->getId())); // Limit file number or size + $data = $cache->load($key, 60 * 60 * 24 * 30 * 3/* 3 months */); if ($data) { $output = (empty($output)) ? $type : $output; @@ -946,7 +1086,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), - App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')), + App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), 0, \hex2bin($file->getAttribute('openSSLIV')), \hex2bin($file->getAttribute('openSSLTag')) @@ -966,11 +1106,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') } if (!empty($background)) { - $image->setBackground('#'.$background); + $image->setBackground('#' . $background); } - - if (!empty($borderWidth) ) { - $image->setBorder($borderWidth, '#'.$borderColor); + + if (!empty($borderWidth)) { + $image->setBorder($borderWidth, '#' . $borderColor); } if (!empty($borderRadius)) { @@ -1016,11 +1156,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ->label('sdk.methodType', 'location') ->param('bucketId', null, new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).') ->param('fileId', '', new UID(), 'File unique ID.') + ->inject('request') ->inject('response') ->inject('dbForInternal') ->inject('dbForExternal') ->inject('usage') - ->action(function ($bucketId, $fileId, $response, $dbForInternal, $dbForExternal, $usage) { + ->action(function ($bucketId, $fileId, $request, $response, $dbForInternal, $dbForExternal, $usage) { + /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Database $dbForExternal */ @@ -1028,7 +1170,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } @@ -1054,26 +1196,10 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') $path = $file->getAttribute('path', ''); - if (!\file_exists($path)) { - throw new Exception('File not found in '.$path, 404); - } - $device = Storage::getDevice('files'); - - $source = $device->read($path); - if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt - $source = OpenSSL::decrypt( - $source, - $file->getAttribute('openSSLCipher'), - App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')), - 0, - \hex2bin($file->getAttribute('openSSLIV')), - \hex2bin($file->getAttribute('openSSLTag')) - ); - } - if(!empty($file->getAttribute('algorithm', ''))) { - $compressor = new GZIP(); - $source = $compressor->decompress($source); + + if (!$device->exists($path)) { + throw new Exception('File not found in ' . $path, 404); } $usage @@ -1081,14 +1207,76 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ->setParam('bucketId', $bucketId) ; - // Response $response ->setContentType($file->getAttribute('mimeType')) - ->addHeader('Content-Disposition', 'attachment; filename="'.$file->getAttribute('name', '').'"') - ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT') // 45 days cache + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache ->addHeader('X-Peak', \memory_get_peak_usage()) - ->send($source) + ->addHeader('Content-Disposition', 'attachment; filename="' . $file->getAttribute('name', '') . '"') ; + + $size = $file->getAttribute('sizeOriginal', 0); + + $rangeHeader = $request->getHeader('range'); + if(!empty($rangeHeader)) { + $start = $request->getRangeStart(); + $end = $request->getRangeEnd(); + $unit = $request->getRangeUnit(); + + if($end == null) { + $end = min(($start + 2000000-1), ($size - 1)); + } + + if($unit != 'bytes' || $start >= $end || $end >= $size) { + throw new Exception('Invalid range', 416); + } + + $response + ->addHeader('Accept-Ranges', 'bytes') + ->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size) + ->addHeader('Content-Length', $end - $start + 1) + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); + } + + $source = ''; + if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt + $source = $device->read($path); + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + if (!empty($file->getAttribute('algorithm', ''))) { + if(empty($source)) { + $source = $device->read($path); + } + $compressor = new GZIP(); + $source = $compressor->decompress($source); + } + + if(!empty($source)) { + if(!empty($rangeHeader)) { + $response->send(substr($source, $start, ($end - $start + 1))); + } + $response->send($source); + } + + if(!empty($rangeHeader)) { + $response->send($device->read($path, $start, ($end - $start + 1))); + } + + if ($size > APP_STORAGE_READ_BUFFER) { + $response->addHeader('Content-Length', $device->getFileSize($path)); + for ($i=0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { + $response->chunk($device->read($path, ($i * MAX_OUTPUT_CHUNK_SIZE), min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE))), (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size); + } + } else { + $response->send($device->read($path)); + } }); App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') @@ -1106,17 +1294,19 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') ->param('bucketId', null, new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') + ->inject('request') ->inject('dbForInternal') ->inject('dbForExternal') ->inject('usage') - ->action(function ($bucketId, $fileId, $response, $dbForInternal, $dbForExternal, $usage) { + ->action(function ($bucketId, $fileId, $response, $request, $dbForInternal, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Swoole\Request $request */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Stats\Stats $usage */ $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } @@ -1145,7 +1335,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $path = $file->getAttribute('path', ''); if (!\file_exists($path)) { - throw new Exception('File not found in '.$path, 404); + throw new Exception('File not found in ' . $path, 404); } $compressor = new GZIP(); @@ -1157,36 +1347,85 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') $contentType = $file->getAttribute('mimeType'); } - $source = $device->read($path); + $response + ->setContentType($contentType) + ->addHeader('Content-Security-Policy', 'script-src none;') + ->addHeader('X-Content-Type-Options', 'nosniff') + ->addHeader('Content-Disposition', 'inline; filename="' . $file->getAttribute('name', '') . '"') + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache + ->addHeader('X-Peak', \memory_get_peak_usage()) + ; + $size = $file->getAttribute('sizeOriginal', 0); + + $rangeHeader = $request->getHeader('range'); + if(!empty($rangeHeader)) { + $start = $request->getRangeStart(); + $end = $request->getRangeEnd(); + $unit = $request->getRangeUnit(); + + if($end == null) { + $end = min(($start + 2000000-1), ($size - 1)); + } + + if($unit != 'bytes' || $start >= $end || $end >= $size) { + throw new Exception('Invalid range', 416); + } + + $response + ->addHeader('Accept-Ranges', 'bytes') + ->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size) + ->addHeader('Content-Length', $end - $start + 1) + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); + } + + $source = ''; if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt + $source = $device->read($path); $source = OpenSSL::decrypt( $source, $file->getAttribute('openSSLCipher'), - App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')), + App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), 0, \hex2bin($file->getAttribute('openSSLIV')), \hex2bin($file->getAttribute('openSSLTag')) ); } - $output = $compressor->decompress($source); - $fileName = $file->getAttribute('name', ''); + if (!empty($file->getAttribute('algorithm', ''))) { + if(empty($source)) { + $source = $device->read($path); + } + $compressor = new GZIP(); + $source = $compressor->decompress($source); + } $usage ->setParam('storage.files.read', 1) ->setParam('bucketId', $bucketId) ; - $response - ->setContentType($contentType) - ->addHeader('Content-Security-Policy', 'script-src none;') - ->addHeader('X-Content-Type-Options', 'nosniff') - ->addHeader('Content-Disposition', 'inline; filename="'.$fileName.'"') - ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT') // 45 days cache - ->addHeader('X-Peak', \memory_get_peak_usage()) - ->send($output) - ; + if(!empty($source)) { + if(!empty($rangeHeader)) { + $response->send(substr($source, $start, ($end - $start + 1))); + } + $response->send($source); + } + + if(!empty($rangeHeader)) { + $response->send($device->read($path, $start, ($end - $start + 1))); + } + + $size = $device->getFileSize($path); + if ($size > APP_STORAGE_READ_BUFFER) { + $response->addHeader('Content-Length', $device->getFileSize($path)); + $chunk = 2000000; // Max chunk of 2 mb + for ($i=0; $i < ceil($size / $chunk); $i++) { + $response->chunk($device->read($path, ($i * $chunk), min($chunk, $size - ($i * $chunk))), (($i + 1) * $chunk) >= $size); + } + } else { + $response->send($device->read($path)); + } }); App::put('/v1/storage/buckets/:bucketId/files/:fileId') @@ -1219,7 +1458,7 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } @@ -1300,7 +1539,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') $bucket = $dbForInternal->getDocument('buckets', $bucketId); - if($bucket->isEmpty()) { + if ($bucket->isEmpty()) { throw new Exception('Bucket not found', 404); } @@ -1337,8 +1576,10 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') if (!$deleted) { throw new Exception('Failed to remove file from DB', 500); } + } else { + throw new Exception('Failed to delete file from device', 500); } - + $audits ->setParam('event', 'storage.files.delete') ->setParam('resource', 'file/'.$file->getId()) diff --git a/app/controllers/general.php b/app/controllers/general.php index 4d2c6b7135..ceb598f0ac 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -332,6 +332,7 @@ App::error(function ($error, $utopia, $request, $response, $layout, $project) { case 404: // Error allowed publicly case 409: // Error allowed publicly case 412: // Error allowed publicly + case 416: // Error allowed publicly case 429: // Error allowed publicly case 501: // Error allowed publicly case 503: // Error allowed publicly diff --git a/app/http.php b/app/http.php index 82b6e73e3e..514427235e 100644 --- a/app/http.php +++ b/app/http.php @@ -19,7 +19,7 @@ use Utopia\Swoole\Request; $http = new Server("0.0.0.0", App::getEnv('PORT', 80)); -$payloadSize = max(4000000 /* 4mb */, App::getEnv('_APP_STORAGE_LIMIT', 10000000 /* 10mb */)); +$payloadSize = 6 * (1024 * 1024); // 6MB $http ->set([ diff --git a/app/init.php b/app/init.php index 455eb4845f..0aa0496d90 100644 --- a/app/init.php +++ b/app/init.php @@ -79,6 +79,7 @@ const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_CACHE = '/storage/cache'; const APP_STORAGE_CERTIFICATES = '/storage/certificates'; const APP_STORAGE_CONFIG = '/storage/config'; +const APP_STORAGE_READ_BUFFER = 20 * (1024 * 1024); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT` const APP_SOCIAL_TWITTER = 'https://twitter.com/appwrite_io'; const APP_SOCIAL_TWITTER_HANDLE = 'appwrite_io'; const APP_SOCIAL_FACEBOOK = 'https://www.facebook.com/appwrite.io'; @@ -119,6 +120,8 @@ const APP_AUTH_TYPE_SESSION = 'Session'; const APP_AUTH_TYPE_JWT = 'JWT'; const APP_AUTH_TYPE_KEY = 'Key'; const APP_AUTH_TYPE_ADMIN = 'Admin'; +// Response related +const MAX_OUTPUT_CHUNK_SIZE = 2*1024*1024; // 2MB $register = new Registry(); diff --git a/composer.json b/composer.json index 8b0d69f1fe..e3d1f162a6 100644 --- a/composer.json +++ b/composer.json @@ -51,8 +51,8 @@ "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", - "utopia-php/swoole": "0.2.*", - "utopia-php/storage": "0.5.*", + "utopia-php/swoole": "0.3.*", + "utopia-php/storage": "dev-feat-large-file-support", "utopia-php/websocket": "0.0.*", "utopia-php/image": "0.5.*", @@ -65,7 +65,20 @@ "adhocore/jwt": "1.1.2", "slickdeals/statsd": "3.1.0" }, - "repositories": [], + "repositories": [ + { + "type": "git", + "url": "https://github.com/utopia-php/storage" + }, + { + "type": "git", + "url": "https://github.com/utopia-php/database" + }, + { + "type": "git", + "url": "https://github.com/utopia-php/framework" + } + ], "require-dev": { "appwrite/sdk-generator": "0.16.0", "phpunit/phpunit": "9.5.6", diff --git a/composer.lock b/composer.lock index 5b09d7195b..e30a37ae8e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fa378feaffc446f557a140035a1c77b6", + "content-hash": "bd77c6854e858381edef57388096a498", "packages": [ { "name": "adhocore/jwt", @@ -2141,15 +2141,9 @@ "version": "0.10.0", "source": { "type": "git", - "url": "https://github.com/utopia-php/database.git", + "url": "https://github.com/utopia-php/database", "reference": "b7c60b0ec769a9050dd2b939b78ff1f5d4fa27e8" }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/b7c60b0ec769a9050dd2b939b78ff1f5d4fa27e8", - "reference": "b7c60b0ec769a9050dd2b939b78ff1f5d4fa27e8", - "shasum": "" - }, "require": { "ext-mongodb": "*", "ext-pdo": "*", @@ -2171,7 +2165,11 @@ "Utopia\\Database\\": "src/Database" } }, - "notification-url": "https://packagist.org/downloads/", + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/Database" + } + }, "license": [ "MIT" ], @@ -2193,10 +2191,6 @@ "upf", "utopia" ], - "support": { - "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.10.0" - }, "time": "2021-10-04T17:23:25+00:00" }, { @@ -2258,15 +2252,9 @@ "version": "0.19.0", "source": { "type": "git", - "url": "https://github.com/utopia-php/framework.git", + "url": "https://github.com/utopia-php/framework", "reference": "c86fc078ef258f3c88d3a25233202267314df3a9" }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/c86fc078ef258f3c88d3a25233202267314df3a9", - "reference": "c86fc078ef258f3c88d3a25233202267314df3a9", - "shasum": "" - }, "require": { "php": ">=7.3.0" }, @@ -2280,7 +2268,6 @@ "Utopia\\": "src/" } }, - "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -2296,10 +2283,6 @@ "php", "upf" ], - "support": { - "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.19.0" - }, "time": "2021-10-08T11:46:20+00:00" }, { @@ -2568,20 +2551,14 @@ }, { "name": "utopia-php/storage", - "version": "0.5.0", + "version": "dev-feat-large-file-support", "source": { "type": "git", - "url": "https://github.com/utopia-php/storage.git", - "reference": "92ae20c7a2ac329f573a58a82dc245134cc63408" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/92ae20c7a2ac329f573a58a82dc245134cc63408", - "reference": "92ae20c7a2ac329f573a58a82dc245134cc63408", - "shasum": "" + "url": "https://github.com/utopia-php/storage", + "reference": "1597e2bd4c1091587d91b46ef2dc2878b9c71721" }, "require": { - "php": ">=7.4", + "php": ">=8.0", "utopia-php/framework": "0.*.*" }, "require-dev": { @@ -2594,7 +2571,6 @@ "Utopia\\Storage\\": "src/Storage" } }, - "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -2612,24 +2588,20 @@ "upf", "utopia" ], - "support": { - "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.5.0" - }, - "time": "2021-04-15T16:43:12+00:00" + "time": "2021-10-04T09:42:56+00:00" }, { "name": "utopia-php/swoole", - "version": "0.2.4", + "version": "0.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/swoole.git", - "reference": "37d8c64b536d6bc7da4f0f5a934a0ec44885abf4" + "reference": "b564dacb13472845f06df158ae5382e8098dfd7a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/swoole/zipball/37d8c64b536d6bc7da4f0f5a934a0ec44885abf4", - "reference": "37d8c64b536d6bc7da4f0f5a934a0ec44885abf4", + "url": "https://api.github.com/repos/utopia-php/swoole/zipball/b564dacb13472845f06df158ae5382e8098dfd7a", + "reference": "b564dacb13472845f06df158ae5382e8098dfd7a", "shasum": "" }, "require": { @@ -2670,9 +2642,9 @@ ], "support": { "issues": "https://github.com/utopia-php/swoole/issues", - "source": "https://github.com/utopia-php/swoole/tree/0.2.4" + "source": "https://github.com/utopia-php/swoole/tree/0.3.1" }, - "time": "2021-06-22T10:49:24+00:00" + "time": "2021-07-27T09:28:10+00:00" }, { "name": "utopia-php/system", @@ -5423,7 +5395,6 @@ "type": "github" } ], - "abandoned": true, "time": "2020-09-28T06:45:17+00:00" }, { @@ -6508,7 +6479,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/storage": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docs/references/storage/create-file.md b/docs/references/storage/create-file.md index 7fd6c5a492..2d2059bfb5 100644 --- a/docs/references/storage/create-file.md +++ b/docs/references/storage/create-file.md @@ -1 +1,7 @@ -Create a new file. The user who creates the file will automatically be assigned to read and write access unless he has passed custom values for read and write arguments. \ No newline at end of file +Create a new file. The user who creates the file will automatically be assigned to read and write access unless they have passed custom values for read and write arguments. + +Larger files should be uploaded using multiple requests with the [content-range](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range) header to send a partial request with a maximum supported chunk of `5MB`. The `content-range` header values should always be in bytes. + +When the first request is sent, the server will return the **File** object, and the subsequent part request must include the file's **id** in `x-appwrite-upload-id` header to allow the server to know the partial upload is for the existing file and not for a new one. + +If you're creating a new file using one of the Appwrite SDKs, the entire chunking logic will be managed by the SDK internally. diff --git a/src/Appwrite/Utopia/Response/Model/File.php b/src/Appwrite/Utopia/Response/Model/File.php index 2aebf1f789..ce9eae2248 100644 --- a/src/Appwrite/Utopia/Response/Model/File.php +++ b/src/Appwrite/Utopia/Response/Model/File.php @@ -66,6 +66,18 @@ class File extends Model 'default' => 0, 'example' => 17890, ]) + ->addRule('chunksTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of chunks available', + 'default' => 0, + 'example' => 17890, + ]) + ->addRule('chunksUploaded', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of chunks uploaded', + 'default' => 0, + 'example' => 17890, + ]) ; } diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 0291de1725..a206b477df 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -292,6 +292,41 @@ class FunctionsCustomServerTest extends Scope $this->assertIsInt($tag['body']['dateCreated']); $this->assertEquals('php index.php', $tag['body']['command']); $this->assertGreaterThan(10000, $tag['body']['size']); + + /** + * Test for Large Code File SUCCESS + */ + $source = realpath(__DIR__ . '/../../../resources/functions/php-large.tar.gz'); + $chunkSize = 5*1024*1024; + $handle = @fopen($source, "rb"); + $mimeType = 'application/x-gzip'; + $counter = 0; + $size = filesize($source); + $headers = [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'] + ]; + $id = ''; + while (!feof($handle)) { + $curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'php-large-fx.tar.gz'); + $headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size; + if(!empty($id)) { + $headers['x-appwrite-id'] = $id; + } + $largeTag = $this->client->call(Client::METHOD_POST, '/functions/'.$data['functionId'].'/tags', array_merge($headers, $this->getHeaders()), [ + 'command' => 'php index.php', + 'code' => $curlFile, + ]); + $counter++; + $id = $largeTag['body']['$id']; + } + @fclose($handle); + + $this->assertEquals(201, $largeTag['headers']['status-code']); + $this->assertNotEmpty($largeTag['body']['$id']); + $this->assertIsInt($largeTag['body']['dateCreated']); + $this->assertEquals('php index.php', $largeTag['body']['command']); + $this->assertGreaterThan(10000, $largeTag['body']['size']); /** * Test for FAILURE @@ -342,9 +377,9 @@ class FunctionsCustomServerTest extends Scope ], $this->getHeaders())); $this->assertEquals($function['headers']['status-code'], 200); - $this->assertEquals($function['body']['sum'], 1); + $this->assertEquals($function['body']['sum'], 2); $this->assertIsArray($function['body']['tags']); - $this->assertCount(1, $function['body']['tags']); + $this->assertCount(2, $function['body']['tags']); /** * Test search queries @@ -357,9 +392,9 @@ class FunctionsCustomServerTest extends Scope ])); $this->assertEquals($function['headers']['status-code'], 200); - $this->assertEquals($function['body']['sum'], 1); + $this->assertEquals($function['body']['sum'], 2); $this->assertIsArray($function['body']['tags']); - $this->assertCount(1, $function['body']['tags']); + $this->assertCount(2, $function['body']['tags']); $this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']); $function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([ @@ -370,9 +405,9 @@ class FunctionsCustomServerTest extends Scope ])); $this->assertEquals($function['headers']['status-code'], 200); - $this->assertEquals($function['body']['sum'], 1); + $this->assertEquals($function['body']['sum'], 2); $this->assertIsArray($function['body']['tags']); - $this->assertCount(1, $function['body']['tags']); + $this->assertCount(2, $function['body']['tags']); $this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']); $function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([ @@ -383,9 +418,9 @@ class FunctionsCustomServerTest extends Scope ])); $this->assertEquals($function['headers']['status-code'], 200); - $this->assertEquals($function['body']['sum'], 1); + $this->assertEquals($function['body']['sum'], 2); $this->assertIsArray($function['body']['tags']); - $this->assertCount(1, $function['body']['tags']); + $this->assertCount(2, $function['body']['tags']); $this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']); return $data; diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 6b765291ff..9606fd3775 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -65,24 +65,48 @@ trait StorageBase ]); $this->assertEquals(201, $bucket2['headers']['status-code']); $this->assertNotEmpty($bucket2['body']['$id']); + + /** + * Chunked Upload + */ - $file2 = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket2['body']['$id'] . '/files', array_merge([ + $source = __DIR__ . "/../../../resources/disk-a/large-file.mp4"; + $totalSize = \filesize($source); + $chunkSize = 5*1024*1024; + $handle = @fopen($source, "rb"); + $fileId = 'unique()'; + $mimeType = mime_content_type($source); + $counter = 0; + $size = filesize($source); + $headers = [ 'content-type' => 'multipart/form-data', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), [ - 'fileId' => 'unique()', - 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4'), 'video/mp4', 'large-file.mp4'), - 'read' => ['role:all'], - 'write' => ['role:all'], - ]); - - $this->assertEquals(201, $file2['headers']['status-code']); - $this->assertNotEmpty($file2['body']['$id']); - $this->assertIsInt($file2['body']['dateCreated']); - $this->assertEquals('large-file.mp4', $file2['body']['name']); - $this->assertEquals('video/mp4', $file2['body']['mimeType']); - $this->assertEquals(23660615, $file2['body']['sizeOriginal']); - $this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $file2['body']['signature']); // should validate that the file is not encrypted + 'x-appwrite-project' => $this->getProject()['$id'] + ]; + $id = ''; + while (!feof($handle)) { + $curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'large-file.mp4'); + $headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size; + if(!empty($id)) { + $headers['x-appwrite-id'] = $id; + } + $largeFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket2['body']['$id'] . '/files', array_merge($headers, $this->getHeaders()), [ + 'fileId' => $fileId, + 'file' => $curlFile, + 'read' => ['role:all'], + 'write' => ['role:all'], + ]); + $counter++; + $id = $largeFile['body']['$id']; + } + @fclose($handle); + + $this->assertEquals(201, $largeFile['headers']['status-code']); + $this->assertNotEmpty($largeFile['body']['$id']); + $this->assertIsInt($largeFile['body']['dateCreated']); + $this->assertEquals('large-file.mp4', $largeFile['body']['name']); + $this->assertEquals('video/mp4', $largeFile['body']['mimeType']); + $this->assertEquals($totalSize, $largeFile['body']['sizeOriginal']); + $this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $largeFile['body']['signature']); // should validate that the file is not encrypted /** * Test for FAILURE unknown Bucket @@ -133,7 +157,7 @@ trait StorageBase $this->assertEquals(400, $res['headers']['status-code']); $this->assertEquals('File extension not allowed', $res['body']['message']); - return ['bucketId' => $bucketId, 'fileId' => $file['body']['$id'], 'largeFileId' => $file2['body']['$id'], 'largeBucketId' => $bucket2['body']['$id']]; + return ['bucketId' => $bucketId, 'fileId' => $file['body']['$id'], 'largeFileId' => $largeFile['body']['$id'], 'largeBucketId' => $bucket2['body']['$id']]; } /** @@ -263,6 +287,49 @@ trait StorageBase $this->assertEquals('image/png', $file5['headers']['content-type']); $this->assertNotEmpty($file5['body']); + // Test ranged download + $file51 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'Range' => 'bytes=0-99', + ], $this->getHeaders())); + + $path = __DIR__ . '/../../../resources/logo.png'; + $originalChunk = \file_get_contents($path, false, null, 0, 100); + + $this->assertEquals(206, $file51['headers']['status-code']); + $this->assertEquals('attachment; filename="logo.png"', $file51['headers']['content-disposition']); + $this->assertEquals('image/png', $file51['headers']['content-type']); + $this->assertNotEmpty($file51['body']); + $this->assertEquals($originalChunk, $file51['body']); + + // Test ranged download - with invalid range + $file52 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'Range' => 'bytes=0-', + ], $this->getHeaders())); + + $this->assertEquals(206, $file52['headers']['status-code']); + + // Test ranged download - with invalid range + $file53 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'Range' => 'bytes=988', + ], $this->getHeaders())); + + $this->assertEquals(416, $file53['headers']['status-code']); + + // Test ranged download - with invalid range + $file54 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'Range' => 'bytes=-988', + ], $this->getHeaders())); + + $this->assertEquals(416, $file54['headers']['status-code']); + $file6 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/view', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], diff --git a/tests/resources/functions/php-large.tar.gz b/tests/resources/functions/php-large.tar.gz new file mode 100644 index 0000000000..7ca5857029 Binary files /dev/null and b/tests/resources/functions/php-large.tar.gz differ