Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat-large-file' into feat-s3-integration

Browse Source
This commit is contained in:
Damodar Lohani
2021-11-14 18:13:43 +05:45
parent 2d63458b4f 692cbe1899
commit e337fe3aca
13 changed files with 703 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/config/collections2.php
+22
View File
@@ -1758,6 +1758,28 @@ $collections = [
'array' => false,
'filters' => [],
],
[
'$id' => 'chunksTotal',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'chunksUploaded',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'search',
'type' => Database::VAR_STRING,
app/controllers/api/functions.php
+80 -20
View File
@@ -431,42 +431,102 @@ App::post('/v1/functions/:functionId/tags')
}
// Make sure we handle a single file and multiple files the same way
$file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
$fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
if (!$fileExt->isValid($file['name'])) { // Check if file type is allowed
throw new Exception('File type not allowed', 400);
}
if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit
$contentRange = $request->getHeader('content-range');
$tagId = $dbForInternal->getId();
$chunk = 1;
$chunks = 1;
if (!empty($contentRange)) {
$start = $request->getContentRangeStart();
$end = $request->getContentRangeEnd();
$size = $request->getContentRangeSize();
$tagId = $request->getHeader('x-appwrite-id', $tagId);
if(is_null($start) || is_null($end) || is_null($size)) {
throw new Exception('Invalid content-range header', 400);
}
if ($end == $size) {
//if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk
$chunks = $chunk = -1;
} else {
// Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart)
$chunks = (int) ceil($size / ($end + 1 - $start));
$chunk = (int) ($start / ($end + 1 - $start));
}
}
if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit
throw new Exception('File size not allowed', 400);
}
if (!$upload->isValid($file['tmp_name'])) {
if (!$upload->isValid($fileTmpName)) {
throw new Exception('Invalid file', 403);
}
// Save to storage
$size = $device->getFileSize($file['tmp_name']);
$path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION));
$size ??= $device->getFileSize($fileTmpName);
$path = $device->getPath($tagId.'.'.\pathinfo($fileName, PATHINFO_EXTENSION));
if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move'
$tag = $dbForInternal->getDocument('tags', $tagId);
if(!$tag->isEmpty()) {
$chunks = $tag->getAttribute('chunksTotal', 1);
if($chunk == -1) {
$chunk = $chunks - 1;
}
}
$chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks);
if (empty($chunksUploaded)) {
throw new Exception('Failed moving file', 500);
}
$tagId = $dbForInternal->getId();
$tag = $dbForInternal->createDocument('tags', new Document([
'$id' => $tagId,
'$read' => [],
'$write' => [],
'functionId' => $function->getId(),
'dateCreated' => time(),
'command' => $command,
'path' => $path,
'size' => $size,
'search' => implode(' ', [$tagId, $command]),
]));
if($chunksUploaded == $chunks) {
$size = $device->getFileSize($path);
if ($tag->isEmpty()) {
$tag = $dbForInternal->createDocument('tags', new Document([
'$id' => $tagId,
'$read' => [],
'$write' => [],
'functionId' => $function->getId(),
'dateCreated' => time(),
'command' => $command,
'path' => $path,
'size' => $size,
'search' => implode(' ', [$tagId, $command]),
]));
} else {
$tag = $dbForInternal->updateDocument('tags', $tagId, $tag->setAttribute('size', $size));
}
} else {
if($tag->isEmpty()) {
$tag = $dbForInternal->createDocument('tags', new Document([
'$id' => $tagId,
'$read' => [],
'$write' => [],
'functionId' => $function->getId(),
'dateCreated' => time(),
'command' => $command,
'path' => $path,
'size' => 0,
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'search' => implode(' ', [$tagId, $command]),
]));
} else {
$tag = $dbForInternal->updateDocument('tags', $tagId, $tag->setAttribute('chunksUploaded', $chunksUploaded));
}
}
$usage
->setParam('storage', $tag->getAttribute('size', 0))
app/controllers/api/storage.php
+412 -171
View File
@@ -1,19 +1,13 @@
<?php
use Utopia\App;
use Utopia\Exception;
use Utopia\Validator\ArrayList;
use Utopia\Validator\WhiteList;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\Boolean;
use Utopia\Validator\HexColor;
use Utopia\Cache\Cache;
use Utopia\Cache\Adapter\Filesystem;
use Appwrite\ClamAV\Network;
use Utopia\Database\Validator\Authorization;
use Appwrite\Database\Validator\CustomId;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Exception;
use Utopia\Database\Validator\UID;
use Utopia\Storage\Storage;
use Utopia\Storage\Validator\File;
@@ -23,11 +17,17 @@ use Utopia\Storage\Compression\Algorithms\GZIP;
use Utopia\Image\Image;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\Utopia\Response;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Exception\Duplicate;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Boolean;
use Utopia\Validator\HexColor;
use Utopia\Validator\Integer;
use Utopia\Database\Query;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
use Utopia\Validator\Range;
use Utopia\Database\Validator\Permissions;
use Utopia\Storage\Validator\FileExt;
use Utopia\Database\Exception\Duplicate as DuplicateException;
@@ -54,7 +54,7 @@ App::post('/v1/storage/buckets')
->param('allowedFileExtensions', [], new ArrayList(new Text(64)), 'Allowed file extensions', true)
->param('enabled', true, new Boolean(), 'Is bucket enabled?', true)
->param('adapter', 'local', new WhiteList(['local']), 'Storage adapter.', true)
->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_LIMIT_ENCRYPTION) . ' encryption is skipped even if it\'s enabled', true)
->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER) . ' encryption is skipped even if it\'s enabled', true)
->param('antiVirus', true, new Boolean(), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS) . ' AntiVirus scanning is skipped even if it\'s enabled', true)
->inject('response')
->inject('dbForInternal')
@@ -226,6 +226,28 @@ App::post('/v1/storage/buckets')
'array' => false,
'filters' => [],
]),
new Document([
'$id' => 'chunksTotal',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
]),
new Document([
'$id' => 'chunksUploaded',
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
]),
new Document([
'$id' => 'search',
'type' => Database::VAR_STRING,
@@ -238,6 +260,13 @@ App::post('/v1/storage/buckets')
'filters' => [],
]),
], [
new Document([
'$id' => '_key_search',
'type' => Database::INDEX_FULLTEXT,
'attributes' => ['search'],
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
]),
new Document([
'$id' => '_key_bucket',
'type' => Database::INDEX_KEY,
@@ -245,13 +274,6 @@ App::post('/v1/storage/buckets')
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
]),
new Document([
'$id' => '_key_search',
'type' => Database::INDEX_FULLTEXT,
'attributes' => ['search'],
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
],),
]);
$bucket = $dbForInternal->createDocument('buckets', new Document([
@@ -380,7 +402,7 @@ App::put('/v1/storage/buckets/:bucketId')
->param('maximumFileSize', null, new Integer(), 'Maximum file size allowed in bytes. Maximum allowed value is ' . App::getEnv('_APP_STORAGE_LIMIT', 0) . '. For self hosted version you can change the limit by changing _APP_STORAGE_LIMIT environment variable. [Learn more about storage environment variables](docs/environment-variables#storage)', true)
->param('allowedFileExtensions', [], new ArrayList(new Text(64)), 'Allowed file extensions', true)
->param('enabled', true, new Boolean(), 'Is bucket enabled?', true)
->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_LIMIT_ENCRYPTION) . ' encryption is skipped even if it\'s enabled', true)
->param('encryption', true, new Boolean(), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER) . ' encryption is skipped even if it\'s enabled', true)
->param('antiVirus', true, new Boolean(), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS) . ' AntiVirus scanning is skipped even if it\'s enabled', true)
->inject('response')
->inject('dbForInternal')
@@ -398,8 +420,6 @@ App::put('/v1/storage/buckets/:bucketId')
throw new Exception('Bucket not found', 404);
}
$read ??= $bucket->getAttribute('$read', []); // By default inherit read permissions
$write ??= $bucket->getAttribute('$write',[]); // By default inherit write permissions
$read ??= $bucket->getAttribute('$read', []); // By default inherit read permissions
$write ??= $bucket->getAttribute('$write', []); // By default inherit write permissions
$maximumFileSize ??= $bucket->getAttribute('maximumFileSize', (int)App::getEnv('_APP_STORAGE_LIMIT', 0));
@@ -409,14 +429,14 @@ App::put('/v1/storage/buckets/:bucketId')
$antiVirus ??= $bucket->getAttribute('antiVirus', true);
$bucket = $dbForInternal->updateDocument('buckets', $bucket->getId(), $bucket
->setAttribute('name',$name)
->setAttribute('$read',$read)
->setAttribute('$write',$write)
->setAttribute('maximumFileSize',$maximumFileSize)
->setAttribute('allowedFileExtensions',$allowedFileExtensions)
->setAttribute('enabled',$enabled)
->setAttribute('encryption',$encryption)
->setAttribute('antiVirus',$antiVirus)
->setAttribute('name', $name)
->setAttribute('$read', $read)
->setAttribute('$write', $write)
->setAttribute('maximumFileSize', $maximumFileSize)
->setAttribute('allowedFileExtensions', $allowedFileExtensions)
->setAttribute('enabled', $enabled)
->setAttribute('encryption', $encryption)
->setAttribute('antiVirus', $antiVirus)
);
$audits
@@ -487,7 +507,7 @@ App::delete('/v1/storage/buckets/:bucketId')
});
App::post('/v1/storage/buckets/:bucketId/files')
->alias('/v1/storage/files',['bucketId' => 'default'])
->alias('/v1/storage/files', ['bucketId' => 'default'])
->desc('Create File')
->groups(['api', 'storage'])
->label('scope', 'files.write')
@@ -524,12 +544,13 @@ App::post('/v1/storage/buckets/:bucketId/files')
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
// Check bucket permissions when enforced
if ($bucket->getAttribute('permission') === 'bucket') {
$permissionBucket = $bucket->getAttribute('permission') === 'bucket';
if ($permissionBucket) {
$validator = new Authorization('write');
if (!$validator->isValid($bucket->getWrite())) {
throw new Exception('Unauthorized permissions', 401);
@@ -538,14 +559,14 @@ App::post('/v1/storage/buckets/:bucketId/files')
$file = $request->getFiles('file');
/*
/**
* Validators
*/
$allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []);
$fileExt = new FileExt($allowedFileExtensions);
$maximumFileSize = $bucket->getAttribute('maximumFileSize', 0);
if($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT',0)) {
if ($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT', 0)) {
throw new Exception('Error bucket maximum file size is larger than _APP_STORAGE_LIMIT', 500);
}
@@ -557,114 +578,233 @@ App::post('/v1/storage/buckets/:bucketId/files')
}
// Make sure we handle a single file and multiple files the same way
$file['name'] = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$file['tmp_name'] = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$file['size'] = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
$fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
$size = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
$contentRange = $request->getHeader('content-range');
$fileId = $fileId == 'unique()' ? $dbForInternal->getId() : $fileId;
$chunk = 1;
$chunks = 1;
if (!empty($contentRange)) {
$start = $request->getContentRangeStart();
$end = $request->getContentRangeEnd();
$size = $request->getContentRangeSize();
$fileId = $request->getHeader('x-appwrite-id', $fileId);
if(is_null($start) || is_null($end) || is_null($size)) {
throw new Exception('Invalid content-range header', 400);
}
if ($end == $size) {
//if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk
$chunks = $chunk = -1;
} else {
// Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart)
$chunks = (int) ceil($size / ($end + 1 - $start));
$chunk = (int) ($start / ($end + 1 - $start));
}
}
// Check if file type is allowed (feature for project settings?)
if (!empty($allowedFileExtensions) && !$fileExt->isValid($file['name'])) {
if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) {
throw new Exception('File extension not allowed', 400);
}
if (!$fileSize->isValid($file['size'])) { // Check if file size is exceeding allowed limit
if (!$fileSize->isValid($size)) { // Check if file size is exceeding allowed limit
throw new Exception('File size not allowed', 400);
}
$device = Storage::getDevice('files');
if (!$upload->isValid($file['tmp_name'])) {
if (!$upload->isValid($fileTmpName)) {
throw new Exception('Invalid file', 403);
}
// Save to storage
$size = $device->getFileSize($file['tmp_name']);
$path = $device->getPath(\uniqid().'.'.\pathinfo($file['name'], PATHINFO_EXTENSION));
$path = $bucket->getId() . '/' . $path;
if (!$device->upload($file['tmp_name'], $path)) { // TODO deprecate 'upload' and replace with 'move'
throw new Exception('Failed moving file', 500);
$size ??= $device->getFileSize($fileTmpName);
$path = $device->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
$path = str_ireplace($device->getRoot(), $device->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path);
$file = $dbForInternal->getDocument('bucket_' . $bucketId, $fileId);
if (!$file->isEmpty()) {
$chunks = $file->getAttribute('chunksTotal', 1);
if ($chunk == -1) {
$chunk = $chunks - 1;
}
}
$mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption
$chunksUploaded = $device->upload($fileTmpName, $path, $chunk, $chunks);
if (empty($chunksUploaded)) {
throw new Exception('Failed uploading file', 500);
}
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) {
$antiVirus = new Network(App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
$read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? [];
$write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? [];
if ($chunksUploaded == $chunks) {
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antiVirus', true) && $size <= APP_LIMIT_ANTIVIRUS) {
$antiVirus = new Network(App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
(int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310));
if (!$antiVirus->fileScan($path)) {
$device->delete($path);
throw new Exception('Invalid file', 403);
}
}
// Compression
$data = $device->read($path);
if($size <= APP_LIMIT_COMPRESSION) {
$compressor = new GZIP();
$data = $compressor->compress($data);
}
if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) {
$key = App::getEnv('_APP_OPENSSL_KEY_V1');
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
$data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag);
}
if (!$device->write($path, $data, $mimeType)) {
throw new Exception('Failed to save file', 500);
}
$sizeActual = $device->getFileSize($path);
$data = [
'$id' => $fileId === 'unique()' ? $dbForInternal->getId() : $fileId,
'$read' => (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? [], // By default set read permissions for user
'$write' => (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? [], // By default set write permissions for user
'dateCreated' => \time(),
'bucketId' => $bucket->getId(),
'name' => $file['name'],
'path' => $path,
'signature' => $device->getFileHash($path),
'mimeType' => $mimeType,
'sizeOriginal' => $size,
'sizeActual' => $sizeActual,
'algorithm' => empty($compressor) ? '' : $compressor->getName(),
'comment' => '',
'search' => implode(' ', [$fileId, $file['name'] ?? '',]),
];
if($bucket->getAttribute('encryption', true) && $size <= APP_LIMIT_ENCRYPTION) {
$data['openSSLVersion'] = '1';
$data['openSSLCipher'] = OpenSSL::CIPHER_AES_128_GCM;
$data['openSSLTag'] = \bin2hex($tag);
$data['openSSLIV'] = \bin2hex($iv);
}
try {
if($bucket->getAttribute('permission') === 'bucket') {
$file = Authorization::skip(function() use ($dbForExternal, $bucket, $data) {
return $dbForExternal->createDocument('bucket_' . $bucket->getId(), new Document($data));
});
} else {
$file = $dbForExternal->createDocument('bucket_' . $bucket->getId(), new Document($data));
if (!$antiVirus->fileScan($path)) {
$device->delete($path);
throw new Exception('Invalid file', 403);
}
}
}
catch (StructureException $exception) {
throw new Exception($exception->getMessage(), 400);
}
catch (DuplicateException $exception) {
throw new Exception('Document already exists', 409);
}
$mimeType = $device->getFileMimeType($path); // Get mime-type before compression and encryption
$data = '';
// Compression
if ($size <= APP_STORAGE_READ_BUFFER) {
$data = $device->read($path);
$compressor = new GZIP();
$data = $compressor->compress($data);
}
if ($bucket->getAttribute('encryption', true) && $size <= APP_STORAGE_READ_BUFFER) {
if(empty($data)) {
$data = $device->read($path);
}
$key = App::getEnv('_APP_OPENSSL_KEY_V1');
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
$data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag);
}
if(!empty($data)) {
if (!$device->write($path, $data, $mimeType)) {
throw new Exception('Failed to save file', 500);
}
}
$sizeActual = $device->getFileSize($path);
$algorithm = empty($compressor) ? '' : $compressor->getName();
$fileHash = $device->getFileHash($path);
if ($bucket->getAttribute('encryption', true) && $size <= APP_STORAGE_READ_BUFFER) {
$openSSLVersion = '1';
$openSSLCipher = OpenSSL::CIPHER_AES_128_GCM;
$openSSLTag = \bin2hex($tag);
$openSSLIV = \bin2hex($iv);
}
try {
if ($file->isEmpty()) {
$doc = new Document([
'$id' => $fileId,
'$read' => $read,
'$write' => $write,
'dateCreated' => \time(),
'bucketId' => $bucket->getId(),
'name' => $fileName,
'path' => $path,
'signature' => $fileHash,
'mimeType' => $mimeType,
'sizeOriginal' => $size,
'sizeActual' => $sizeActual,
'algorithm' => $algorithm,
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'openSSLVersion' => $openSSLVersion,
'openSSLCipher' => $openSSLCipher,
'openSSLTag' => $openSSLTag,
'openSSLIV' => $openSSLIV,
'search' => implode(' ', [$fileId, $fileName,]),
]);
if($permissionBucket) {
$file = Authorization::skip(function() use ($dbForInternal, $bucketId, $doc) {
return $dbForInternal->createDocument('bucket_' . $bucketId, $doc);
});
} else {
$file = $dbForInternal->createDocument('bucket_' . $bucketId, $doc);
}
} else {
$file = $file
->setAttribute('$read', $read)
->setAttribute('$write', $write)
->setAttribute('signature', $fileHash)
->setAttribute('mimeType', $mimeType)
->setAttribute('sizeActual', $sizeActual)
->setAttribute('algorithm', $algorithm)
->setAttribute('openSSLVersion', $openSSLVersion)
->setAttribute('openSSLCipher', $openSSLCipher)
->setAttribute('openSSLTag', $openSSLTag)
->setAttribute('openSSLIV', $openSSLIV);
if($permissionBucket) {
$file = Authorization::skip(function() use ($dbForInternal, $bucketId, $fileId, $file) {
return $dbForInternal->updateDocument('bucket_' . $bucketId, $fileId, $file);
});
} else {
$file = $dbForInternal->updateDocument('bucket_' . $bucketId, $fileId, $file);
}
}
}
catch (StructureException $exception) {
throw new Exception($exception->getMessage(), 400);
}
catch (DuplicateException $exception) {
throw new Exception('Document already exists', 409);
}
} else {
try {
if ($file->isEmpty()) {
$doc = new Document([
'$id' => $fileId,
'$read' => $read,
'$write' => $write,
'dateCreated' => \time(),
'bucketId' => $bucket->getId(),
'name' => $fileName,
'path' => $path,
'signature' => '',
'mimeType' => '',
'sizeOriginal' => $size,
'sizeActual' => 0,
'algorithm' => '',
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'search' => implode(' ', [$fileId, $fileName,]),
]);
if($permissionBucket) {
$file = Authorization::skip(function() use ($dbForInternal, $bucketId, $doc) {
return $dbForInternal->createDocument('bucket_' . $bucketId, $doc);
});
} else {
$file = $dbForInternal->createDocument('bucket_' . $bucketId, $doc);
}
} else {
$file = $file
->setAttribute('chunksUploaded', $chunksUploaded);
if($permissionBucket) {
$file = Authorization::skip(function() use ($dbForInternal, $bucketId, $fileId, $file) {
return $dbForInternal->updateDocument('bucket_' . $bucketId, $fileId, $file);
});
} else {
$file = $dbForInternal->updateDocument('bucket_' . $bucketId, $fileId, $file);
}
}
}
catch (StructureException $exception) {
throw new Exception($exception->getMessage(), 400);
}
catch (DuplicateException $exception) {
throw new Exception('Document already exists', 409);
}
}
$audits
->setParam('event', 'storage.files.create')
->setParam('resource', 'file/'.$file->getId())
->setParam('resource', 'storage/files/' . $file->getId())
;
$usage
->setParam('storage', $sizeActual)
->setParam('storage', $sizeActual ?? 0)
->setParam('storage.files.create', 1)
->setParam('bucketId', $bucketId)
;
@@ -705,7 +845,7 @@ App::get('/v1/storage/buckets/:bucketId/files')
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
@@ -719,7 +859,7 @@ App::get('/v1/storage/buckets/:bucketId/files')
$queries = [new Query('bucketId', Query::TYPE_EQUAL, [$bucketId])];
if($search) {
if ($search) {
$queries[] = [new Query('name', Query::TYPE_SEARCH, [$search])];
}
@@ -788,7 +928,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId')
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
@@ -839,8 +979,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
->param('borderWidth', 0, new Range(0, 100), 'Preview image border in pixels. Pass an integer between 0 to 100. Defaults to 0.', true)
->param('borderColor', '', new HexColor(), 'Preview image border color. Use a valid HEX color, no # is needed for prefix.', true)
->param('borderRadius', 0, new Range(0, 4000), 'Preview image border radius in pixels. Pass an integer between 0 to 4000.', true)
->param('opacity', 1, new Range(0,1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true)
->param('rotation', 0, new Range(0,360), 'Preview image rotation in degrees. Pass an integer between 0 and 360.', true)
->param('opacity', 1, new Range(0, 1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true)
->param('rotation', 0, new Range(0, 360), 'Preview image rotation in degrees. Pass an integer between 0 and 360.', true)
->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true)
->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true)
->inject('request')
@@ -868,7 +1008,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
}
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
@@ -926,8 +1066,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
throw new Exception('File not found', 404);
}
$cache = new Cache(new Filesystem(APP_STORAGE_CACHE.'/app-'.$project->getId())); // Limit file number or size
$data = $cache->load($key, 60 * 60 * 24 * 30 * 3 /* 3 months */);
$cache = new Cache(new Filesystem(APP_STORAGE_CACHE . '/app-' . $project->getId())); // Limit file number or size
$data = $cache->load($key, 60 * 60 * 24 * 30 * 3/* 3 months */);
if ($data) {
$output = (empty($output)) ? $type : $output;
@@ -946,7 +1086,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')),
App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
0,
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
@@ -966,11 +1106,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
}
if (!empty($background)) {
$image->setBackground('#'.$background);
$image->setBackground('#' . $background);
}
if (!empty($borderWidth) ) {
$image->setBorder($borderWidth, '#'.$borderColor);
if (!empty($borderWidth)) {
$image->setBorder($borderWidth, '#' . $borderColor);
}
if (!empty($borderRadius)) {
@@ -1016,11 +1156,13 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
->label('sdk.methodType', 'location')
->param('bucketId', null, new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).')
->param('fileId', '', new UID(), 'File unique ID.')
->inject('request')
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('usage')
->action(function ($bucketId, $fileId, $response, $dbForInternal, $dbForExternal, $usage) {
->action(function ($bucketId, $fileId, $request, $response, $dbForInternal, $dbForExternal, $usage) {
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Database\Database $dbForExternal */
@@ -1028,7 +1170,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
@@ -1054,26 +1196,10 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
$path = $file->getAttribute('path', '');
if (!\file_exists($path)) {
throw new Exception('File not found in '.$path, 404);
}
$device = Storage::getDevice('files');
$source = $device->read($path);
if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')),
0,
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
);
}
if(!empty($file->getAttribute('algorithm', ''))) {
$compressor = new GZIP();
$source = $compressor->decompress($source);
if (!$device->exists($path)) {
throw new Exception('File not found in ' . $path, 404);
}
$usage
@@ -1081,14 +1207,76 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
->setParam('bucketId', $bucketId)
;
// Response
$response
->setContentType($file->getAttribute('mimeType'))
->addHeader('Content-Disposition', 'attachment; filename="'.$file->getAttribute('name', '').'"')
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT') // 45 days cache
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->addHeader('X-Peak', \memory_get_peak_usage())
->send($source)
->addHeader('Content-Disposition', 'attachment; filename="' . $file->getAttribute('name', '') . '"')
;
$size = $file->getAttribute('sizeOriginal', 0);
$rangeHeader = $request->getHeader('range');
if(!empty($rangeHeader)) {
$start = $request->getRangeStart();
$end = $request->getRangeEnd();
$unit = $request->getRangeUnit();
if($end == null) {
$end = min(($start + 2000000-1), ($size - 1));
}
if($unit != 'bytes' || $start >= $end || $end >= $size) {
throw new Exception('Invalid range', 416);
}
$response
->addHeader('Accept-Ranges', 'bytes')
->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size)
->addHeader('Content-Length', $end - $start + 1)
->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT);
}
$source = '';
if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt
$source = $device->read($path);
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
0,
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
);
}
if (!empty($file->getAttribute('algorithm', ''))) {
if(empty($source)) {
$source = $device->read($path);
}
$compressor = new GZIP();
$source = $compressor->decompress($source);
}
if(!empty($source)) {
if(!empty($rangeHeader)) {
$response->send(substr($source, $start, ($end - $start + 1)));
}
$response->send($source);
}
if(!empty($rangeHeader)) {
$response->send($device->read($path, $start, ($end - $start + 1)));
}
if ($size > APP_STORAGE_READ_BUFFER) {
$response->addHeader('Content-Length', $device->getFileSize($path));
for ($i=0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) {
$response->chunk($device->read($path, ($i * MAX_OUTPUT_CHUNK_SIZE), min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE))), (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size);
}
} else {
$response->send($device->read($path));
}
});
App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
@@ -1106,17 +1294,19 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
->param('bucketId', null, new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).')
->param('fileId', '', new UID(), 'File unique ID.')
->inject('response')
->inject('request')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('usage')
->action(function ($bucketId, $fileId, $response, $dbForInternal, $dbForExternal, $usage) {
->action(function ($bucketId, $fileId, $response, $request, $dbForInternal, $dbForExternal, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Swoole\Request $request */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Appwrite\Stats\Stats $usage */
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
@@ -1145,7 +1335,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
$path = $file->getAttribute('path', '');
if (!\file_exists($path)) {
throw new Exception('File not found in '.$path, 404);
throw new Exception('File not found in ' . $path, 404);
}
$compressor = new GZIP();
@@ -1157,36 +1347,85 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
$contentType = $file->getAttribute('mimeType');
}
$source = $device->read($path);
$response
->setContentType($contentType)
->addHeader('Content-Security-Policy', 'script-src none;')
->addHeader('X-Content-Type-Options', 'nosniff')
->addHeader('Content-Disposition', 'inline; filename="' . $file->getAttribute('name', '') . '"')
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->addHeader('X-Peak', \memory_get_peak_usage())
;
$size = $file->getAttribute('sizeOriginal', 0);
$rangeHeader = $request->getHeader('range');
if(!empty($rangeHeader)) {
$start = $request->getRangeStart();
$end = $request->getRangeEnd();
$unit = $request->getRangeUnit();
if($end == null) {
$end = min(($start + 2000000-1), ($size - 1));
}
if($unit != 'bytes' || $start >= $end || $end >= $size) {
throw new Exception('Invalid range', 416);
}
$response
->addHeader('Accept-Ranges', 'bytes')
->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size)
->addHeader('Content-Length', $end - $start + 1)
->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT);
}
$source = '';
if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt
$source = $device->read($path);
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
App::getEnv('_APP_OPENSSL_KEY_V'.$file->getAttribute('openSSLVersion')),
App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
0,
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
);
}
$output = $compressor->decompress($source);
$fileName = $file->getAttribute('name', '');
if (!empty($file->getAttribute('algorithm', ''))) {
if(empty($source)) {
$source = $device->read($path);
}
$compressor = new GZIP();
$source = $compressor->decompress($source);
}
$usage
->setParam('storage.files.read', 1)
->setParam('bucketId', $bucketId)
;
$response
->setContentType($contentType)
->addHeader('Content-Security-Policy', 'script-src none;')
->addHeader('X-Content-Type-Options', 'nosniff')
->addHeader('Content-Disposition', 'inline; filename="'.$fileName.'"')
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)).' GMT') // 45 days cache
->addHeader('X-Peak', \memory_get_peak_usage())
->send($output)
;
if(!empty($source)) {
if(!empty($rangeHeader)) {
$response->send(substr($source, $start, ($end - $start + 1)));
}
$response->send($source);
}
if(!empty($rangeHeader)) {
$response->send($device->read($path, $start, ($end - $start + 1)));
}
$size = $device->getFileSize($path);
if ($size > APP_STORAGE_READ_BUFFER) {
$response->addHeader('Content-Length', $device->getFileSize($path));
$chunk = 2000000; // Max chunk of 2 mb
for ($i=0; $i < ceil($size / $chunk); $i++) {
$response->chunk($device->read($path, ($i * $chunk), min($chunk, $size - ($i * $chunk))), (($i + 1) * $chunk) >= $size);
}
} else {
$response->send($device->read($path));
}
});
App::put('/v1/storage/buckets/:bucketId/files/:fileId')
@@ -1219,7 +1458,7 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
@@ -1300,7 +1539,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
$bucket = $dbForInternal->getDocument('buckets', $bucketId);
if($bucket->isEmpty()) {
if ($bucket->isEmpty()) {
throw new Exception('Bucket not found', 404);
}
@@ -1337,8 +1576,10 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
if (!$deleted) {
throw new Exception('Failed to remove file from DB', 500);
}
} else {
throw new Exception('Failed to delete file from device', 500);
}
$audits
->setParam('event', 'storage.files.delete')
->setParam('resource', 'file/'.$file->getId())
app/controllers/general.php
+1
View File
@@ -332,6 +332,7 @@ App::error(function ($error, $utopia, $request, $response, $layout, $project) {
case 404: // Error allowed publicly
case 409: // Error allowed publicly
case 412: // Error allowed publicly
case 416: // Error allowed publicly
case 429: // Error allowed publicly
case 501: // Error allowed publicly
case 503: // Error allowed publicly
app/http.php
+1 -1
View File
@@ -19,7 +19,7 @@ use Utopia\Swoole\Request;
$http = new Server("0.0.0.0", App::getEnv('PORT', 80));
$payloadSize = max(4000000 /* 4mb */, App::getEnv('_APP_STORAGE_LIMIT', 10000000 /* 10mb */));
$payloadSize = 6 * (1024 * 1024); // 6MB
$http
->set([
app/init.php
+3
View File
@@ -79,6 +79,7 @@ const APP_STORAGE_FUNCTIONS = '/storage/functions';
const APP_STORAGE_CACHE = '/storage/cache';
const APP_STORAGE_CERTIFICATES = '/storage/certificates';
const APP_STORAGE_CONFIG = '/storage/config';
const APP_STORAGE_READ_BUFFER = 20 * (1024 * 1024); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT`
const APP_SOCIAL_TWITTER = 'https://twitter.com/appwrite_io';
const APP_SOCIAL_TWITTER_HANDLE = 'appwrite_io';
const APP_SOCIAL_FACEBOOK = 'https://www.facebook.com/appwrite.io';
@@ -119,6 +120,8 @@ const APP_AUTH_TYPE_SESSION = 'Session';
const APP_AUTH_TYPE_JWT = 'JWT';
const APP_AUTH_TYPE_KEY = 'Key';
const APP_AUTH_TYPE_ADMIN = 'Admin';
// Response related
const MAX_OUTPUT_CHUNK_SIZE = 2*1024*1024; // 2MB
$register = new Registry();
composer.json
+16 -3
View File
@@ -51,8 +51,8 @@
"utopia-php/registry": "0.5.*",
"utopia-php/preloader": "0.2.*",
"utopia-php/domains": "1.1.*",
"utopia-php/swoole": "0.2.*",
"utopia-php/storage": "0.5.*",
"utopia-php/swoole": "0.3.*",
"utopia-php/storage": "dev-feat-large-file-support",
"utopia-php/websocket": "0.0.*",
"utopia-php/image": "0.5.*",
@@ -65,7 +65,20 @@
"adhocore/jwt": "1.1.2",
"slickdeals/statsd": "3.1.0"
},
"repositories": [],
"repositories": [
{
"type": "git",
"url": "https://github.com/utopia-php/storage"
},
{
"type": "git",
"url": "https://github.com/utopia-php/database"
},
{
"type": "git",
"url": "https://github.com/utopia-php/framework"
}
],
"require-dev": {
"appwrite/sdk-generator": "0.16.0",
"phpunit/phpunit": "9.5.6",
composer.lock
Generated
+22 -49
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "fa378feaffc446f557a140035a1c77b6",
"content-hash": "bd77c6854e858381edef57388096a498",
"packages": [
{
"name": "adhocore/jwt",
@@ -2141,15 +2141,9 @@
"version": "0.10.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"url": "https://github.com/utopia-php/database",
"reference": "b7c60b0ec769a9050dd2b939b78ff1f5d4fa27e8"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/b7c60b0ec769a9050dd2b939b78ff1f5d4fa27e8",
"reference": "b7c60b0ec769a9050dd2b939b78ff1f5d4fa27e8",
"shasum": ""
},
"require": {
"ext-mongodb": "*",
"ext-pdo": "*",
@@ -2171,7 +2165,11 @@
"Utopia\\Database\\": "src/Database"
}
},
"notification-url": "https://packagist.org/downloads/",
"autoload-dev": {
"psr-4": {
"Utopia\\Tests\\": "tests/Database"
}
},
"license": [
"MIT"
],
@@ -2193,10 +2191,6 @@
"upf",
"utopia"
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.10.0"
},
"time": "2021-10-04T17:23:25+00:00"
},
{
@@ -2258,15 +2252,9 @@
"version": "0.19.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/framework.git",
"url": "https://github.com/utopia-php/framework",
"reference": "c86fc078ef258f3c88d3a25233202267314df3a9"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/framework/zipball/c86fc078ef258f3c88d3a25233202267314df3a9",
"reference": "c86fc078ef258f3c88d3a25233202267314df3a9",
"shasum": ""
},
"require": {
"php": ">=7.3.0"
},
@@ -2280,7 +2268,6 @@
"Utopia\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
@@ -2296,10 +2283,6 @@
"php",
"upf"
],
"support": {
"issues": "https://github.com/utopia-php/framework/issues",
"source": "https://github.com/utopia-php/framework/tree/0.19.0"
},
"time": "2021-10-08T11:46:20+00:00"
},
{
@@ -2568,20 +2551,14 @@
},
{
"name": "utopia-php/storage",
"version": "0.5.0",
"version": "dev-feat-large-file-support",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/storage.git",
"reference": "92ae20c7a2ac329f573a58a82dc245134cc63408"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/92ae20c7a2ac329f573a58a82dc245134cc63408",
"reference": "92ae20c7a2ac329f573a58a82dc245134cc63408",
"shasum": ""
"url": "https://github.com/utopia-php/storage",
"reference": "1597e2bd4c1091587d91b46ef2dc2878b9c71721"
},
"require": {
"php": ">=7.4",
"php": ">=8.0",
"utopia-php/framework": "0.*.*"
},
"require-dev": {
@@ -2594,7 +2571,6 @@
"Utopia\\Storage\\": "src/Storage"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
@@ -2612,24 +2588,20 @@
"upf",
"utopia"
],
"support": {
"issues": "https://github.com/utopia-php/storage/issues",
"source": "https://github.com/utopia-php/storage/tree/0.5.0"
},
"time": "2021-04-15T16:43:12+00:00"
"time": "2021-10-04T09:42:56+00:00"
},
{
"name": "utopia-php/swoole",
"version": "0.2.4",
"version": "0.3.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/swoole.git",
"reference": "37d8c64b536d6bc7da4f0f5a934a0ec44885abf4"
"reference": "b564dacb13472845f06df158ae5382e8098dfd7a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/swoole/zipball/37d8c64b536d6bc7da4f0f5a934a0ec44885abf4",
"reference": "37d8c64b536d6bc7da4f0f5a934a0ec44885abf4",
"url": "https://api.github.com/repos/utopia-php/swoole/zipball/b564dacb13472845f06df158ae5382e8098dfd7a",
"reference": "b564dacb13472845f06df158ae5382e8098dfd7a",
"shasum": ""
},
"require": {
@@ -2670,9 +2642,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/swoole/issues",
"source": "https://github.com/utopia-php/swoole/tree/0.2.4"
"source": "https://github.com/utopia-php/swoole/tree/0.3.1"
},
"time": "2021-06-22T10:49:24+00:00"
"time": "2021-07-27T09:28:10+00:00"
},
{
"name": "utopia-php/system",
@@ -5423,7 +5395,6 @@
"type": "github"
}
],
"abandoned": true,
"time": "2020-09-28T06:45:17+00:00"
},
{
@@ -6508,7 +6479,9 @@
],
"aliases": [],
"minimum-stability": "stable",
"stability-flags": [],
"stability-flags": {
"utopia-php/storage": 20
},
"prefer-stable": false,
"prefer-lowest": false,
"platform": {
docs/references/storage/create-file.md
+7 -1
View File
@@ -1 +1,7 @@
Create a new file. The user who creates the file will automatically be assigned to read and write access unless he has passed custom values for read and write arguments.
Create a new file. The user who creates the file will automatically be assigned to read and write access unless they have passed custom values for read and write arguments.
Larger files should be uploaded using multiple requests with the [content-range](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range) header to send a partial request with a maximum supported chunk of `5MB`. The `content-range` header values should always be in bytes.
When the first request is sent, the server will return the **File** object, and the subsequent part request must include the file's **id** in `x-appwrite-upload-id` header to allow the server to know the partial upload is for the existing file and not for a new one.
If you're creating a new file using one of the Appwrite SDKs, the entire chunking logic will be managed by the SDK internally.
src/Appwrite/Utopia/Response/Model/File.php
+12
View File
@@ -66,6 +66,18 @@ class File extends Model
'default' => 0,
'example' => 17890,
])
->addRule('chunksTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total number of chunks available',
'default' => 0,
'example' => 17890,
])
->addRule('chunksUploaded', [
'type' => self::TYPE_INTEGER,
'description' => 'Total number of chunks uploaded',
'default' => 0,
'example' => 17890,
])
;
}
tests/e2e/Services/Functions/FunctionsCustomServerTest.php
+43 -8
View File
@@ -292,6 +292,41 @@ class FunctionsCustomServerTest extends Scope
$this->assertIsInt($tag['body']['dateCreated']);
$this->assertEquals('php index.php', $tag['body']['command']);
$this->assertGreaterThan(10000, $tag['body']['size']);
/**
* Test for Large Code File SUCCESS
*/
$source = realpath(__DIR__ . '/../../../resources/functions/php-large.tar.gz');
$chunkSize = 5*1024*1024;
$handle = @fopen($source, "rb");
$mimeType = 'application/x-gzip';
$counter = 0;
$size = filesize($source);
$headers = [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id']
];
$id = '';
while (!feof($handle)) {
$curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'php-large-fx.tar.gz');
$headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size;
if(!empty($id)) {
$headers['x-appwrite-id'] = $id;
}
$largeTag = $this->client->call(Client::METHOD_POST, '/functions/'.$data['functionId'].'/tags', array_merge($headers, $this->getHeaders()), [
'command' => 'php index.php',
'code' => $curlFile,
]);
$counter++;
$id = $largeTag['body']['$id'];
}
@fclose($handle);
$this->assertEquals(201, $largeTag['headers']['status-code']);
$this->assertNotEmpty($largeTag['body']['$id']);
$this->assertIsInt($largeTag['body']['dateCreated']);
$this->assertEquals('php index.php', $largeTag['body']['command']);
$this->assertGreaterThan(10000, $largeTag['body']['size']);
/**
* Test for FAILURE
@@ -342,9 +377,9 @@ class FunctionsCustomServerTest extends Scope
], $this->getHeaders()));
$this->assertEquals($function['headers']['status-code'], 200);
$this->assertEquals($function['body']['sum'], 1);
$this->assertEquals($function['body']['sum'], 2);
$this->assertIsArray($function['body']['tags']);
$this->assertCount(1, $function['body']['tags']);
$this->assertCount(2, $function['body']['tags']);
/**
* Test search queries
@@ -357,9 +392,9 @@ class FunctionsCustomServerTest extends Scope
]));
$this->assertEquals($function['headers']['status-code'], 200);
$this->assertEquals($function['body']['sum'], 1);
$this->assertEquals($function['body']['sum'], 2);
$this->assertIsArray($function['body']['tags']);
$this->assertCount(1, $function['body']['tags']);
$this->assertCount(2, $function['body']['tags']);
$this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']);
$function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([
@@ -370,9 +405,9 @@ class FunctionsCustomServerTest extends Scope
]));
$this->assertEquals($function['headers']['status-code'], 200);
$this->assertEquals($function['body']['sum'], 1);
$this->assertEquals($function['body']['sum'], 2);
$this->assertIsArray($function['body']['tags']);
$this->assertCount(1, $function['body']['tags']);
$this->assertCount(2, $function['body']['tags']);
$this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']);
$function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([
@@ -383,9 +418,9 @@ class FunctionsCustomServerTest extends Scope
]));
$this->assertEquals($function['headers']['status-code'], 200);
$this->assertEquals($function['body']['sum'], 1);
$this->assertEquals($function['body']['sum'], 2);
$this->assertIsArray($function['body']['tags']);
$this->assertCount(1, $function['body']['tags']);
$this->assertCount(2, $function['body']['tags']);
$this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']);
return $data;
tests/e2e/Services/Storage/StorageBase.php
+84 -17
View File
@@ -65,24 +65,48 @@ trait StorageBase
]);
$this->assertEquals(201, $bucket2['headers']['status-code']);
$this->assertNotEmpty($bucket2['body']['$id']);
/**
* Chunked Upload
*/
$file2 = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket2['body']['$id'] . '/files', array_merge([
$source = __DIR__ . "/../../../resources/disk-a/large-file.mp4";
$totalSize = \filesize($source);
$chunkSize = 5*1024*1024;
$handle = @fopen($source, "rb");
$fileId = 'unique()';
$mimeType = mime_content_type($source);
$counter = 0;
$size = filesize($source);
$headers = [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'fileId' => 'unique()',
'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4'), 'video/mp4', 'large-file.mp4'),
'read' => ['role:all'],
'write' => ['role:all'],
]);
$this->assertEquals(201, $file2['headers']['status-code']);
$this->assertNotEmpty($file2['body']['$id']);
$this->assertIsInt($file2['body']['dateCreated']);
$this->assertEquals('large-file.mp4', $file2['body']['name']);
$this->assertEquals('video/mp4', $file2['body']['mimeType']);
$this->assertEquals(23660615, $file2['body']['sizeOriginal']);
$this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $file2['body']['signature']); // should validate that the file is not encrypted
'x-appwrite-project' => $this->getProject()['$id']
];
$id = '';
while (!feof($handle)) {
$curlFile = new \CURLFile('data://' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'large-file.mp4');
$headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size;
if(!empty($id)) {
$headers['x-appwrite-id'] = $id;
}
$largeFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket2['body']['$id'] . '/files', array_merge($headers, $this->getHeaders()), [
'fileId' => $fileId,
'file' => $curlFile,
'read' => ['role:all'],
'write' => ['role:all'],
]);
$counter++;
$id = $largeFile['body']['$id'];
}
@fclose($handle);
$this->assertEquals(201, $largeFile['headers']['status-code']);
$this->assertNotEmpty($largeFile['body']['$id']);
$this->assertIsInt($largeFile['body']['dateCreated']);
$this->assertEquals('large-file.mp4', $largeFile['body']['name']);
$this->assertEquals('video/mp4', $largeFile['body']['mimeType']);
$this->assertEquals($totalSize, $largeFile['body']['sizeOriginal']);
$this->assertEquals(md5_file(realpath(__DIR__ . '/../../../resources/disk-a/large-file.mp4')), $largeFile['body']['signature']); // should validate that the file is not encrypted
/**
* Test for FAILURE unknown Bucket
@@ -133,7 +157,7 @@ trait StorageBase
$this->assertEquals(400, $res['headers']['status-code']);
$this->assertEquals('File extension not allowed', $res['body']['message']);
return ['bucketId' => $bucketId, 'fileId' => $file['body']['$id'], 'largeFileId' => $file2['body']['$id'], 'largeBucketId' => $bucket2['body']['$id']];
return ['bucketId' => $bucketId, 'fileId' => $file['body']['$id'], 'largeFileId' => $largeFile['body']['$id'], 'largeBucketId' => $bucket2['body']['$id']];
}
/**
@@ -263,6 +287,49 @@ trait StorageBase
$this->assertEquals('image/png', $file5['headers']['content-type']);
$this->assertNotEmpty($file5['body']);
// Test ranged download
$file51 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'Range' => 'bytes=0-99',
], $this->getHeaders()));
$path = __DIR__ . '/../../../resources/logo.png';
$originalChunk = \file_get_contents($path, false, null, 0, 100);
$this->assertEquals(206, $file51['headers']['status-code']);
$this->assertEquals('attachment; filename="logo.png"', $file51['headers']['content-disposition']);
$this->assertEquals('image/png', $file51['headers']['content-type']);
$this->assertNotEmpty($file51['body']);
$this->assertEquals($originalChunk, $file51['body']);
// Test ranged download - with invalid range
$file52 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'Range' => 'bytes=0-',
], $this->getHeaders()));
$this->assertEquals(206, $file52['headers']['status-code']);
// Test ranged download - with invalid range
$file53 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'Range' => 'bytes=988',
], $this->getHeaders()));
$this->assertEquals(416, $file53['headers']['status-code']);
// Test ranged download - with invalid range
$file54 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/download', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'Range' => 'bytes=-988',
], $this->getHeaders()));
$this->assertEquals(416, $file54['headers']['status-code']);
$file6 = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $data['fileId'] . '/view', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
tests/resources/functions/php-large.tar.gz
BIN
View File
Binary file not shown.