From c2e599222e833aa179eb6ae71fccfd6bc18f46fa Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 3 Jul 2022 16:59:40 +0300 Subject: [PATCH] big file --- .env | 2 +- .gitignore | 2 ++ app/controllers/api/projects.php | 1 - app/controllers/api/storage.php | 13 ++++++++++++- tests/e2e/Services/Video/VideoCustomServerTest.php | 7 +++++-- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 1f0e7a152e..598e6b40ed 100644 --- a/.env +++ b/.env @@ -56,7 +56,7 @@ _APP_SMTP_PORT=1025 _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= -_APP_STORAGE_LIMIT=30000000 +_APP_STORAGE_LIMIT=300000000000 _APP_STORAGE_PREVIEW_LIMIT=20000000 _APP_FUNCTIONS_SIZE_LIMIT=30000000 _APP_FUNCTIONS_TIMEOUT=900 diff --git a/.gitignore b/.gitignore index 3d6001ca06..f9ef9a6f21 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ debug/ app/sdks dev/yasd_init.php +/tests/resources/disk-a/very-large-file-1.mov +/tests/tmp/ \ No newline at end of file diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 9af792d503..98c08f6e94 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -149,7 +149,6 @@ App::post('/v1/projects') 'orders' => $index['orders'], ]); } - var_dump($key); $dbForProject->createCollection($key, $attributes, $indexes); } diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index da70665e8b..c53e7953f0 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -404,6 +404,7 @@ App::post('/v1/storage/buckets/:bucketId/files') $fileExt = new FileExt($allowedFileExtensions); $maximumFileSize = $bucket->getAttribute('maximumFileSize', 0); + if ($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT', 0)) { throw new Exception('Maximum bucket file size is larger than _APP_STORAGE_LIMIT', 500, Exception::GENERAL_SERVER_ERROR); } @@ -419,6 +420,7 @@ App::post('/v1/storage/buckets/:bucketId/files') $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; $contentRange = $request->getHeader('content-range'); + $fileId = $fileId === 'unique()' ? $dbForProject->getId() : $fileId; $chunk = 1; $chunks = 1; @@ -474,9 +476,13 @@ App::post('/v1/storage/buckets/:bucketId/files') }); } else { $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); + var_dump('&&&&&&&'); + var_dump($fileId); + var_dump($file); } $metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)]; + if (!$file->isEmpty()) { $chunks = $file->getAttribute('chunksTotal', 1); $metadata = $file->getAttribute('metadata', []); @@ -492,6 +498,7 @@ App::post('/v1/storage/buckets/:bucketId/files') $read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; $write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; + if ($chunksUploaded === $chunks) { if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) { $antivirus = new Network( @@ -530,7 +537,6 @@ App::post('/v1/storage/buckets/:bucketId/files') } $sizeActual = $deviceFiles->getFileSize($path); - $algorithm = empty($compressor) ? '' : $compressor->getName(); $fileHash = $deviceFiles->getFileHash($path); @@ -541,6 +547,7 @@ App::post('/v1/storage/buckets/:bucketId/files') $openSSLIV = \bin2hex($iv); } + try { if ($file->isEmpty()) { $doc = new Document([ @@ -566,12 +573,14 @@ App::post('/v1/storage/buckets/:bucketId/files') 'search' => implode(' ', [$fileId, $fileName]), 'metadata' => $metadata, ]); + if ($permissionBucket) { $file = Authorization::skip(fn () => $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc)); } else { $file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc); } } else { + $file = $file ->setAttribute('$read', $read) ->setAttribute('$write', $write) @@ -608,6 +617,7 @@ App::post('/v1/storage/buckets/:bucketId/files') ->setParam('bucketId', $bucketId) ; } else { + try { if ($file->isEmpty()) { $doc = new Document([ @@ -629,6 +639,7 @@ App::post('/v1/storage/buckets/:bucketId/files') 'search' => implode(' ', [$fileId, $fileName]), 'metadata' => $metadata, ]); + if ($permissionBucket) { $file = Authorization::skip(fn () => $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc)); } else { diff --git a/tests/e2e/Services/Video/VideoCustomServerTest.php b/tests/e2e/Services/Video/VideoCustomServerTest.php index a1b7035cb0..6cb065d337 100644 --- a/tests/e2e/Services/Video/VideoCustomServerTest.php +++ b/tests/e2e/Services/Video/VideoCustomServerTest.php @@ -27,7 +27,8 @@ class VideoCustomServerTest extends Scope 'write' => ['role:all'] ]); - $source = __DIR__ . "/../../../resources/disk-a/large-file.mp4"; + //$source = __DIR__ . "/../../../resources/disk-a/large-file.mp4"; + $source = __DIR__ . "/../../../resources/disk-a/very-large-file-1.mov"; $totalSize = \filesize($source); $chunkSize = 5 * 1024 * 1024; $handle = @fopen($source, "rb"); @@ -42,7 +43,7 @@ class VideoCustomServerTest extends Scope $id = ''; while (!feof($handle)) { - $curlFile = new \CURLFile('data:' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'in1.mp4'); + $curlFile = new \CURLFile('data:' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'very-large-file-1.mov'); $headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size; if (!empty($id)) { @@ -56,6 +57,8 @@ class VideoCustomServerTest extends Scope 'write' => ['role:all'], ]); $counter++; + + $this->assertNotEmpty($file['body']['$id']); $id = $file['body']['$id']; } @fclose($handle);