diff --git a/.env b/.env index 351447a8b4..ade9e7566d 100644 --- a/.env +++ b/.env @@ -58,7 +58,7 @@ _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= _APP_PHONE_PROVIDER=phone://mock _APP_PHONE_FROM=+123456789 -_APP_STORAGE_LIMIT=30000000 +_APP_STORAGE_LIMIT=300000000000 _APP_STORAGE_PREVIEW_LIMIT=20000000 _APP_FUNCTIONS_SIZE_LIMIT=30000000 _APP_FUNCTIONS_TIMEOUT=900 diff --git a/app/config/collections.php b/app/config/collections.php index a71eb542c1..49d91429e5 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -3239,10 +3239,10 @@ $collections = [ ], [ '$id' => 'videoBitrate', - 'type' => Database::VAR_STRING, + 'type' => Database::VAR_INTEGER, 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, + 'size' => 0, + 'signed' => false, 'required' => false, 'default' => null, 'array' => false, @@ -3272,10 +3272,10 @@ $collections = [ ], [ '$id' => 'audioBitrate', - 'type' => Database::VAR_STRING, + 'type' => Database::VAR_INTEGER, 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, + 'size' => 0, + 'signed' => false, 'required' => false, 'default' => null, 'array' => false, @@ -3283,10 +3283,10 @@ $collections = [ ], [ '$id' => 'audioSamplerate', - 'type' => Database::VAR_STRING, + 'type' => Database::VAR_INTEGER, 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, + 'size' => 0, + 'signed' => false, 'required' => false, 'default' => null, 'array' => false, @@ -3296,7 +3296,7 @@ $collections = [ '$id' => 'metadata', 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 2048, + 'size' => 4096, 'signed' => true, 'required' => false, 'default' => null, diff --git a/app/config/video-profiles.php b/app/config/video-profiles.php index a32788dbd7..a7d3874635 100644 --- a/app/config/video-profiles.php +++ b/app/config/video-profiles.php @@ -8,6 +8,7 @@ return [ 'audioBitrate' => 64, //audio bitrate in Kbps 'width' => 640, //width resolution in px 'height' => 360, //height resolution in px + 'stream' => 'mpeg-dash' ], [ 'name' => '576p', @@ -15,6 +16,7 @@ return [ 'audioBitrate' => 128, 'width' => 1024, 'height' => 576, + 'stream' => 'mpeg-dash' ], [ 'name' => '720p', @@ -22,6 +24,7 @@ return [ 'audioBitrate' => 128, 'width' => 1280, 'height' => 720, + 'stream' => 'mpeg-dash' ], ]; diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 24c0e783ef..b6a9150564 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -155,7 +155,7 @@ App::post('/v1/projects') $dbForProject->createCollection($key, $attributes, $indexes); } - if($dbForProject->exists($dbForProject->getDefaultDatabase(), 'video_profiles')) { + if ($dbForProject->exists($dbForProject->getDefaultDatabase(), 'video_profiles')) { foreach (Config::getParam('video-profiles', []) as $profile) { Authorization::skip(function () use ($project, $profile, $dbForProject) { return $dbForProject->createDocument('video_profiles', new Document([ @@ -164,7 +164,7 @@ App::post('/v1/projects') 'audioBitrate' => $profile['audioBitrate'], 'width' => $profile['width'], 'height' => $profile['height'], - 'stream' => 'hls' + 'stream' => $profile['stream'] ])); }); } diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f195613f7d..9cc9d9194d 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -122,7 +122,6 @@ App::post('/v1/storage/buckets') $bucket = $dbForProject->getDocument('buckets', $bucketId); $dbForProject->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); - } catch (Duplicate $th) { throw new Exception('Bucket already exists', 409, Exception::STORAGE_BUCKET_ALREADY_EXISTS); } @@ -474,9 +473,6 @@ App::post('/v1/storage/buckets/:bucketId/files') }); } else { $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); - var_dump('&&&&&&&'); - var_dump($fileId); - var_dump($file); } $metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)]; @@ -577,7 +573,6 @@ App::post('/v1/storage/buckets/:bucketId/files') $file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc); } } else { - $file = $file ->setAttribute('$read', $read) ->setAttribute('$write', $write) @@ -614,7 +609,6 @@ App::post('/v1/storage/buckets/:bucketId/files') ->setParam('bucketId', $bucketId) ; } else { - try { if ($file->isEmpty()) { $doc = new Document([ diff --git a/app/controllers/api/video.php b/app/controllers/api/video.php index 2ba4407148..6512fb9991 100644 --- a/app/controllers/api/video.php +++ b/app/controllers/api/video.php @@ -3,6 +3,8 @@ use Appwrite\Auth\Auth; use Appwrite\ClamAV\Network; use Appwrite\Event\Audit; +use Appwrite\Event\Audit as EventAudit; +use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Transcoding; @@ -44,6 +46,70 @@ use Utopia\Validator\WhiteList; use Utopia\Swoole\Request; use Streaming\Representation; +/** + * Validate file Permissions + * + * @param Database $dbForProject + * @param string $bucketId + * @param string $fileId + * @param array|null $read + * @param array|null $write + * @param string $mode + * @return Document $file + * @throws Exception + */ +function validateFilePermissions(Database $dbForProject, string $bucketId, string $fileId, string $mode, ?array $read, ?array $write): Document +{ + /** @var Utopia\Database\Document $project */ + /** @var Utopia\Database\Document $user */ + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + if ( + $bucket->isEmpty() + || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN) + ) { + throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND); + } + + // Check bucket permissions when enforced + $permissionBucket = $bucket->getAttribute('permission') === 'bucket'; + if ($permissionBucket) { + $validator = new Authorization('write'); + if (!$validator->isValid($bucket->getWrite())) { + throw new Exception('Unauthorized file permissions', 401, Exception::USER_UNAUTHORIZED); + } + } + + $read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user + $write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; + + // Users can only add their roles to files, API keys and Admin users can add any + $roles = Authorization::getRoles(); + + if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles)) { + foreach ($read as $role) { + if (!Authorization::isRole($role)) { + throw new Exception('Read permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED); + } + } + foreach ($write as $role) { + if (!Authorization::isRole($role)) { + throw new Exception('Write permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED); + } + } + } + + if ($bucket->getAttribute('permission') === 'bucket') { + // skip authorization + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId)); + } else { + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); + } + + return $file; +} + App::get('/v1/video/profiles') ->alias('/v1/video/video/profiles', []) ->desc('Get all video profiles') @@ -91,53 +157,12 @@ App::post('/v1/video/buckets/:bucketId/files/:fileId') /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Document $user */ - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + $file = validateFilePermissions($dbForProject, $bucketId, $fileId, $mode, $read, $write); - if ( - $bucket->isEmpty() - || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN) - ) { - throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND); - } - - // Check bucket permissions when enforced - $permissionBucket = $bucket->getAttribute('permission') === 'bucket'; - if ($permissionBucket) { - $validator = new Authorization('write'); - if (!$validator->isValid($bucket->getWrite())) { - throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED); - } - } - - $read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user - $write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; - - // Users can only add their roles to files, API keys and Admin users can add any - $roles = Authorization::getRoles(); - - if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles)) { - foreach ($read as $role) { - if (!Authorization::isRole($role)) { - throw new Exception('Read permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED); - } - } - foreach ($write as $role) { - if (!Authorization::isRole($role)) { - throw new Exception('Write permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED); - } - } - } - - if ($bucket->getAttribute('permission') === 'bucket') { - // skip authorization - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId)); - } else { - $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); - } try { - $video = Authorization::skip(function () use ($dbForProject, $bucket, $file) { + $video = Authorization::skip(function () use ($dbForProject, $bucketId, $file) { return $dbForProject->createDocument('videos', new Document([ - 'bucketId' => $bucket->getId(), + 'bucketId' => $bucketId, 'fileId' => $file->getId(), 'size' => $file->getAttribute('sizeOriginal'), ])); @@ -186,49 +211,7 @@ App::post('/v1/video/:videoId/rendition/:profileId') throw new Exception('Video not found', 400, Exception::VIDEO_NOT_FOUND); } - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $video['bucketId'])); - - if ( - $bucket->isEmpty() - || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN) - ) { - throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND); - } - - // Check bucket permissions when enforced - $permissionBucket = $bucket->getAttribute('permission') === 'bucket'; - if ($permissionBucket) { - $validator = new Authorization('write'); - if (!$validator->isValid($bucket->getWrite())) { - throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED); - } - } - - $read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user - $write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; - - // Users can only add their roles to files, API keys and Admin users can add any - $roles = Authorization::getRoles(); - - if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles)) { - foreach ($read as $role) { - if (!Authorization::isRole($role)) { - throw new Exception('Read permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED); - } - } - foreach ($write as $role) { - if (!Authorization::isRole($role)) { - throw new Exception('Write permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED); - } - } - } - - if ($bucket->getAttribute('permission') === 'bucket') { - // skip authorization - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId'])); - } else { - $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId']); - } + validateFilePermissions($dbForProject, $video['bucketId'], $video['fileId'], $mode, $read, $write); $profile = Authorization::skip(fn() => $dbForProject->findOne('video_profiles', [new Query('_uid', Query::TYPE_EQUAL, [$profileId])])); @@ -261,7 +244,7 @@ App::get('/v1/video/:videoId/:stream/renditions') ->inject('dbForProject') ->inject('usage') ->inject('mode') - ->action(function (string $videoId, string $stream, ?array $read, ?array $write, Response $response, Database $dbForProject, Stats $usage, string $mode) { + ->action(function (string $videoId, string $stream, ?array $read, ?array $write, Response $response, Database $dbForProject, Stats $usage, string $mode) { /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Document $user */ @@ -272,49 +255,7 @@ App::get('/v1/video/:videoId/:stream/renditions') throw new Exception('Video not found', 400, Exception::VIDEO_NOT_FOUND); } - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $video['bucketId'])); - - if ( - $bucket->isEmpty() - || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN) - ) { - throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND); - } - - // Check bucket permissions when enforced - $permissionBucket = $bucket->getAttribute('permission') === 'bucket'; - if ($permissionBucket) { - $validator = new Authorization('write'); - if (!$validator->isValid($bucket->getWrite())) { - throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED); - } - } - - $read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user - $write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; - - // Users can only add their roles to files, API keys and Admin users can add any - $roles = Authorization::getRoles(); - - if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles)) { - foreach ($read as $role) { - if (!Authorization::isRole($role)) { - throw new Exception('Read permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED); - } - } - foreach ($write as $role) { - if (!Authorization::isRole($role)) { - throw new Exception('Write permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED); - } - } - } - - if ($bucket->getAttribute('permission') === 'bucket') { - // skip authorization - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId'])); - } else { - $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId']); - } + $file = validateFilePermissions($dbForProject, $video['bucketId'], $video['fileId'], $mode, $read, $write); $queries = [ new Query('videoId', Query::TYPE_EQUAL, [$video->getId()]), @@ -331,122 +272,94 @@ App::get('/v1/video/:videoId/:stream/renditions') ]), Response::MODEL_VIDEO_RENDITIONS_LIST); }); -App::get('/v1/video/:videoId/:stream/:namespace/:fileName') - ->alias('/v1/video/buckets/:bucketId/files/:stream/:fileId', []) - ->desc('Get video playlist manifest') + + +App::get('/v1/video/:videoId/:stream/:profile/:fileName') + ->alias('/v1/video/:videoId/:stream/:profile/:fileName', []) + ->desc('Get video playlist manifests') ->groups(['api', 'storage']) ->label('scope', 'files.read') ->param('videoId', null, new UID(), 'Video unique ID.') ->param('stream', '', new WhiteList(['hls', 'mpeg-dash']), 'stream protocol name') - ->param('namespace', '', new WhiteList(['master']), 'stream protocol name') + ->param('profile', '', new Text(18), 'folder name') + ->param('fileName', '', new Text(128), 'playlist file name') ->param('read', null, new Permissions(), 'An array of strings with read permissions. By default only the current user is granted with read permissions. [learn more about permissions](https://appwrite.io/docs/permissions) and get a full list of available permissions.', true) ->param('write', null, new Permissions(), 'An array of strings with write permissions. By default only the current user is granted with write permissions. [learn more about permissions](https://appwrite.io/docs/permissions) and get a full list of available permissions.', true) ->inject('response') ->inject('dbForProject') + ->inject('videosDevice') ->inject('usage') ->inject('mode') - ->action(function (string $videoId, string $stream, string $namespace, ?array $read, ?array $write, Response $response, Database $dbForProject, Stats $usage, string $mode) { + ->action(function (string $videoId, string $stream, string $profile, string $fileName, ?array $read, ?array $write, Response $response, Database $dbForProject, Device $videosDevice, Stats $usage, string $mode) { /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Document $user */ - $video = Authorization::skip(fn() => $dbForProject->findOne('videos', [new Query('_uid', Query::TYPE_EQUAL, [$videoId])])); + $video = Authorization::skip(fn() => $dbForProject->findOne('videos', [ + new Query('_uid', Query::TYPE_EQUAL, [$videoId]) + ])); if ($video->isEmpty()) { throw new Exception('Video not found', 400, Exception::VIDEO_NOT_FOUND); } - $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $video['bucketId'])); + validateFilePermissions($dbForProject, $video['bucketId'], $video['fileId'], $mode, $read, $write); - if ( - $bucket->isEmpty() - || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN) - ) { - throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND); - } - - // Check bucket permissions when enforced - $permissionBucket = $bucket->getAttribute('permission') === 'bucket'; - if ($permissionBucket) { - $validator = new Authorization('write'); - if (!$validator->isValid($bucket->getWrite())) { - throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED); - } - } - - $read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user - $write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; - - // Users can only add their roles to files, API keys and Admin users can add any - $roles = Authorization::getRoles(); - - if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles)) { - foreach ($read as $role) { - if (!Authorization::isRole($role)) { - throw new Exception('Read permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED); - } - } - foreach ($write as $role) { - if (!Authorization::isRole($role)) { - throw new Exception('Write permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED); - } - } - } - - if ($bucket->getAttribute('permission') === 'bucket') { - // skip authorization - $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId'])); - } else { - $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId']); - } - - $queries = [ + $renditions = Authorization::skip(fn () => $dbForProject->find('video_renditions', [ new Query('videoId', Query::TYPE_EQUAL, [$video->getId()]), new Query('endedAt', Query::TYPE_GREATER, [0]), new Query('status', Query::TYPE_EQUAL, ['ready']), - new Query('stream', Query::TYPE_EQUAL, ['stream']), - ]; - - $renditions = Authorization::skip(fn () => $dbForProject->find('video_renditions', $queries, 12, 0, [], ['ASC'])); + new Query('stream', Query::TYPE_EQUAL, [$stream]), + ], 12, 0, [], ['ASC'])); if (empty($renditions)) { throw new Exception('Renditions not found'); } - if ($stream === 'hls') { - foreach ($renditions as $rendition) { - $metadata = $rendition->getAttribute('metadata'); - $fileId = $rendition->getAttribute('fileId'); - $t['bandwidth'] = (($metadata['general']['video']['bitrate'] + $metadata['general']['audio']['bitrate']) * 1024); - $t['resolution'] = $metadata['general']['resolution']; - $t['name'] = $rendition->getAttribute('renditionName'); - $t['path'] = $rendition->getAttribute('renditionName') . DIRECTORY_SEPARATOR . $fileId . '_' . $rendition->getAttribute('renditionName') . '.m3u8'; - $params[] = $t; - } + $ct['m3u8'] = 'application/x-mpegurl'; + $ct['mpd'] = 'application/dash+xml'; + $ct['ts'] = 'video/MP2T'; + $ct['m4s'] = 'video/iso.segment'; + $ext = pathinfo($fileName, PATHINFO_EXTENSION); + $baseUrl = 'http://127.0.0.1/v1/video/' . $videoId . '/' . $stream . '/' ; - $template = new View(__DIR__ . '/../../views/video/hls.phtml'); - $template->setParam('params', $params); - $response->setContentType('application/x-mpegurl'); - $response->send($template->render()); - } else { - - $adaptations = []; - foreach ($renditions as $rendition) { - $metadata = $rendition->getAttribute('metadata'); - foreach ($metadata['dash']['Period']['AdaptationSet'] as $set) { - $adaption = $set['@attributes']; - $adaption['baseUrl'] = $rendition->getAttribute('renditionName') . DIRECTORY_SEPARATOR; - $adaption['representation'] = $set['Representation']['@attributes']; - $adaption['representation']['SegmentTemplate'] = $set['Representation']['SegmentTemplate']['@attributes']; - $adaption['representation']['segmentTemplate']['segmentTimeline'] = $set['Representation']['SegmentTemplate']['SegmentTimeline']; - $adaptations[] = $adaption; + if ($profile === 'master') { + if ($stream === 'hls') { + foreach ($renditions as $rendition) { + $t['bandwidth'] = $rendition->getAttribute('videoBitrate') + $rendition->getAttribute('audioBitrate'); + $t['resolution'] = $rendition->getAttribute('width') . 'X' . $rendition->getAttribute('height'); + $t['name'] = $rendition->getAttribute('name'); + $t['path'] = $baseUrl . $rendition->getAttribute('name') . '/' . $rendition->getAttribute('videoId') . '_' . $rendition->getAttribute('name') . '.m3u8'; + $params[] = $t; } - } - $template = new View(__DIR__ . '/../../views/video/dash.phtml'); - $template->setParam('params', $adaptations); - $response->setContentType('application/dash+xml'); - $response->send($template->render()); + $template = new View(__DIR__ . '/../../views/video/hls.phtml'); + $template->setParam('params', $params); + $output = $template->render(); + } else { + $adaptations = []; + foreach ($renditions as $rendition) { + $metadata = $rendition->getAttribute('metadata'); + foreach ($metadata['mpeg-dash']['Period']['AdaptationSet'] as $set) { + $adaption = $set['@attributes']; + $adaption['baseUrl'] = $baseUrl . $rendition->getAttribute('name') . '/'; + $adaption['representation'] = $set['Representation']['@attributes']; + $adaption['representation']['SegmentTemplate'] = $set['Representation']['SegmentTemplate']['@attributes']; + $adaption['representation']['segmentTemplate']['segmentTimeline'] = $set['Representation']['SegmentTemplate']['SegmentTimeline']; + $adaptations[] = $adaption; + } + } + + $template = new View(__DIR__ . '/../../views/video/dash.phtml'); + $template->setParam('params', $adaptations); + $response->setContentType($ct[$ext]); + $output = $template->render(); + } + } else { + $output = $videosDevice->read($videosDevice->getRoot() . '/' . $videoId . '/' . $profile . '/' . $fileName); } + + $response->setContentType($ct[$ext]) + ->send($output); }); diff --git a/app/init.php b/app/init.php index 26f8b2a04c..bbbda4175d 100644 --- a/app/init.php +++ b/app/init.php @@ -915,6 +915,10 @@ App::setResource('deviceFiles', function ($project) { return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); }, ['project']); +App::setResource('videosDevice', function ($project) { + return getDevice(APP_STORAGE_VIDEO . '/app-' . $project->getId()); +}, ['project']); + App::setResource('deviceFunctions', function ($project) { return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); }, ['project']); diff --git a/app/workers/messaging.php b/app/workers/messaging.php index 044107af4b..409304c595 100644 --- a/app/workers/messaging.php +++ b/app/workers/messaging.php @@ -67,4 +67,4 @@ class MessagingV1 extends Worker public function shutdown(): void { } -} \ No newline at end of file +} diff --git a/app/workers/transcoding.php b/app/workers/transcoding.php index a53a9d9db0..26a71e8546 100644 --- a/app/workers/transcoding.php +++ b/app/workers/transcoding.php @@ -9,7 +9,6 @@ use Streaming\Metadata; use Streaming\Representation; use Utopia\App; use Utopia\CLI\Console; -use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; @@ -33,7 +32,7 @@ class TranscodingV1 extends Worker const STATUS_PACKAGE_END = 'ready'; const STATUS_ERROR = 'error'; - const HLS_BASE_URL = ''; + const STREAM_HLS = 'hls'; //protected string $basePath = '/tmp/'; protected string $basePath = '/usr/src/code/tests/tmp/'; @@ -44,6 +43,8 @@ class TranscodingV1 extends Worker protected string $outPath; + protected string $renditionName; + protected Database $database; @@ -79,11 +80,9 @@ class TranscodingV1 extends Worker throw new Exception('profile not found'); } - $user = new Document($this->args['user'] ?? []); $bucket = Authorization::skip(fn() => $this->database->getDocument('buckets', $sourceVideo['bucketId'])); - if ($bucket->getAttribute('permission') === 'bucket') { - $file = Authorization::skip(fn() => $this->database->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId'])); + $file = Authorization::skip(fn() => $this->database->getDocument('bucket_' . $bucket->getInternalId(), $sourceVideo['fileId'])); } else { $file = $this->database->getDocument('bucket_' . $bucket->getInternalId(), $sourceVideo['fileId']); } @@ -134,9 +133,8 @@ class TranscodingV1 extends Worker } } - $stream = !empty($rendition['stream']) ? $rendition['stream'] : 'hls'; $general = $this->getVideoInfo($ffprobe->streams($inPath)); - if(!empty($general)) { + if (!empty($general)) { foreach ($general as $key => $value) { $sourceVideo->setAttribute($key, $value); } @@ -148,17 +146,17 @@ class TranscodingV1 extends Worker )); } - $video = $ffmpeg->open($inPath); + $this->setRenditionName($profile); - $query = Authorization::skip(function () use ($collection, $profile, $stream) { + $query = Authorization::skip(function () use ($collection, $profile) { return $this->database->createDocument($collection, new Document([ 'videoId' => $this->args['videoId'], 'profileId' => $profile->getId(), - 'name' => $profile->getAttribute('name'), + 'name' => $this->getRenditionName(), 'startedAt' => time(), 'status' => self::STATUS_TRANSCODE_START, - 'stream' => $stream, + 'stream' => $profile['stream'], ])); }); @@ -181,8 +179,7 @@ class TranscodingV1 extends Worker }); - list($general, $metadata) = $this->transcode($stream, $video, $format, $representation); - + list($general, $metadata) = $this->transcode($profile['stream'], $video, $format, $representation); if (!empty($metadata)) { $query->setAttribute('metadata', json_encode($metadata)); } @@ -214,13 +211,11 @@ class TranscodingV1 extends Worker continue; } - $deviceFiles = $this->getVideoDevice($project->getId()); - $devicePath = $deviceFiles->getPath($this->args['videoId']); + $deviceFiles = $this->getVideoDevice($project->getId()); + $devicePath = $deviceFiles->getPath($this->args['videoId']); $data = $this->getFilesDevice($project->getId())->read($this->outDir . $fileName); - $renditionDir = $profile->getAttribute('width') . 'X' . $profile->getAttribute('height') . '@' . $profile->getAttribute('videoBitrate'); - $renditionPath = $devicePath . DIRECTORY_SEPARATOR . $renditionDir; + $renditionPath = $devicePath . DIRECTORY_SEPARATOR . $this->getRenditionName(); $this->getVideoDevice($project->getId())->write($renditionPath . DIRECTORY_SEPARATOR . $fileName, $data, \mime_content_type($this->outDir . $fileName)); - if ($start === 0) { $query->setAttribute('status', self::STATUS_UPLOADING); $query->setAttribute('path', $renditionPath); @@ -275,13 +270,14 @@ class TranscodingV1 extends Worker { $additionalParams = [ + '-dn', '-sn', '-vf', 'scale=iw:-2:force_original_aspect_ratio=increase,setsar=1:1' ]; $segementSize = 10; - if ($stream === 'dash') { + if ($stream === 'mpeg-dash') { $dash = $video->dash() ->setFormat($format) ->setSegDuration($segementSize) @@ -290,14 +286,14 @@ class TranscodingV1 extends Worker ->save($this->outPath); $xml = simplexml_load_string( - file_get_contents($this->outDir . $this->args['fileId'] . '.mpd') + file_get_contents($this->outDir . $this->args['videoId'] . '.mpd') ); - $metadata = $this->getVideoInfo($dash->metadata()); - $metadata['width'] = $representation->getWidth(); - $metadata['height'] = $representation->getHeight(); + $general = $this->getVideoInfo($dash->metadata()->getVideoStreams()); + $general['width'] = $representation->getWidth(); + $general['height'] = $representation->getHeight(); return [ - $metadata, + $general, ['mpeg-dash' => !empty($xml) ? json_decode(json_encode((array)$xml), true) : []], ]; } @@ -308,31 +304,35 @@ class TranscodingV1 extends Worker ->setHlsTime($segementSize) ->addRepresentation($representation) ->setAdditionalParams($additionalParams) - ->setHlsBaseUrl(self::HLS_BASE_URL) + ->setHlsBaseUrl('http://127.0.0.1/v1/video/' . $this->args['videoId'] . '/' . self::STREAM_HLS . '/' . $this->getRenditionName() . '/') ->save($this->outPath); - $metadata = $this->getVideoInfo($hls->metadata()->getVideoStreams()); - $metadata['width'] = $representation->getWidth(); - $metadata['height'] = $representation->getHeight(); + $general = $this->getVideoInfo($hls->metadata()->getVideoStreams()); + $general['width'] = $representation->getWidth(); + $general['height'] = $representation->getHeight(); - $this->rewriteHlsRefs($this->outPath . '_' . $representation->getHeight() . 'P.m3u8'); + //$this->rewriteHlsLines($this->outPath . '_' . $this->getRenditionName() . '.m3u8'); return [ - $metadata, [] + $general, [] ]; } - private function rewriteHlsRefs($path) + private function rewriteHlsLines($path) { $handle = fopen($path, "r"); $destination = fopen($path . '_tmp', "w"); if ($handle) { while (($line = fgets($handle)) !== false) { + $newLine = str_replace(array("\r","\n"), "", $line); if (str_contains($line, ".ts")) { - //$return['data'][$i]['url'] = str_replace(array("\r","\n"),"",$line); + $newLine = 'http://127.0.0.1/v1/video/' . $this->args['videoId'] . '/' . self::STREAM_HLS . '/' . $this->getRenditionName() . '/' . $newLine; + var_dump($newLine); } - fwrite($destination, str_replace(array("\r","\n"), "", $line) . PHP_EOL); + fwrite($destination, $newLine . PHP_EOL); } + var_dump($path . '_tmp -> ' . $path); + rename($path . '_tmp', $path); fclose($handle); fclose($destination); } @@ -344,17 +344,6 @@ class TranscodingV1 extends Worker */ private function getVideoInfo(StreamCollection $streams): array { - var_dump([ - 'duration' => $streams->videos()->first()->get('duration'), - 'height' => $streams->videos()->first()->get('height'), - 'width' => $streams->videos()->first()->get('width'), - 'videoCodec' => $streams->videos()->first()->get('codec_name') . ',' . $streams->videos()->first()->get('codec_tag_string'), - 'videoFramerate' => $streams->videos()->first()->get('avg_frame_rate'), - 'videoBitrate' => (int)$streams->videos()->first()->get('bit_rate'), - 'audioCodec' => $streams->audios()->first()->get('codec_name') . ',' . $streams->audios()->first()->get('codec_tag_string'), - 'audioSamplerate' => (int)$streams->audios()->first()->get('sample_rate'), - 'audioBitrate' => (int)$streams->audios()->first()->get('bit_rate'), - ]); return [ 'duration' => $streams->videos()->first()->get('duration'), 'height' => $streams->videos()->first()->get('height'), @@ -368,6 +357,19 @@ class TranscodingV1 extends Worker ]; } + private function setRenditionName($profile) + { + $this->renditionName = $profile->getAttribute('width') + . 'X' . $profile->getAttribute('height') + . '@' . ($profile->getAttribute('videoBitrate') + $profile->getAttribute('audioBitrate')); + } + + private function getRenditionName(): string + { + return $this->renditionName; + } + + private function cleanup(): bool { var_dump("rm -rf {$this->basePath}"); diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 222cf59444..a6f9420ae2 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -32,6 +32,9 @@ class Event public const BUILDS_QUEUE_NAME = 'v1-builds'; public const BUILDS_CLASS_NAME = 'BuildsV1'; + public const TRANSCODING_QUEUE_NAME = 'v1-transcoding'; + public const TRANSCODING_CLASS_NAME = 'TranscodingV1'; + public const MESSAGING_QUEUE_NAME = 'v1-messaging'; public const MESSAGING_CLASS_NAME = 'MessagingV1'; diff --git a/src/Appwrite/Event/Transcoding.php b/src/Appwrite/Event/Transcoding.php index 2dac45e178..e2e070de6d 100644 --- a/src/Appwrite/Event/Transcoding.php +++ b/src/Appwrite/Event/Transcoding.php @@ -7,7 +7,6 @@ use Utopia\Database\Document; class Transcoding extends Event { - protected string $videoId = ''; protected string $profileId = ''; diff --git a/tests/e2e/Services/Video/VideoCustomServerTest.php b/tests/e2e/Services/Video/VideoCustomServerTest.php index ed253af93d..f54fa5f51d 100644 --- a/tests/e2e/Services/Video/VideoCustomServerTest.php +++ b/tests/e2e/Services/Video/VideoCustomServerTest.php @@ -27,41 +27,41 @@ class VideoCustomServerTest extends Scope 'write' => ['role:all'] ]); -// //$source = __DIR__ . "/../../../resources/disk-a/large-file.mp4"; -// $source = __DIR__ . "/../../../resources/disk-a/very-large-file-1.mov"; -// $totalSize = \filesize($source); -// $chunkSize = 5 * 1024 * 1024; -// $handle = @fopen($source, "rb"); -// $fileId = 'unique()'; -// $mimeType = mime_content_type($source); -// $counter = 0; -// $size = filesize($source); -// $headers = [ -// 'content-type' => 'multipart/form-data', -// 'x-appwrite-project' => $this->getProject()['$id'] -// ]; -// $id = ''; -// -// while (!feof($handle)) { -// $curlFile = new \CURLFile('data:' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'very-large-file-1.mov'); -// $headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size; -// -// if (!empty($id)) { -// $headers['x-appwrite-id'] = $id; -// } -// -// $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket['body']['$id'] . '/files', array_merge($headers, $this->getHeaders()), [ -// 'fileId' => $fileId, -// 'file' => $curlFile, -// 'read' => ['role:all'], -// 'write' => ['role:all'], -// ]); -// $counter++; -// -// $this->assertNotEmpty($file['body']['$id']); -// $id = $file['body']['$id']; -// } -// @fclose($handle); + //$source = __DIR__ . "/../../../resources/disk-a/large-file.mp4"; + $source = __DIR__ . "/../../../resources/disk-a/very-big-file-2.mov"; + $totalSize = \filesize($source); + $chunkSize = 5 * 1024 * 1024; + $handle = @fopen($source, "rb"); + $fileId = 'unique()'; + $mimeType = mime_content_type($source); + $counter = 0; + $size = filesize($source); + $headers = [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'] + ]; + $id = ''; + + while (!feof($handle)) { + $curlFile = new \CURLFile('data:' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'very-large-file-1.mov'); + $headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size; + + if (!empty($id)) { + $headers['x-appwrite-id'] = $id; + } + + $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket['body']['$id'] . '/files', array_merge($headers, $this->getHeaders()), [ + 'fileId' => $fileId, + 'file' => $curlFile, + 'read' => ['role:all'], + 'write' => ['role:all'], + ]); + $counter++; + + $this->assertNotEmpty($file['body']['$id']); + $id = $file['body']['$id']; + } + @fclose($handle); return [ 'bucketId' => $bucket['body']['$id'], @@ -116,7 +116,7 @@ class VideoCustomServerTest extends Scope /** * @depends testTranscodingRendition */ - public function testGetRenditions($data): void + public function testGetRendition(array $data): array { sleep(30); @@ -125,28 +125,84 @@ class VideoCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'], - ],[ + ], [ 'read' => ['role:all'], 'write' => ['role:all'] ]); - var_dump($response['body']); + + + $this->assertNotEmpty($response['body']['renditions']); + + $videoId = $response['body']['renditions'][0]['videoId']; + $profileId = $response['body']['renditions'][0]['profileId']; + $profileName = $response['body']['renditions'][0]['name']; + $stream = $response['body']['renditions'][0]['stream']; + + + return [ + 'videoId' => $videoId, + 'profileId' => $profileId, + 'profileName' => $profileName, + 'stream' => $stream + ]; + } + + /** + * @depends testGetRendition + */ + public function testHlsStreamRender($data): void + { + sleep(20); + + $response = $this->client->call(Client::METHOD_GET, '/video/' . $data['videoId'] . '/' . $data['stream'] . '/master/' . $data['videoId'] . '.m3u8', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'read' => ['role:all'], + 'write' => ['role:all'] + ]); + + $response = $this->client->call(Client::METHOD_GET, '/video/' . $data['videoId'] . '/' . $data['stream'] . '/' . $data['profileName'] . '/' . $data['videoId'] . '_360p.m3u8', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'read' => ['role:all'], + 'write' => ['role:all'] + ]); + + var_dump($response['body']); + + $response = $this->client->call(Client::METHOD_GET, '/video/' . $data['videoId'] . '/hls/' . $data['profileName'] . '/' . $data['videoId'] . '_360p_0000.ts', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'read' => ['role:all'], + 'write' => ['role:all'] + ]); + + var_dump($response['body']); } /** * @depends testTranscodingRendition */ - public function testPlaylist($data): void + public function testDashStreamRender($data): void { - sleep(60); + sleep(20); - $response = $this->client->call(Client::METHOD_GET, '/video/' . $data['videoId'] . '/master/hls/' . $data['videoId'].'.m3u8', [ + $response = $this->client->call(Client::METHOD_GET, '/video/' . $data['videoId'] . '/mpeg-dash/master/' . $data['videoId'] . '.mpd', [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'], - ],[ + ], [ 'read' => ['role:all'], 'write' => ['role:all'] ]); + + var_dump($response['body']); } }