Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

small fix

Browse Source
This commit is contained in:
shimon
2022-07-06 19:12:58 +03:00
parent d8f4d097d1
commit 6c7c5c1891
12 changed files with 297 additions and 323 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+1 -1
View File
@@ -58,7 +58,7 @@ _APP_SMTP_USERNAME=
_APP_SMTP_PASSWORD=
_APP_PHONE_PROVIDER=phone://mock
_APP_PHONE_FROM=+123456789
_APP_STORAGE_LIMIT=30000000
_APP_STORAGE_LIMIT=300000000000
_APP_STORAGE_PREVIEW_LIMIT=20000000
_APP_FUNCTIONS_SIZE_LIMIT=30000000
_APP_FUNCTIONS_TIMEOUT=900
app/config/collections.php
+10 -10
View File
@@ -3239,10 +3239,10 @@ $collections = [
],
[
'$id' => 'videoBitrate',
'type' => Database::VAR_STRING,
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
@@ -3272,10 +3272,10 @@ $collections = [
],
[
'$id' => 'audioBitrate',
'type' => Database::VAR_STRING,
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
@@ -3283,10 +3283,10 @@ $collections = [
],
[
'$id' => 'audioSamplerate',
'type' => Database::VAR_STRING,
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
@@ -3296,7 +3296,7 @@ $collections = [
'$id' => 'metadata',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
'size' => 4096,
'signed' => true,
'required' => false,
'default' => null,
app/config/video-profiles.php
+3
View File
@@ -8,6 +8,7 @@ return [
'audioBitrate' => 64, //audio bitrate in Kbps
'width' => 640, //width resolution in px
'height' => 360, //height resolution in px
'stream' => 'mpeg-dash'
],
[
'name' => '576p',
@@ -15,6 +16,7 @@ return [
'audioBitrate' => 128,
'width' => 1024,
'height' => 576,
'stream' => 'mpeg-dash'
],
[
'name' => '720p',
@@ -22,6 +24,7 @@ return [
'audioBitrate' => 128,
'width' => 1280,
'height' => 720,
'stream' => 'mpeg-dash'
],
];
app/controllers/api/projects.php
+2 -2
View File
@@ -155,7 +155,7 @@ App::post('/v1/projects')
$dbForProject->createCollection($key, $attributes, $indexes);
}
if($dbForProject->exists($dbForProject->getDefaultDatabase(), 'video_profiles')) {
if ($dbForProject->exists($dbForProject->getDefaultDatabase(), 'video_profiles')) {
foreach (Config::getParam('video-profiles', []) as $profile) {
Authorization::skip(function () use ($project, $profile, $dbForProject) {
return $dbForProject->createDocument('video_profiles', new Document([
@@ -164,7 +164,7 @@ App::post('/v1/projects')
'audioBitrate' => $profile['audioBitrate'],
'width' => $profile['width'],
'height' => $profile['height'],
'stream' => 'hls'
'stream' => $profile['stream']
]));
});
}
app/controllers/api/storage.php
-6
View File
@@ -122,7 +122,6 @@ App::post('/v1/storage/buckets')
$bucket = $dbForProject->getDocument('buckets', $bucketId);
$dbForProject->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes);
} catch (Duplicate $th) {
throw new Exception('Bucket already exists', 409, Exception::STORAGE_BUCKET_ALREADY_EXISTS);
}
@@ -474,9 +473,6 @@ App::post('/v1/storage/buckets/:bucketId/files')
});
} else {
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
var_dump('&&&&&&&');
var_dump($fileId);
var_dump($file);
}
$metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)];
@@ -577,7 +573,6 @@ App::post('/v1/storage/buckets/:bucketId/files')
$file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc);
}
} else {
$file = $file
->setAttribute('$read', $read)
->setAttribute('$write', $write)
@@ -614,7 +609,6 @@ App::post('/v1/storage/buckets/:bucketId/files')
->setParam('bucketId', $bucketId)
;
} else {
try {
if ($file->isEmpty()) {
$doc = new Document([
app/controllers/api/video.php
+129 -216
View File
@@ -3,6 +3,8 @@
use Appwrite\Auth\Auth;
use Appwrite\ClamAV\Network;
use Appwrite\Event\Audit;
use Appwrite\Event\Audit as EventAudit;
use Appwrite\Event\Database as EventDatabase;
use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Transcoding;
@@ -44,6 +46,70 @@ use Utopia\Validator\WhiteList;
use Utopia\Swoole\Request;
use Streaming\Representation;
/**
* Validate file Permissions
*
* @param Database $dbForProject
* @param string $bucketId
* @param string $fileId
* @param array|null $read
* @param array|null $write
* @param string $mode
* @return Document $file
* @throws Exception
*/
function validateFilePermissions(Database $dbForProject, string $bucketId, string $fileId, string $mode, ?array $read, ?array $write): Document
{
/** @var Utopia\Database\Document $project */
/** @var Utopia\Database\Document $user */
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if (
$bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)
) {
throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND);
}
// Check bucket permissions when enforced
$permissionBucket = $bucket->getAttribute('permission') === 'bucket';
if ($permissionBucket) {
$validator = new Authorization('write');
if (!$validator->isValid($bucket->getWrite())) {
throw new Exception('Unauthorized file permissions', 401, Exception::USER_UNAUTHORIZED);
}
}
$read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user
$write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? [];
// Users can only add their roles to files, API keys and Admin users can add any
$roles = Authorization::getRoles();
if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles)) {
foreach ($read as $role) {
if (!Authorization::isRole($role)) {
throw new Exception('Read permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED);
}
}
foreach ($write as $role) {
if (!Authorization::isRole($role)) {
throw new Exception('Write permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED);
}
}
}
if ($bucket->getAttribute('permission') === 'bucket') {
// skip authorization
$file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId));
} else {
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
}
return $file;
}
App::get('/v1/video/profiles')
->alias('/v1/video/video/profiles', [])
->desc('Get all video profiles')
@@ -91,53 +157,12 @@ App::post('/v1/video/buckets/:bucketId/files/:fileId')
/** @var Utopia\Database\Document $project */
/** @var Utopia\Database\Document $user */
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
$file = validateFilePermissions($dbForProject, $bucketId, $fileId, $mode, $read, $write);
if (
$bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)
) {
throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND);
}
// Check bucket permissions when enforced
$permissionBucket = $bucket->getAttribute('permission') === 'bucket';
if ($permissionBucket) {
$validator = new Authorization('write');
if (!$validator->isValid($bucket->getWrite())) {
throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED);
}
}
$read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user
$write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? [];
// Users can only add their roles to files, API keys and Admin users can add any
$roles = Authorization::getRoles();
if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles)) {
foreach ($read as $role) {
if (!Authorization::isRole($role)) {
throw new Exception('Read permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED);
}
}
foreach ($write as $role) {
if (!Authorization::isRole($role)) {
throw new Exception('Write permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED);
}
}
}
if ($bucket->getAttribute('permission') === 'bucket') {
// skip authorization
$file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId));
} else {
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
}
try {
$video = Authorization::skip(function () use ($dbForProject, $bucket, $file) {
$video = Authorization::skip(function () use ($dbForProject, $bucketId, $file) {
return $dbForProject->createDocument('videos', new Document([
'bucketId' => $bucket->getId(),
'bucketId' => $bucketId,
'fileId' => $file->getId(),
'size' => $file->getAttribute('sizeOriginal'),
]));
@@ -186,49 +211,7 @@ App::post('/v1/video/:videoId/rendition/:profileId')
throw new Exception('Video not found', 400, Exception::VIDEO_NOT_FOUND);
}
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $video['bucketId']));
if (
$bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)
) {
throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND);
}
// Check bucket permissions when enforced
$permissionBucket = $bucket->getAttribute('permission') === 'bucket';
if ($permissionBucket) {
$validator = new Authorization('write');
if (!$validator->isValid($bucket->getWrite())) {
throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED);
}
}
$read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user
$write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? [];
// Users can only add their roles to files, API keys and Admin users can add any
$roles = Authorization::getRoles();
if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles)) {
foreach ($read as $role) {
if (!Authorization::isRole($role)) {
throw new Exception('Read permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED);
}
}
foreach ($write as $role) {
if (!Authorization::isRole($role)) {
throw new Exception('Write permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED);
}
}
}
if ($bucket->getAttribute('permission') === 'bucket') {
// skip authorization
$file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId']));
} else {
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId']);
}
validateFilePermissions($dbForProject, $video['bucketId'], $video['fileId'], $mode, $read, $write);
$profile = Authorization::skip(fn() => $dbForProject->findOne('video_profiles', [new Query('_uid', Query::TYPE_EQUAL, [$profileId])]));
@@ -261,7 +244,7 @@ App::get('/v1/video/:videoId/:stream/renditions')
->inject('dbForProject')
->inject('usage')
->inject('mode')
->action(function (string $videoId, string $stream, ?array $read, ?array $write, Response $response, Database $dbForProject, Stats $usage, string $mode) {
->action(function (string $videoId, string $stream, ?array $read, ?array $write, Response $response, Database $dbForProject, Stats $usage, string $mode) {
/** @var Utopia\Database\Document $project */
/** @var Utopia\Database\Document $user */
@@ -272,49 +255,7 @@ App::get('/v1/video/:videoId/:stream/renditions')
throw new Exception('Video not found', 400, Exception::VIDEO_NOT_FOUND);
}
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $video['bucketId']));
if (
$bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)
) {
throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND);
}
// Check bucket permissions when enforced
$permissionBucket = $bucket->getAttribute('permission') === 'bucket';
if ($permissionBucket) {
$validator = new Authorization('write');
if (!$validator->isValid($bucket->getWrite())) {
throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED);
}
}
$read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user
$write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? [];
// Users can only add their roles to files, API keys and Admin users can add any
$roles = Authorization::getRoles();
if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles)) {
foreach ($read as $role) {
if (!Authorization::isRole($role)) {
throw new Exception('Read permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED);
}
}
foreach ($write as $role) {
if (!Authorization::isRole($role)) {
throw new Exception('Write permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED);
}
}
}
if ($bucket->getAttribute('permission') === 'bucket') {
// skip authorization
$file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId']));
} else {
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId']);
}
$file = validateFilePermissions($dbForProject, $video['bucketId'], $video['fileId'], $mode, $read, $write);
$queries = [
new Query('videoId', Query::TYPE_EQUAL, [$video->getId()]),
@@ -331,122 +272,94 @@ App::get('/v1/video/:videoId/:stream/renditions')
]), Response::MODEL_VIDEO_RENDITIONS_LIST);
});
App::get('/v1/video/:videoId/:stream/:namespace/:fileName')
->alias('/v1/video/buckets/:bucketId/files/:stream/:fileId', [])
->desc('Get video playlist manifest')
App::get('/v1/video/:videoId/:stream/:profile/:fileName')
->alias('/v1/video/:videoId/:stream/:profile/:fileName', [])
->desc('Get video playlist manifests')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->param('videoId', null, new UID(), 'Video unique ID.')
->param('stream', '', new WhiteList(['hls', 'mpeg-dash']), 'stream protocol name')
->param('namespace', '', new WhiteList(['master']), 'stream protocol name')
->param('profile', '', new Text(18), 'folder name')
->param('fileName', '', new Text(128), 'playlist file name')
->param('read', null, new Permissions(), 'An array of strings with read permissions. By default only the current user is granted with read permissions. [learn more about permissions](https://appwrite.io/docs/permissions) and get a full list of available permissions.', true)
->param('write', null, new Permissions(), 'An array of strings with write permissions. By default only the current user is granted with write permissions. [learn more about permissions](https://appwrite.io/docs/permissions) and get a full list of available permissions.', true)
->inject('response')
->inject('dbForProject')
->inject('videosDevice')
->inject('usage')
->inject('mode')
->action(function (string $videoId, string $stream, string $namespace, ?array $read, ?array $write, Response $response, Database $dbForProject, Stats $usage, string $mode) {
->action(function (string $videoId, string $stream, string $profile, string $fileName, ?array $read, ?array $write, Response $response, Database $dbForProject, Device $videosDevice, Stats $usage, string $mode) {
/** @var Utopia\Database\Document $project */
/** @var Utopia\Database\Document $user */
$video = Authorization::skip(fn() => $dbForProject->findOne('videos', [new Query('_uid', Query::TYPE_EQUAL, [$videoId])]));
$video = Authorization::skip(fn() => $dbForProject->findOne('videos', [
new Query('_uid', Query::TYPE_EQUAL, [$videoId])
]));
if ($video->isEmpty()) {
throw new Exception('Video not found', 400, Exception::VIDEO_NOT_FOUND);
}
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $video['bucketId']));
validateFilePermissions($dbForProject, $video['bucketId'], $video['fileId'], $mode, $read, $write);
if (
$bucket->isEmpty()
|| (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)
) {
throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND);
}
// Check bucket permissions when enforced
$permissionBucket = $bucket->getAttribute('permission') === 'bucket';
if ($permissionBucket) {
$validator = new Authorization('write');
if (!$validator->isValid($bucket->getWrite())) {
throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED);
}
}
$read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user
$write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? [];
// Users can only add their roles to files, API keys and Admin users can add any
$roles = Authorization::getRoles();
if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles)) {
foreach ($read as $role) {
if (!Authorization::isRole($role)) {
throw new Exception('Read permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED);
}
}
foreach ($write as $role) {
if (!Authorization::isRole($role)) {
throw new Exception('Write permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED);
}
}
}
if ($bucket->getAttribute('permission') === 'bucket') {
// skip authorization
$file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId']));
} else {
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId']);
}
$queries = [
$renditions = Authorization::skip(fn () => $dbForProject->find('video_renditions', [
new Query('videoId', Query::TYPE_EQUAL, [$video->getId()]),
new Query('endedAt', Query::TYPE_GREATER, [0]),
new Query('status', Query::TYPE_EQUAL, ['ready']),
new Query('stream', Query::TYPE_EQUAL, ['stream']),
];
$renditions = Authorization::skip(fn () => $dbForProject->find('video_renditions', $queries, 12, 0, [], ['ASC']));
new Query('stream', Query::TYPE_EQUAL, [$stream]),
], 12, 0, [], ['ASC']));
if (empty($renditions)) {
throw new Exception('Renditions not found');
}
if ($stream === 'hls') {
foreach ($renditions as $rendition) {
$metadata = $rendition->getAttribute('metadata');
$fileId = $rendition->getAttribute('fileId');
$t['bandwidth'] = (($metadata['general']['video']['bitrate'] + $metadata['general']['audio']['bitrate']) * 1024);
$t['resolution'] = $metadata['general']['resolution'];
$t['name'] = $rendition->getAttribute('renditionName');
$t['path'] = $rendition->getAttribute('renditionName') . DIRECTORY_SEPARATOR . $fileId . '_' . $rendition->getAttribute('renditionName') . '.m3u8';
$params[] = $t;
}
$ct['m3u8'] = 'application/x-mpegurl';
$ct['mpd'] = 'application/dash+xml';
$ct['ts'] = 'video/MP2T';
$ct['m4s'] = 'video/iso.segment';
$ext = pathinfo($fileName, PATHINFO_EXTENSION);
$baseUrl = 'http://127.0.0.1/v1/video/' . $videoId . '/' . $stream . '/' ;
$template = new View(__DIR__ . '/../../views/video/hls.phtml');
$template->setParam('params', $params);
$response->setContentType('application/x-mpegurl');
$response->send($template->render());
} else {
$adaptations = [];
foreach ($renditions as $rendition) {
$metadata = $rendition->getAttribute('metadata');
foreach ($metadata['dash']['Period']['AdaptationSet'] as $set) {
$adaption = $set['@attributes'];
$adaption['baseUrl'] = $rendition->getAttribute('renditionName') . DIRECTORY_SEPARATOR;
$adaption['representation'] = $set['Representation']['@attributes'];
$adaption['representation']['SegmentTemplate'] = $set['Representation']['SegmentTemplate']['@attributes'];
$adaption['representation']['segmentTemplate']['segmentTimeline'] = $set['Representation']['SegmentTemplate']['SegmentTimeline'];
$adaptations[] = $adaption;
if ($profile === 'master') {
if ($stream === 'hls') {
foreach ($renditions as $rendition) {
$t['bandwidth'] = $rendition->getAttribute('videoBitrate') + $rendition->getAttribute('audioBitrate');
$t['resolution'] = $rendition->getAttribute('width') . 'X' . $rendition->getAttribute('height');
$t['name'] = $rendition->getAttribute('name');
$t['path'] = $baseUrl . $rendition->getAttribute('name') . '/' . $rendition->getAttribute('videoId') . '_' . $rendition->getAttribute('name') . '.m3u8';
$params[] = $t;
}
}
$template = new View(__DIR__ . '/../../views/video/dash.phtml');
$template->setParam('params', $adaptations);
$response->setContentType('application/dash+xml');
$response->send($template->render());
$template = new View(__DIR__ . '/../../views/video/hls.phtml');
$template->setParam('params', $params);
$output = $template->render();
} else {
$adaptations = [];
foreach ($renditions as $rendition) {
$metadata = $rendition->getAttribute('metadata');
foreach ($metadata['mpeg-dash']['Period']['AdaptationSet'] as $set) {
$adaption = $set['@attributes'];
$adaption['baseUrl'] = $baseUrl . $rendition->getAttribute('name') . '/';
$adaption['representation'] = $set['Representation']['@attributes'];
$adaption['representation']['SegmentTemplate'] = $set['Representation']['SegmentTemplate']['@attributes'];
$adaption['representation']['segmentTemplate']['segmentTimeline'] = $set['Representation']['SegmentTemplate']['SegmentTimeline'];
$adaptations[] = $adaption;
}
}
$template = new View(__DIR__ . '/../../views/video/dash.phtml');
$template->setParam('params', $adaptations);
$response->setContentType($ct[$ext]);
$output = $template->render();
}
} else {
$output = $videosDevice->read($videosDevice->getRoot() . '/' . $videoId . '/' . $profile . '/' . $fileName);
}
$response->setContentType($ct[$ext])
->send($output);
});
app/init.php
+4
View File
@@ -915,6 +915,10 @@ App::setResource('deviceFiles', function ($project) {
return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId());
}, ['project']);
App::setResource('videosDevice', function ($project) {
return getDevice(APP_STORAGE_VIDEO . '/app-' . $project->getId());
}, ['project']);
App::setResource('deviceFunctions', function ($project) {
return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId());
}, ['project']);
app/workers/messaging.php
+1 -1
View File
@@ -67,4 +67,4 @@ class MessagingV1 extends Worker
public function shutdown(): void
{
}
}
}
app/workers/transcoding.php
+46 -44
View File
@@ -9,7 +9,6 @@ use Streaming\Metadata;
use Streaming\Representation;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
@@ -33,7 +32,7 @@ class TranscodingV1 extends Worker
const STATUS_PACKAGE_END = 'ready';
const STATUS_ERROR = 'error';
const HLS_BASE_URL = '';
const STREAM_HLS = 'hls';
//protected string $basePath = '/tmp/';
protected string $basePath = '/usr/src/code/tests/tmp/';
@@ -44,6 +43,8 @@ class TranscodingV1 extends Worker
protected string $outPath;
protected string $renditionName;
protected Database $database;
@@ -79,11 +80,9 @@ class TranscodingV1 extends Worker
throw new Exception('profile not found');
}
$user = new Document($this->args['user'] ?? []);
$bucket = Authorization::skip(fn() => $this->database->getDocument('buckets', $sourceVideo['bucketId']));
if ($bucket->getAttribute('permission') === 'bucket') {
$file = Authorization::skip(fn() => $this->database->getDocument('bucket_' . $bucket->getInternalId(), $video['fileId']));
$file = Authorization::skip(fn() => $this->database->getDocument('bucket_' . $bucket->getInternalId(), $sourceVideo['fileId']));
} else {
$file = $this->database->getDocument('bucket_' . $bucket->getInternalId(), $sourceVideo['fileId']);
}
@@ -134,9 +133,8 @@ class TranscodingV1 extends Worker
}
}
$stream = !empty($rendition['stream']) ? $rendition['stream'] : 'hls';
$general = $this->getVideoInfo($ffprobe->streams($inPath));
if(!empty($general)) {
if (!empty($general)) {
foreach ($general as $key => $value) {
$sourceVideo->setAttribute($key, $value);
}
@@ -148,17 +146,17 @@ class TranscodingV1 extends Worker
));
}
$video = $ffmpeg->open($inPath);
$this->setRenditionName($profile);
$query = Authorization::skip(function () use ($collection, $profile, $stream) {
$query = Authorization::skip(function () use ($collection, $profile) {
return $this->database->createDocument($collection, new Document([
'videoId' => $this->args['videoId'],
'profileId' => $profile->getId(),
'name' => $profile->getAttribute('name'),
'name' => $this->getRenditionName(),
'startedAt' => time(),
'status' => self::STATUS_TRANSCODE_START,
'stream' => $stream,
'stream' => $profile['stream'],
]));
});
@@ -181,8 +179,7 @@ class TranscodingV1 extends Worker
});
list($general, $metadata) = $this->transcode($stream, $video, $format, $representation);
list($general, $metadata) = $this->transcode($profile['stream'], $video, $format, $representation);
if (!empty($metadata)) {
$query->setAttribute('metadata', json_encode($metadata));
}
@@ -214,13 +211,11 @@ class TranscodingV1 extends Worker
continue;
}
$deviceFiles = $this->getVideoDevice($project->getId());
$devicePath = $deviceFiles->getPath($this->args['videoId']);
$deviceFiles = $this->getVideoDevice($project->getId());
$devicePath = $deviceFiles->getPath($this->args['videoId']);
$data = $this->getFilesDevice($project->getId())->read($this->outDir . $fileName);
$renditionDir = $profile->getAttribute('width') . 'X' . $profile->getAttribute('height') . '@' . $profile->getAttribute('videoBitrate');
$renditionPath = $devicePath . DIRECTORY_SEPARATOR . $renditionDir;
$renditionPath = $devicePath . DIRECTORY_SEPARATOR . $this->getRenditionName();
$this->getVideoDevice($project->getId())->write($renditionPath . DIRECTORY_SEPARATOR . $fileName, $data, \mime_content_type($this->outDir . $fileName));
if ($start === 0) {
$query->setAttribute('status', self::STATUS_UPLOADING);
$query->setAttribute('path', $renditionPath);
@@ -275,13 +270,14 @@ class TranscodingV1 extends Worker
{
$additionalParams = [
'-dn',
'-sn',
'-vf', 'scale=iw:-2:force_original_aspect_ratio=increase,setsar=1:1'
];
$segementSize = 10;
if ($stream === 'dash') {
if ($stream === 'mpeg-dash') {
$dash = $video->dash()
->setFormat($format)
->setSegDuration($segementSize)
@@ -290,14 +286,14 @@ class TranscodingV1 extends Worker
->save($this->outPath);
$xml = simplexml_load_string(
file_get_contents($this->outDir . $this->args['fileId'] . '.mpd')
file_get_contents($this->outDir . $this->args['videoId'] . '.mpd')
);
$metadata = $this->getVideoInfo($dash->metadata());
$metadata['width'] = $representation->getWidth();
$metadata['height'] = $representation->getHeight();
$general = $this->getVideoInfo($dash->metadata()->getVideoStreams());
$general['width'] = $representation->getWidth();
$general['height'] = $representation->getHeight();
return [
$metadata,
$general,
['mpeg-dash' => !empty($xml) ? json_decode(json_encode((array)$xml), true) : []],
];
}
@@ -308,31 +304,35 @@ class TranscodingV1 extends Worker
->setHlsTime($segementSize)
->addRepresentation($representation)
->setAdditionalParams($additionalParams)
->setHlsBaseUrl(self::HLS_BASE_URL)
->setHlsBaseUrl('http://127.0.0.1/v1/video/' . $this->args['videoId'] . '/' . self::STREAM_HLS . '/' . $this->getRenditionName() . '/')
->save($this->outPath);
$metadata = $this->getVideoInfo($hls->metadata()->getVideoStreams());
$metadata['width'] = $representation->getWidth();
$metadata['height'] = $representation->getHeight();
$general = $this->getVideoInfo($hls->metadata()->getVideoStreams());
$general['width'] = $representation->getWidth();
$general['height'] = $representation->getHeight();
$this->rewriteHlsRefs($this->outPath . '_' . $representation->getHeight() . 'P.m3u8');
//$this->rewriteHlsLines($this->outPath . '_' . $this->getRenditionName() . '.m3u8');
return [
$metadata, []
$general, []
];
}
private function rewriteHlsRefs($path)
private function rewriteHlsLines($path)
{
$handle = fopen($path, "r");
$destination = fopen($path . '_tmp', "w");
if ($handle) {
while (($line = fgets($handle)) !== false) {
$newLine = str_replace(array("\r","\n"), "", $line);
if (str_contains($line, ".ts")) {
//$return['data'][$i]['url'] = str_replace(array("\r","\n"),"",$line);
$newLine = 'http://127.0.0.1/v1/video/' . $this->args['videoId'] . '/' . self::STREAM_HLS . '/' . $this->getRenditionName() . '/' . $newLine;
var_dump($newLine);
}
fwrite($destination, str_replace(array("\r","\n"), "", $line) . PHP_EOL);
fwrite($destination, $newLine . PHP_EOL);
}
var_dump($path . '_tmp -> ' . $path);
rename($path . '_tmp', $path);
fclose($handle);
fclose($destination);
}
@@ -344,17 +344,6 @@ class TranscodingV1 extends Worker
*/
private function getVideoInfo(StreamCollection $streams): array
{
var_dump([
'duration' => $streams->videos()->first()->get('duration'),
'height' => $streams->videos()->first()->get('height'),
'width' => $streams->videos()->first()->get('width'),
'videoCodec' => $streams->videos()->first()->get('codec_name') . ',' . $streams->videos()->first()->get('codec_tag_string'),
'videoFramerate' => $streams->videos()->first()->get('avg_frame_rate'),
'videoBitrate' => (int)$streams->videos()->first()->get('bit_rate'),
'audioCodec' => $streams->audios()->first()->get('codec_name') . ',' . $streams->audios()->first()->get('codec_tag_string'),
'audioSamplerate' => (int)$streams->audios()->first()->get('sample_rate'),
'audioBitrate' => (int)$streams->audios()->first()->get('bit_rate'),
]);
return [
'duration' => $streams->videos()->first()->get('duration'),
'height' => $streams->videos()->first()->get('height'),
@@ -368,6 +357,19 @@ class TranscodingV1 extends Worker
];
}
private function setRenditionName($profile)
{
$this->renditionName = $profile->getAttribute('width')
. 'X' . $profile->getAttribute('height')
. '@' . ($profile->getAttribute('videoBitrate') + $profile->getAttribute('audioBitrate'));
}
private function getRenditionName(): string
{
return $this->renditionName;
}
private function cleanup(): bool
{
var_dump("rm -rf {$this->basePath}");
src/Appwrite/Event/Event.php
+3
View File
@@ -32,6 +32,9 @@ class Event
public const BUILDS_QUEUE_NAME = 'v1-builds';
public const BUILDS_CLASS_NAME = 'BuildsV1';
public const TRANSCODING_QUEUE_NAME = 'v1-transcoding';
public const TRANSCODING_CLASS_NAME = 'TranscodingV1';
public const MESSAGING_QUEUE_NAME = 'v1-messaging';
public const MESSAGING_CLASS_NAME = 'MessagingV1';
src/Appwrite/Event/Transcoding.php
-1
View File
@@ -7,7 +7,6 @@ use Utopia\Database\Document;
class Transcoding extends Event
{
protected string $videoId = '';
protected string $profileId = '';
tests/e2e/Services/Video/VideoCustomServerTest.php
+98 -42
View File
@@ -27,41 +27,41 @@ class VideoCustomServerTest extends Scope
'write' => ['role:all']
]);
// //$source = __DIR__ . "/../../../resources/disk-a/large-file.mp4";
// $source = __DIR__ . "/../../../resources/disk-a/very-large-file-1.mov";
// $totalSize = \filesize($source);
// $chunkSize = 5 * 1024 * 1024;
// $handle = @fopen($source, "rb");
// $fileId = 'unique()';
// $mimeType = mime_content_type($source);
// $counter = 0;
// $size = filesize($source);
// $headers = [
// 'content-type' => 'multipart/form-data',
// 'x-appwrite-project' => $this->getProject()['$id']
// ];
// $id = '';
//
// while (!feof($handle)) {
// $curlFile = new \CURLFile('data:' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'very-large-file-1.mov');
// $headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size;
//
// if (!empty($id)) {
// $headers['x-appwrite-id'] = $id;
// }
//
// $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket['body']['$id'] . '/files', array_merge($headers, $this->getHeaders()), [
// 'fileId' => $fileId,
// 'file' => $curlFile,
// 'read' => ['role:all'],
// 'write' => ['role:all'],
// ]);
// $counter++;
//
// $this->assertNotEmpty($file['body']['$id']);
// $id = $file['body']['$id'];
// }
// @fclose($handle);
//$source = __DIR__ . "/../../../resources/disk-a/large-file.mp4";
$source = __DIR__ . "/../../../resources/disk-a/very-big-file-2.mov";
$totalSize = \filesize($source);
$chunkSize = 5 * 1024 * 1024;
$handle = @fopen($source, "rb");
$fileId = 'unique()';
$mimeType = mime_content_type($source);
$counter = 0;
$size = filesize($source);
$headers = [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id']
];
$id = '';
while (!feof($handle)) {
$curlFile = new \CURLFile('data:' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'very-large-file-1.mov');
$headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size;
if (!empty($id)) {
$headers['x-appwrite-id'] = $id;
}
$file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket['body']['$id'] . '/files', array_merge($headers, $this->getHeaders()), [
'fileId' => $fileId,
'file' => $curlFile,
'read' => ['role:all'],
'write' => ['role:all'],
]);
$counter++;
$this->assertNotEmpty($file['body']['$id']);
$id = $file['body']['$id'];
}
@fclose($handle);
return [
'bucketId' => $bucket['body']['$id'],
@@ -116,7 +116,7 @@ class VideoCustomServerTest extends Scope
/**
* @depends testTranscodingRendition
*/
public function testGetRenditions($data): void
public function testGetRendition(array $data): array
{
sleep(30);
@@ -125,28 +125,84 @@ class VideoCustomServerTest extends Scope
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
],[
], [
'read' => ['role:all'],
'write' => ['role:all']
]);
var_dump($response['body']);
$this->assertNotEmpty($response['body']['renditions']);
$videoId = $response['body']['renditions'][0]['videoId'];
$profileId = $response['body']['renditions'][0]['profileId'];
$profileName = $response['body']['renditions'][0]['name'];
$stream = $response['body']['renditions'][0]['stream'];
return [
'videoId' => $videoId,
'profileId' => $profileId,
'profileName' => $profileName,
'stream' => $stream
];
}
/**
* @depends testGetRendition
*/
public function testHlsStreamRender($data): void
{
sleep(20);
$response = $this->client->call(Client::METHOD_GET, '/video/' . $data['videoId'] . '/' . $data['stream'] . '/master/' . $data['videoId'] . '.m3u8', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'read' => ['role:all'],
'write' => ['role:all']
]);
$response = $this->client->call(Client::METHOD_GET, '/video/' . $data['videoId'] . '/' . $data['stream'] . '/' . $data['profileName'] . '/' . $data['videoId'] . '_360p.m3u8', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'read' => ['role:all'],
'write' => ['role:all']
]);
var_dump($response['body']);
$response = $this->client->call(Client::METHOD_GET, '/video/' . $data['videoId'] . '/hls/' . $data['profileName'] . '/' . $data['videoId'] . '_360p_0000.ts', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'read' => ['role:all'],
'write' => ['role:all']
]);
var_dump($response['body']);
}
/**
* @depends testTranscodingRendition
*/
public function testPlaylist($data): void
public function testDashStreamRender($data): void
{
sleep(60);
sleep(20);
$response = $this->client->call(Client::METHOD_GET, '/video/' . $data['videoId'] . '/master/hls/' . $data['videoId'].'.m3u8', [
$response = $this->client->call(Client::METHOD_GET, '/video/' . $data['videoId'] . '/mpeg-dash/master/' . $data['videoId'] . '.mpd', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
],[
], [
'read' => ['role:all'],
'write' => ['role:all']
]);
var_dump($response['body']);
}
}