From 100fb7332316a3b5cbbdbd55c6d4b249a88d22e9 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 16 Jun 2022 23:01:42 +0300 Subject: [PATCH] view template --- Dockerfile | 7 +- app/config/collections.php | 126 ++ app/config/renditions.php | 29 + app/controllers/api/video.php | 271 ++++ app/views/video/dash.phtml | 55 + app/views/video/hls.phtml | 7 + app/workers/transcoding.php | 334 +++++ bin/worker-transcoding | 10 + composer.json | 6 +- composer.lock | 1125 +++++++++++++++-- docker-compose.yml | 112 +- src/Appwrite/Event/Transcoding.php | 81 ++ src/Appwrite/Utopia/Response.php | 6 + .../Utopia/Response/Model/FileRendition.php | 89 ++ .../Services/Video/VideoCustomServerTest.php | 133 ++ 15 files changed, 2228 insertions(+), 163 deletions(-) create mode 100644 app/config/renditions.php create mode 100644 app/controllers/api/video.php create mode 100644 app/views/video/dash.phtml create mode 100644 app/views/video/hls.phtml create mode 100644 app/workers/transcoding.php create mode 100644 bin/worker-transcoding create mode 100644 src/Appwrite/Event/Transcoding.php create mode 100644 src/Appwrite/Utopia/Response/Model/FileRendition.php create mode 100644 tests/e2e/Services/Video/VideoCustomServerTest.php diff --git a/Dockerfile b/Dockerfile index e2b991cf6a..70d135746d 100755 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,8 @@ RUN \ yaml-dev \ imagemagick \ imagemagick-dev \ - libmaxminddb-dev + libmaxminddb-dev \ + ffmpeg RUN docker-php-ext-install sockets @@ -234,6 +235,7 @@ RUN \ docker-cli \ docker-compose \ libgomp \ + ffmpeg \ && docker-php-ext-install sockets opcache pdo_mysql \ && apk del .deps \ && rm -rf /var/cache/apk/* @@ -264,12 +266,14 @@ COPY ./src /usr/src/code/src # Set Volumes RUN mkdir -p /storage/uploads && \ + mkdir -p /storage/video && \ mkdir -p /storage/cache && \ mkdir -p /storage/config && \ mkdir -p /storage/certificates && \ mkdir -p /storage/functions && \ mkdir -p /storage/debug && \ chown -Rf www-data.www-data /storage/uploads && chmod -Rf 0755 /storage/uploads && \ + chown -Rf www-data.www-data /storage/video && chmod -Rf 0755 /storage/video && \ chown -Rf www-data.www-data /storage/cache && chmod -Rf 0755 /storage/cache && \ chown -Rf www-data.www-data /storage/config && chmod -Rf 0755 /storage/config && \ chown -Rf www-data.www-data /storage/certificates && chmod -Rf 0755 /storage/certificates && \ @@ -297,6 +301,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-functions && \ chmod +x /usr/local/bin/worker-builds && \ chmod +x /usr/local/bin/worker-mails && \ + chmod +x /usr/local/bin/worker-transcoding && \ chmod +x /usr/local/bin/worker-webhooks # Letsencrypt Permissions diff --git a/app/config/collections.php b/app/config/collections.php index 4c666caa33..c50c51b4dd 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2845,6 +2845,132 @@ $collections = [ ], ] ], + 'videos' => [ + '$collection' => 'video_renditions', + '$id' => 'video_renditions', + '$name' => 'Video_renditions', + 'attributes' => [ + [ + '$id' => 'bucketId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'fileId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'renditionId', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => 0, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'renditionName', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'startedAt', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'endedAt', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'metadata', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, // https://tools.ietf.org/html/rfc4288#section-4.2 + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['json'], + ], + [ + '$id' => 'status', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 100, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'progress', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 4, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'stream', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_bucket_file', + 'type' => Database::INDEX_KEY, + 'attributes' => ['bucketId','fileId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ] + ], ]; return $collections; diff --git a/app/config/renditions.php b/app/config/renditions.php new file mode 100644 index 0000000000..de6d09c930 --- /dev/null +++ b/app/config/renditions.php @@ -0,0 +1,29 @@ + 1, + 'name' => '360p', + 'videoBitrate' => 890, //video bitrate in Kbps + 'audioBitrate' => 64, //audio bitrate in Kbps + 'width' => 640, //width resolution in px + 'height' => 360, //height resolution in px + ], + [ + 'id' => 2, + 'name' => '576p', + 'videoBitrate' => 2538, + 'audioBitrate' => 128, + 'width' => 1024, + 'height' => 576, + ], + [ + 'id' => 3, + 'name' => '720p', + 'videoBitrate' => 3551, + 'audioBitrate' => 128, + 'width' => 1280, + 'height' => 720, + ], + +]; diff --git a/app/controllers/api/video.php b/app/controllers/api/video.php new file mode 100644 index 0000000000..cda290459b --- /dev/null +++ b/app/controllers/api/video.php @@ -0,0 +1,271 @@ +alias('/v1/video/files', ['bucketId' => 'default']) + ->desc('Start transcoding video') + ->groups(['api', 'storage']) + ->label('scope', 'files.write') +// ->label('event', 'buckets.[bucketId].files.[fileId].create') + ->param('bucketId', null, new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).') + ->param('fileId', '', new CustomId(), 'File ID. Choose your own unique ID or pass the string "unique()" to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('read', null, new Permissions(), 'An array of strings with read permissions. By default only the current user is granted with read permissions. [learn more about permissions](https://appwrite.io/docs/permissions) and get a full list of available permissions.', true) + ->param('write', null, new Permissions(), 'An array of strings with write permissions. By default only the current user is granted with write permissions. [learn more about permissions](https://appwrite.io/docs/permissions) and get a full list of available permissions.', true) + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('user') + ->inject('audits') + ->inject('usage') + ->inject('events') + ->inject('mode') + ->inject('deviceFiles') + ->inject('deviceLocal') + ->action(action: function (string $bucketId, string $fileId, ?array $read, ?array $write, Request $request, Response $response, Database $dbForProject, $project, Document $user, Audit $audits, Stats $usage, Event $events, string $mode, Device $deviceFiles, Device $deviceLocal) { + /** @var Utopia\Database\Document $project */ + /** @var Utopia\Database\Document $user */ + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + if ( + $bucket->isEmpty() + || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN) + ) { + throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND); + } + + // Check bucket permissions when enforced + $permissionBucket = $bucket->getAttribute('permission') === 'bucket'; + if ($permissionBucket) { + $validator = new Authorization('write'); + if (!$validator->isValid($bucket->getWrite())) { + throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED); + } + } + + $read = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user + $write = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; + + // Users can only add their roles to files, API keys and Admin users can add any + $roles = Authorization::getRoles(); + + if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles)) { + foreach ($read as $role) { + if (!Authorization::isRole($role)) { + throw new Exception('Read permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED); + } + } + foreach ($write as $role) { + if (!Authorization::isRole($role)) { + throw new Exception('Write permissions must be one of: (' . \implode(', ', $roles) . ')', 400, Exception::USER_UNAUTHORIZED); + } + } + } + + $transcoder = new Transcoding(); + $transcoder + ->setUser($user) + ->setProject($project) + ->setBucketId($bucketId) + ->setFileId($fileId) + ->trigger(); + + $response->json(['result' => 'ok']); + }); + + +App::get('/v1/video/buckets/:bucketId/files/:fileId/renditions') + ->alias('/v1/storage/files/:fileId/renditions', ['bucketId' => 'default']) + ->desc('Get File renditions') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) + ->label('sdk.namespace', 'storage') + ->label('sdk.method', 'getFile') + ->label('sdk.description', '/docs/references/storage/get-file.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_FILE) + ->param('bucketId', null, new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('usage') + ->inject('mode') + ->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, Stats $usage, string $mode) { + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + if ( + $bucket->isEmpty() + || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN) + ) { + throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND); + } + + // Check bucket permissions when enforced + if ($bucket->getAttribute('permission') === 'bucket') { + $validator = new Authorization('read'); + if (!$validator->isValid($bucket->getRead())) { + throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED); + } + } + + if ($bucket->getAttribute('permission') === 'bucket') { + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId)); + } else { + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); + } + + if ($file->isEmpty() || $file->getAttribute('bucketId') !== $bucketId) { + throw new Exception('File not found', 404, Exception::STORAGE_FILE_NOT_FOUND); + } + + $queries = [ + new Query('bucketId', Query::TYPE_EQUAL, [$bucketId]), + new Query('fileId', Query::TYPE_EQUAL, [$fileId]), + new Query('stream', Query::TYPE_EQUAL, ['dash']), + ]; + + $renditions = Authorization::skip(fn () => $dbForProject->find('bucket_' . $bucket->getInternalId() . '_video_renditions', $queries, 12, 0, [], ['ASC'])); + + $response->dynamic(new Document([ + 'total' => $dbForProject->count('bucket_' . $bucket->getInternalId() . '_video_renditions', $queries, APP_LIMIT_COUNT), + 'renditions' => $renditions, + ]), Response::MODEL_FILE_RENDITIONS_LIST); + }); + +App::get('/v1/video/buckets/:bucketId/files/:stream/:fileId') + ->alias('/v1/video/buckets/:bucketId/files/:stream/:fileId', []) + ->desc('Get video playlist manifest') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->param('bucketId', null, new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).') + ->param('stream', '', new WhiteList(['hls', 'dash']), 'stream protocol name') + ->param('fileId', '', new UID(), 'File ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('usage') + ->inject('mode') + ->action(function (string $bucketId, string $stream, string $fileId, Response $response, Database $dbForProject, Stats $usage, string $mode) { + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + if ( + $bucket->isEmpty() + || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN) + ) { + throw new Exception('Bucket not found', 404, Exception::STORAGE_BUCKET_NOT_FOUND); + } + + // Check bucket permissions when enforced + if ($bucket->getAttribute('permission') === 'bucket') { + $validator = new Authorization('read'); + if (!$validator->isValid($bucket->getRead())) { + throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED); + } + } + + if ($bucket->getAttribute('permission') === 'bucket') { + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId)); + } else { + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); + } + + if ($file->isEmpty() || $file->getAttribute('bucketId') !== $bucketId) { + throw new Exception('File not found', 404, Exception::STORAGE_FILE_NOT_FOUND); + } + + $queries = [ + new Query('bucketId', Query::TYPE_EQUAL, [$bucketId]), + new Query('fileId', Query::TYPE_EQUAL, [$fileId]), + new Query('stream', Query::TYPE_EQUAL, [$stream]), + new Query('endedAt', Query::TYPE_GREATER, [0]), + new Query('status', Query::TYPE_EQUAL, ['ready']) + ]; + + $renditions = Authorization::skip(fn () => $dbForProject->find('bucket_' . $bucket->getInternalId() . '_video_renditions', $queries, 12, 0, [], ['ASC'])); + + if (empty($renditions)) { + throw new Exception('Renditions not found'); + } + + if ($stream === 'hls') { + foreach ($renditions as $rendition) { + $metadata = $rendition->getAttribute('metadata'); + $fileId = $rendition->getAttribute('fileId'); + $t['bandwidth'] = (($metadata['general']['video']['bitrate'] + $metadata['general']['audio']['bitrate']) * 1024); + $t['resolution'] = $metadata['general']['resolution']; + $t['name'] = $rendition->getAttribute('renditionName'); + $t['path'] = $rendition->getAttribute('renditionName') . DIRECTORY_SEPARATOR . $fileId . '_' . $rendition->getAttribute('renditionName') . '.m3u8'; + $params[] = $t; + } + + $template = new View(__DIR__ . '/../../views/video/hls.phtml'); + $template->setParam('params', $params); + $response->setContentType('application/x-mpegurl'); + $response->send($template->render()); + } else { + $adaptations = []; + foreach ($renditions as $rendition) { + $metadata = $rendition->getAttribute('metadata'); + foreach ($metadata['dash']['Period']['AdaptationSet'] as $set) { + $adaption = $set['@attributes']; + $adaption['baseUrl'] = $rendition->getAttribute('renditionName') . DIRECTORY_SEPARATOR; + $adaption['representation'] = $set['Representation']['@attributes']; + $adaption['representation']['SegmentTemplate'] = $set['Representation']['SegmentTemplate']['@attributes']; + $adaption['representation']['segmentTemplate']['segmentTimeline'] = $set['Representation']['SegmentTemplate']['SegmentTimeline']; + $adaptations[] = $adaption; + } + } + + $template = new View(__DIR__ . '/../../views/video/dash.phtml'); + $template->setParam('params', $adaptations); + $response->setContentType('application/dash+xml'); + $response->send($template->render()); + } + }); diff --git a/app/views/video/dash.phtml b/app/views/video/dash.phtml new file mode 100644 index 0000000000..9b3847307f --- /dev/null +++ b/app/views/video/dash.phtml @@ -0,0 +1,55 @@ +getParam('params', []); +?> +'; ?> + + + + + getParam('params', []) as $param): ?> + id="" + contentType="" + startWithSAP="" + segmentAlignment="" + bitstreamSwitching="" + frameRate="" + maxWidth="" + par="" + lang="" + > + + id="" + mimeType="" + codecs="" + width="" + height="" + sar="" + audioSamplingRate="" + > + + + + timescale="" + initialization="" + media="" + startNumber="" + > + + + t=" " + d="" + /> + + + + + + + + \ No newline at end of file diff --git a/app/views/video/hls.phtml b/app/views/video/hls.phtml new file mode 100644 index 0000000000..fe3913eda9 --- /dev/null +++ b/app/views/video/hls.phtml @@ -0,0 +1,7 @@ +#EXTM3U +#EXT-X-VERSION:3 +getParam('params', []) as $param): ?> + #EXT-X-STREAM-INF:BANDWIDTH="" ,RESOLUTION="" ,NAME="" + + +} \ No newline at end of file diff --git a/app/workers/transcoding.php b/app/workers/transcoding.php new file mode 100644 index 0000000000..09700a66e5 --- /dev/null +++ b/app/workers/transcoding.php @@ -0,0 +1,334 @@ +basePath .= $this->args['fileId']; + $this->inDir = $this->basePath . '/in/'; + $this->outDir = $this->basePath . '/out/'; + @mkdir($this->inDir, 0755, true); + @mkdir($this->outDir, 0755, true); + $this->outPath = $this->outDir . $this->args['fileId']; /** TODO figure a way to write dir tree without this **/ + } + + public function run(): void + { + $project = new Document($this->args['project']); + $user = new Document($this->args['user'] ?? []); + $this->database = $this->getProjectDB($project->getId()); + + $bucket = Authorization::skip(fn() => $this->database->getDocument('buckets', $this->args['bucketId'])); + + if ($bucket->getAttribute('permission') === 'bucket') { + $file = Authorization::skip(fn() => $this->database->getDocument('bucket_' . $bucket->getInternalId(), $this->args['fileId'])); + } else { + $file = $this->database->getDocument('bucket_' . $bucket->getInternalId(), $this->args['fileId']); + } + + $data = $this->getFilesDevice($project->getId())->read($file->getAttribute('path')); + $fileName = basename($file->getAttribute('path')); + $inPath = $this->inDir . $fileName; + $collection = 'bucket_' . $bucket->getInternalId() . '_video_renditions'; + + if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt + $data = OpenSSL::decrypt( + $data, + $file->getAttribute('openSSLCipher'), + App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + \hex2bin($file->getAttribute('openSSLIV')), + \hex2bin($file->getAttribute('openSSLTag')) + ); + } + + if (!empty($file->getAttribute('algorithm', ''))) { + $compressor = new GZIP(); + $data = $compressor->decompress($data); + } + + $this->getFilesDevice($project->getId())->write($this->inDir . $fileName, $data, $file->getAttribute('mimeType')); + + $ffprobe = FFMpeg\FFProbe::create([]); + $ffmpeg = Streaming\FFMpeg::create([]); + + if (!$ffprobe->isValid($inPath)) { + throw new Exception('Not an valid FFMpeg file "' . $inPath . '"'); + } + + //TODO Can you retranscode? + $queries = [ + new Query('bucketId', Query::TYPE_EQUAL, [$this->args['bucketId']]), + new Query('fileId', Query::TYPE_EQUAL, [$this->args['fileId']]) + ]; + + $renditions = Authorization::skip(fn() => $this->database->find($collection, $queries, 12, 0, [], ['ASC'])); + if (!empty($renditions)) { + foreach ($renditions as $rendition) { + Authorization::skip(fn() => $this->database->deleteDocument($collection, $rendition->getId())); + } + + $deviceFiles = $this->getVideoDevice($project->getId()); + $devicePath = $deviceFiles->getPath($this->args['fileId']); + $devicePath = str_ireplace($deviceFiles->getRoot(), $deviceFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $devicePath); + $deviceFiles->deletePath($devicePath); + } + + $sourceInfo = $this->getVideoInfo($ffprobe->streams($inPath)); + $video = $ffmpeg->open($inPath); + + foreach (Config::getParam('renditions', []) as $rendition) { + foreach (['hls', 'dash'] as $stream) { + $query = Authorization::skip(function () use ($collection, $rendition, $stream) { + return $this->database->createDocument($collection, new Document([ + 'bucketId' => $this->args['bucketId'], + 'fileId' => $this->args['fileId'], + 'renditionId' => $rendition['id'], + 'renditionName' => $rendition['name'], + 'startedAt' => time(), + 'status' => 'started', + 'stream' => $stream, + ])); + }); + + try { + $representation = (new Representation())-> + setKiloBitrate($rendition['videoBitrate'])-> + setAudioKiloBitrate($rendition['audioBitrate'])-> + setResize($rendition['width'], $rendition['height']); + + $format = new Streaming\Format\X264(); + $format->on('progress', function ($video, $format, $percentage) use ($query, $collection) { + if ($percentage % 3 === 0) { + $query->setAttribute('progress', (string)$percentage); + Authorization::skip(fn() => $this->database->updateDocument( + $collection, + $query->getId(), + $query + )); + } + }); + + $metadata = $this->transcode($stream, $video, $format, $representation); + + if (!empty($metadata)) { + $query->setAttribute('metadata', json_encode($metadata)); + } + + $query->setAttribute('status', 'ended'); + $query->setAttribute('endedAt', time()); + Authorization::skip(fn() => $this->database->updateDocument( + $collection, + $query->getId(), + $query + )); + + /** Upload & remove files **/ + $start = 0; + $fileNames = scandir($this->outDir); + + foreach ($fileNames as $fileName) { + if ( + $fileName === '.' || + $fileName === '..' || + str_contains($fileName, '.json') + ) { + continue; + } + + $deviceFiles = $this->getVideoDevice($project->getId()); + $devicePath = $deviceFiles->getPath($this->args['fileId']); + $devicePath = str_ireplace($deviceFiles->getRoot(), $deviceFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $devicePath); + $data = $this->getFilesDevice($project->getId())->read($this->outDir . $fileName); + $this->getVideoDevice($project->getId())->write($devicePath . DIRECTORY_SEPARATOR . $rendition['name'] . DIRECTORY_SEPARATOR . $fileName, $data, \mime_content_type($this->outDir . $fileName)); + + if ($start === 0) { + $query->setAttribute('status', 'uploading'); + Authorization::skip(fn() => $this->database->updateDocument( + $collection, + $query->getId(), + $query + )); + $start = 1; + } + + //$metadata=[]; + //$chunksUploaded = $deviceFiles->upload($file, $path, -1, 1, $metadata); + //var_dump($chunksUploaded); + // if (empty($chunksUploaded)) { + // throw new Exception('Failed uploading file', 500, Exception::GENERAL_SERVER_ERROR); + //} + // } + + @unlink($this->outDir . $fileName); + } + + $query->setAttribute('status', 'ready'); + Authorization::skip(fn() => $this->database->updateDocument( + $collection, + $query->getId(), + $query + )); + } catch (\Throwable $th) { + $query->setAttribute('metadata', json_encode([ + 'code' => $th->getCode(), + 'message' => $th->getMessage(), + ])); + + $query->setAttribute('status', 'error'); + Authorization::skip(fn() => $this->database->updateDocument( + $collection, + $query->getId(), + $query + )); + } + } + } + } + + /** + * @param $metadata array + * @return array + */ + private function getMetadataExport(array $metadata): array + { + $info = []; + + if (!empty($metadata['stream']['resolutions'][0])) { + $general = $metadata['stream']['resolutions'][0]; + $info['resolution'] = $general['dimension']; + } + + if (!empty($metadata['video']['streams'])) { + foreach ($metadata['video']['streams'] as $streams) { + if ($streams['codec_type'] === 'video') { + $info['duration'] = $streams['duration']; + $info['video']['codec'] = $streams['codec_name'] . ',' . $streams['codec_tag_string']; + $info['video']['bitRate'] = $streams['bit_rate']; + $info['video']['frameRate'] = $streams['avg_frame_rate']; + } elseif ($streams['codec_type'] === 'audio') { + $info['audio']['codec'] = $streams['codec_name'] . ',' . $streams['codec_tag_string']; + $info['audio']['bitRate'] = $streams['sample_rate']; + $info['audio']['samplRate'] = $streams['bit_rate']; + } + } + } + + return $info; + } + + /** + * @param string $stream + * @param $video Media + * @param $format StreamFormat + * @param $representation Representation + * @return array + */ + private function transcode(string $stream, Media $video, StreamFormat $format, Representation $representation): string | array + { + + $additionalParams = [ + '-sn', + '-vf', 'scale=iw:-2:force_original_aspect_ratio=increase,setsar=1:1' + ]; + + $segementSize = 6; + + if ($stream === 'dash') { + $dash = $video->dash() + ->setFormat($format) + ->setSegDuration($segementSize) + ->addRepresentation($representation) + ->setAdditionalParams($additionalParams) + ->save($this->outPath); + + $xml = simplexml_load_string( + file_get_contents($this->outDir . $this->args['fileId'] . '.mpd') + ); + + return [ + 'general' => $this->getMetadataExport($dash->metadata()->export()), + 'dash' => !empty($xml) ? json_decode(json_encode((array)$xml), true) : [], + ]; + } + + + $hls = $video->hls() + ->setFormat($format) + ->setHlsTime($segementSize) + ->addRepresentation($representation) + ->setAdditionalParams($additionalParams) + ->setHlsBaseUrl(self::HLS_BASE_URL) + ->save($this->outPath); + + return ['general' => $this->getMetadataExport($hls->metadata()->export())]; + } + + + /** + * @param $streams StreamCollection + * @return array + */ + private function getVideoInfo(StreamCollection $streams): array + { + return [ + 'duration' => $streams->videos()->first()->get('duration'), + 'height' => $streams->videos()->first()->get('height'), + 'width' => $streams->videos()->first()->get('width'), + 'frameRate' => $streams->videos()->first()->get('r_frame_rate'), + 'bitrateKb' => $streams->videos()->first()->get('bit_rate') / 1000, + 'bitrateMb' => $streams->videos()->first()->get('bit_rate') / 1000 / 1000, + ]; + } + + private function cleanup(): bool + { + return \exec("rm -rf {$this->basePath}"); + } + + public function shutdown(): void + { + $this->cleanup(); + } +} diff --git a/bin/worker-transcoding b/bin/worker-transcoding new file mode 100644 index 0000000000..71d2674309 --- /dev/null +++ b/bin/worker-transcoding @@ -0,0 +1,10 @@ +#!/bin/sh + +if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +INTERVAL=0.1 QUEUE='v1-transcoding' APP_INCLUDE='/usr/src/code/app/workers/transcoding.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/composer.json b/composer.json index 6ce1bd218f..576e00ddeb 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", "utopia-php/swoole": "0.3.*", - "utopia-php/storage": "0.9.*", + "utopia-php/storage": "dev-main", "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", "utopia-php/orchestration": "0.4.*", @@ -63,7 +63,9 @@ "chillerlan/php-qrcode": "4.3.3", "adhocore/jwt": "1.1.2", "slickdeals/statsd": "3.1.0", - "squizlabs/php_codesniffer": "^3.6" + "aminyazdanpanah/php-ffmpeg-video-streaming": "^1.2", + "squizlabs/php_codesniffer": "^3.6", + "ext-simplexml": "*" }, "repositories": [ { diff --git a/composer.lock b/composer.lock index 77a95ecc29..0487b79ff2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9450d0e08c0f21141af8d99232d37df9", + "content-hash": "8b3774f0a5532b7ab1aeaf5d8654cdc2", "packages": [ { "name": "adhocore/jwt", @@ -63,6 +63,158 @@ ], "time": "2021-02-20T09:56:44+00:00" }, + { + "name": "alchemy/binary-driver", + "version": "v5.2.0", + "source": { + "type": "git", + "url": "https://github.com/alchemy-fr/BinaryDriver.git", + "reference": "e0615cdff315e6b4b05ada67906df6262a020d22" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/alchemy-fr/BinaryDriver/zipball/e0615cdff315e6b4b05ada67906df6262a020d22", + "reference": "e0615cdff315e6b4b05ada67906df6262a020d22", + "shasum": "" + }, + "require": { + "evenement/evenement": "^3.0|^2.0|^1.0", + "php": ">=5.5", + "psr/log": "^1.0", + "symfony/process": "^2.3|^3.0|^4.0|^5.0" + }, + "require-dev": { + "phpunit/phpunit": "^4.0|^5.0" + }, + "type": "library", + "autoload": { + "psr-0": { + "Alchemy": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Le Goff", + "email": "legoff.n@gmail.com" + }, + { + "name": "Romain Neutron", + "email": "imprec@gmail.com", + "homepage": "http://www.lickmychip.com/" + }, + { + "name": "Phraseanet Team", + "email": "info@alchemy.fr", + "homepage": "http://www.phraseanet.com/" + }, + { + "name": "Jens Hausdorf", + "email": "mail@jens-hausdorf.de", + "homepage": "https://jens-hausdorf.de", + "role": "Maintainer" + } + ], + "description": "A set of tools to build binary drivers", + "keywords": [ + "binary", + "driver" + ], + "support": { + "issues": "https://github.com/alchemy-fr/BinaryDriver/issues", + "source": "https://github.com/alchemy-fr/BinaryDriver/tree/master" + }, + "time": "2020-02-12T19:35:11+00:00" + }, + { + "name": "aminyazdanpanah/php-ffmpeg-video-streaming", + "version": "v1.2.15", + "source": { + "type": "git", + "url": "https://github.com/aminyazdanpanah/PHP-FFmpeg-video-streaming.git", + "reference": "ea5ffdb3ffd6459c846e3b2ccb7b47f793aaa254" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/aminyazdanpanah/PHP-FFmpeg-video-streaming/zipball/ea5ffdb3ffd6459c846e3b2ccb7b47f793aaa254", + "reference": "ea5ffdb3ffd6459c846e3b2ccb7b47f793aaa254", + "shasum": "" + }, + "require": { + "php": "^7.2 || ^8.0", + "php-ffmpeg/php-ffmpeg": "^0.15 || 0.16 || 0.17 || 0.18 || 0.19", + "symfony/filesystem": "^4.0 || ^5.0 || ^6.0" + }, + "require-dev": { + "aws/aws-sdk-php": "^3.0@dev", + "google/cloud-storage": "dev-master", + "microsoft/azure-storage-blob": "dev-master", + "phpunit/phpunit": "^8.4" + }, + "suggest": { + "aminyazdanpanah/handling-file-uploads": "aminyazdanpanah/handling-file-uploads is suggested for uploading and validating a new video.", + "aminyazdanpanah/php-shaka": "aminyazdanpanah/php-shaka is suggested for using DRM and encryption", + "aws/aws-sdk-php": "Allow sending stream content to AWS services like S3", + "google/cloud-storage": "Allow sending stream content to Google services like Google Storage", + "microsoft/azure-storage-blob": "Allow sending stream content to Google services like Google Storage" + }, + "type": "library", + "autoload": { + "files": [ + "src/helpers.php" + ], + "psr-4": { + "Streaming\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Amin Yazdanpanah", + "email": "contact@aminyazdanpanah.com", + "homepage": "https://www.aminyazdanpanah.com" + } + ], + "description": "đŸ“¼ PHP FFMpeg - Video Streaming - DASH, HLS http://video.aminyazdanpanah.com", + "homepage": "https://github.com/aminyazdanpanah/PHP-FFmpeg-video-streaming", + "keywords": [ + "DRM", + "MPD", + "adaptive-http-streaming", + "amazons3", + "clearkey", + "cloud", + "clouds", + "dash", + "dynamic-adaptive-streaming-over-http", + "encoding", + "encryption", + "google-cloud-storage", + "hls", + "http-liv-streaming", + "key-rotation", + "live-streaming", + "m3u8", + "microsoft-azure", + "mp4", + "mpeg-dash", + "s3", + "streaming", + "video", + "video-streaming" + ], + "support": { + "issues": "https://github.com/aminyazdanpanah/PHP-FFmpeg-video-streaming/issues", + "source": "https://github.com/aminyazdanpanah/PHP-FFmpeg-video-streaming/tree/v1.2.15" + }, + "time": "2022-01-14T17:19:59+00:00" + }, { "name": "appwrite/php-clamav", "version": "1.1.0", @@ -479,6 +631,53 @@ ], "time": "2022-01-18T15:43:28+00:00" }, + { + "name": "evenement/evenement", + "version": "v3.0.1", + "source": { + "type": "git", + "url": "https://github.com/igorw/evenement.git", + "reference": "531bfb9d15f8aa57454f5f0285b18bec903b8fb7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/igorw/evenement/zipball/531bfb9d15f8aa57454f5f0285b18bec903b8fb7", + "reference": "531bfb9d15f8aa57454f5f0285b18bec903b8fb7", + "shasum": "" + }, + "require": { + "php": ">=7.0" + }, + "require-dev": { + "phpunit/phpunit": "^6.0" + }, + "type": "library", + "autoload": { + "psr-0": { + "Evenement": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Igor Wiedler", + "email": "igor@wiedler.ch" + } + ], + "description": "ÉvĂ©nement is a very simple event dispatching library for PHP", + "keywords": [ + "event-dispatcher", + "event-emitter" + ], + "support": { + "issues": "https://github.com/igorw/evenement/issues", + "source": "https://github.com/igorw/evenement/tree/master" + }, + "time": "2017-07-23T21:35:13+00:00" + }, { "name": "guzzlehttp/guzzle", "version": "7.4.4", @@ -1110,6 +1309,137 @@ ], "time": "2019-09-10T13:16:29+00:00" }, + { + "name": "neutron/temporary-filesystem", + "version": "3.0.1", + "source": { + "type": "git", + "url": "https://github.com/romainneutron/Temporary-Filesystem.git", + "reference": "55f3d4896eff3bf070e491916e6c564db5e640b5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/romainneutron/Temporary-Filesystem/zipball/55f3d4896eff3bf070e491916e6c564db5e640b5", + "reference": "55f3d4896eff3bf070e491916e6c564db5e640b5", + "shasum": "" + }, + "require": { + "php": ">=5.6", + "symfony/filesystem": "^2.3 || ^3.0 || ^4.0 || ^5.0 || ^6.0" + }, + "require-dev": { + "symfony/phpunit-bridge": "^5.0.4 || ^6.0" + }, + "type": "library", + "autoload": { + "psr-0": { + "Neutron": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Romain Neutron", + "email": "imprec@gmail.com" + } + ], + "description": "Symfony filesystem extension to handle temporary files", + "support": { + "issues": "https://github.com/romainneutron/Temporary-Filesystem/issues", + "source": "https://github.com/romainneutron/Temporary-Filesystem/tree/3.0.1" + }, + "time": "2021-12-14T07:30:33+00:00" + }, + { + "name": "php-ffmpeg/php-ffmpeg", + "version": "v0.19.0", + "source": { + "type": "git", + "url": "https://github.com/PHP-FFMpeg/PHP-FFMpeg.git", + "reference": "22b71931fd1a97207788636b283eee1c0067eff7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/PHP-FFMpeg/PHP-FFMpeg/zipball/22b71931fd1a97207788636b283eee1c0067eff7", + "reference": "22b71931fd1a97207788636b283eee1c0067eff7", + "shasum": "" + }, + "require": { + "alchemy/binary-driver": "^1.5 || ~2.0.0 || ^5.0", + "evenement/evenement": "^3.0 || ^2.0 || ^1.0", + "neutron/temporary-filesystem": "^2.1.1 || ^3.0", + "php": ">=5.5.9", + "symfony/cache": "^3.1 || ^4.0 || ^5.0 || ^6.0" + }, + "require-dev": { + "symfony/phpunit-bridge": "^5.0.4", + "symfony/process": "2.8 || 3.3" + }, + "suggest": { + "php-ffmpeg/extras": "A compilation of common audio & video drivers for PHP-FFMpeg" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "0.x-dev" + } + }, + "autoload": { + "psr-0": { + "FFMpeg": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Romain Neutron", + "email": "imprec@gmail.com", + "homepage": "http://www.lickmychip.com/" + }, + { + "name": "Phraseanet Team", + "email": "info@alchemy.fr", + "homepage": "http://www.phraseanet.com/" + }, + { + "name": "Patrik Karisch", + "email": "patrik@karisch.guru", + "homepage": "http://www.karisch.guru" + }, + { + "name": "Romain Biard", + "email": "romain.biard@gmail.com", + "homepage": "https://www.strime.io/" + }, + { + "name": "Jens Hausdorf", + "email": "hello@jens-hausdorf.de", + "homepage": "https://jens-hausdorf.de" + } + ], + "description": "FFMpeg PHP, an Object Oriented library to communicate with AVconv / ffmpeg", + "keywords": [ + "audio", + "audio processing", + "avconv", + "avprobe", + "ffmpeg", + "ffprobe", + "video", + "video processing" + ], + "support": { + "issues": "https://github.com/PHP-FFMpeg/PHP-FFMpeg/issues", + "source": "https://github.com/PHP-FFMpeg/PHP-FFMpeg/tree/v0.19.0" + }, + "time": "2021-12-20T11:51:26+00:00" + }, { "name": "phpmailer/phpmailer", "version": "v6.6.0", @@ -1188,6 +1518,108 @@ ], "time": "2022-02-28T15:31:21+00:00" }, + { + "name": "psr/cache", + "version": "3.0.0", + "source": { + "type": "git", + "url": "https://github.com/php-fig/cache.git", + "reference": "aa5030cfa5405eccfdcb1083ce040c2cb8d253bf" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/cache/zipball/aa5030cfa5405eccfdcb1083ce040c2cb8d253bf", + "reference": "aa5030cfa5405eccfdcb1083ce040c2cb8d253bf", + "shasum": "" + }, + "require": { + "php": ">=8.0.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Cache\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for caching libraries", + "keywords": [ + "cache", + "psr", + "psr-6" + ], + "support": { + "source": "https://github.com/php-fig/cache/tree/3.0.0" + }, + "time": "2021-02-03T23:26:27+00:00" + }, + { + "name": "psr/container", + "version": "2.0.2", + "source": { + "type": "git", + "url": "https://github.com/php-fig/container.git", + "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/container/zipball/c71ecc56dfe541dbd90c5360474fbc405f8d5963", + "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963", + "shasum": "" + }, + "require": { + "php": ">=7.4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Container\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common Container Interface (PHP FIG PSR-11)", + "homepage": "https://github.com/php-fig/container", + "keywords": [ + "PSR-11", + "container", + "container-interface", + "container-interop", + "psr" + ], + "support": { + "issues": "https://github.com/php-fig/container/issues", + "source": "https://github.com/php-fig/container/tree/2.0.2" + }, + "time": "2021-11-05T16:47:00+00:00" + }, { "name": "psr/http-client", "version": "1.0.1", @@ -1583,16 +2015,16 @@ }, { "name": "squizlabs/php_codesniffer", - "version": "3.6.2", + "version": "3.7.0", "source": { "type": "git", "url": "https://github.com/squizlabs/PHP_CodeSniffer.git", - "reference": "5e4e71592f69da17871dba6e80dd51bce74a351a" + "reference": "a2cd51b45bcaef9c1f2a4bda48f2dd2fa2b95563" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/5e4e71592f69da17871dba6e80dd51bce74a351a", - "reference": "5e4e71592f69da17871dba6e80dd51bce74a351a", + "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/a2cd51b45bcaef9c1f2a4bda48f2dd2fa2b95563", + "reference": "a2cd51b45bcaef9c1f2a4bda48f2dd2fa2b95563", "shasum": "" }, "require": { @@ -1635,29 +2067,204 @@ "source": "https://github.com/squizlabs/PHP_CodeSniffer", "wiki": "https://github.com/squizlabs/PHP_CodeSniffer/wiki" }, - "time": "2021-12-12T21:44:58+00:00" + "time": "2022-06-13T06:31:38+00:00" }, { - "name": "symfony/deprecation-contracts", - "version": "v2.5.1", + "name": "symfony/cache", + "version": "v6.1.1", "source": { "type": "git", - "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "e8b495ea28c1d97b5e0c121748d6f9b53d075c66" + "url": "https://github.com/symfony/cache.git", + "reference": "364fc90734230d936ac2db8e897cc03ec8497bbb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/e8b495ea28c1d97b5e0c121748d6f9b53d075c66", - "reference": "e8b495ea28c1d97b5e0c121748d6f9b53d075c66", + "url": "https://api.github.com/repos/symfony/cache/zipball/364fc90734230d936ac2db8e897cc03ec8497bbb", + "reference": "364fc90734230d936ac2db8e897cc03ec8497bbb", "shasum": "" }, "require": { - "php": ">=7.1" + "php": ">=8.1", + "psr/cache": "^2.0|^3.0", + "psr/log": "^1.1|^2|^3", + "symfony/cache-contracts": "^1.1.7|^2|^3", + "symfony/service-contracts": "^1.1|^2|^3", + "symfony/var-exporter": "^5.4|^6.0" + }, + "conflict": { + "doctrine/dbal": "<2.13.1", + "symfony/dependency-injection": "<5.4", + "symfony/http-kernel": "<5.4", + "symfony/var-dumper": "<5.4" + }, + "provide": { + "psr/cache-implementation": "2.0|3.0", + "psr/simple-cache-implementation": "1.0|2.0|3.0", + "symfony/cache-implementation": "1.1|2.0|3.0" + }, + "require-dev": { + "cache/integration-tests": "dev-master", + "doctrine/dbal": "^2.13.1|^3.0", + "predis/predis": "^1.1", + "psr/simple-cache": "^1.0|^2.0|^3.0", + "symfony/config": "^5.4|^6.0", + "symfony/dependency-injection": "^5.4|^6.0", + "symfony/filesystem": "^5.4|^6.0", + "symfony/http-kernel": "^5.4|^6.0", + "symfony/messenger": "^5.4|^6.0", + "symfony/var-dumper": "^5.4|^6.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Cache\\": "" + }, + "classmap": [ + "Traits/ValueWrapper.php" + ], + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Provides an extended PSR-6, PSR-16 (and tags) implementation", + "homepage": "https://symfony.com", + "keywords": [ + "caching", + "psr6" + ], + "support": { + "source": "https://github.com/symfony/cache/tree/v6.1.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2022-06-06T19:15:01+00:00" + }, + { + "name": "symfony/cache-contracts", + "version": "v3.1.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/cache-contracts.git", + "reference": "2eab7fa459af6d75c6463e63e633b667a9b761d3" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/cache-contracts/zipball/2eab7fa459af6d75c6463e63e633b667a9b761d3", + "reference": "2eab7fa459af6d75c6463e63e633b667a9b761d3", + "shasum": "" + }, + "require": { + "php": ">=8.1", + "psr/cache": "^3.0" + }, + "suggest": { + "symfony/cache-implementation": "" }, "type": "library", "extra": { "branch-alias": { - "dev-main": "2.5-dev" + "dev-main": "3.1-dev" + }, + "thanks": { + "name": "symfony/contracts", + "url": "https://github.com/symfony/contracts" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Contracts\\Cache\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Generic abstractions related to caching", + "homepage": "https://symfony.com", + "keywords": [ + "abstractions", + "contracts", + "decoupling", + "interfaces", + "interoperability", + "standards" + ], + "support": { + "source": "https://github.com/symfony/cache-contracts/tree/v3.1.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2022-02-25T11:15:52+00:00" + }, + { + "name": "symfony/deprecation-contracts", + "version": "v3.1.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/deprecation-contracts.git", + "reference": "07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918", + "reference": "07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.1-dev" }, "thanks": { "name": "symfony/contracts", @@ -1686,7 +2293,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v2.5.1" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.1.0" }, "funding": [ { @@ -1702,7 +2309,70 @@ "type": "tidelift" } ], - "time": "2022-01-02T09:53:40+00:00" + "time": "2022-02-25T11:15:52+00:00" + }, + { + "name": "symfony/filesystem", + "version": "v6.1.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/filesystem.git", + "reference": "3132d2f43ca799c2aa099f9738d98228c56baa5d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/3132d2f43ca799c2aa099f9738d98228c56baa5d", + "reference": "3132d2f43ca799c2aa099f9738d98228c56baa5d", + "shasum": "" + }, + "require": { + "php": ">=8.1", + "symfony/polyfill-ctype": "~1.8", + "symfony/polyfill-mbstring": "~1.8" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Filesystem\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Provides basic utilities for the filesystem", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/filesystem/tree/v6.1.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2022-05-21T13:34:40+00:00" }, { "name": "symfony/polyfill-ctype", @@ -1786,6 +2456,89 @@ ], "time": "2022-05-24T11:49:31+00:00" }, + { + "name": "symfony/polyfill-mbstring", + "version": "v1.26.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-mbstring.git", + "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", + "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "provide": { + "ext-mbstring": "*" + }, + "suggest": { + "ext-mbstring": "For best performance" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "1.26-dev" + }, + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Mbstring\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for the Mbstring extension", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "mbstring", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.26.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2022-05-24T11:49:31+00:00" + }, { "name": "symfony/polyfill-php80", "version": "v1.26.0", @@ -1869,6 +2622,225 @@ ], "time": "2022-05-10T07:21:04+00:00" }, + { + "name": "symfony/process", + "version": "v5.4.8", + "source": { + "type": "git", + "url": "https://github.com/symfony/process.git", + "reference": "597f3fff8e3e91836bb0bd38f5718b56ddbde2f3" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/process/zipball/597f3fff8e3e91836bb0bd38f5718b56ddbde2f3", + "reference": "597f3fff8e3e91836bb0bd38f5718b56ddbde2f3", + "shasum": "" + }, + "require": { + "php": ">=7.2.5", + "symfony/polyfill-php80": "^1.16" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Process\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Executes commands in sub-processes", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/process/tree/v5.4.8" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2022-04-08T05:07:18+00:00" + }, + { + "name": "symfony/service-contracts", + "version": "v3.1.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/service-contracts.git", + "reference": "d66cd8ab656780f62c4215b903a420eb86358957" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/d66cd8ab656780f62c4215b903a420eb86358957", + "reference": "d66cd8ab656780f62c4215b903a420eb86358957", + "shasum": "" + }, + "require": { + "php": ">=8.1", + "psr/container": "^2.0" + }, + "conflict": { + "ext-psr": "<1.1|>=2" + }, + "suggest": { + "symfony/service-implementation": "" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.1-dev" + }, + "thanks": { + "name": "symfony/contracts", + "url": "https://github.com/symfony/contracts" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Contracts\\Service\\": "" + }, + "exclude-from-classmap": [ + "/Test/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Generic abstractions related to writing services", + "homepage": "https://symfony.com", + "keywords": [ + "abstractions", + "contracts", + "decoupling", + "interfaces", + "interoperability", + "standards" + ], + "support": { + "source": "https://github.com/symfony/service-contracts/tree/v3.1.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2022-05-07T08:07:09+00:00" + }, + { + "name": "symfony/var-exporter", + "version": "v6.1.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/var-exporter.git", + "reference": "ce1452317b1210ddfe18d143fa8a09c18f034b89" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/var-exporter/zipball/ce1452317b1210ddfe18d143fa8a09c18f034b89", + "reference": "ce1452317b1210ddfe18d143fa8a09c18f034b89", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "symfony/var-dumper": "^5.4|^6.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\VarExporter\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Allows exporting any serializable PHP data structure to plain PHP code", + "homepage": "https://symfony.com", + "keywords": [ + "clone", + "construct", + "export", + "hydrate", + "instantiate", + "serialize" + ], + "support": { + "source": "https://github.com/symfony/var-exporter/tree/v6.1.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2022-05-27T12:58:07+00:00" + }, { "name": "utopia-php/abuse", "version": "0.7.0", @@ -2684,16 +3656,16 @@ }, { "name": "utopia-php/storage", - "version": "0.9.0", + "version": "dev-main", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "c7912481a56e17cc86358fa8de57309de5e88ef7" + "reference": "83bac7127b038b5c09cdce42e397db5931b0f34f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/c7912481a56e17cc86358fa8de57309de5e88ef7", - "reference": "c7912481a56e17cc86358fa8de57309de5e88ef7", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/83bac7127b038b5c09cdce42e397db5931b0f34f", + "reference": "83bac7127b038b5c09cdce42e397db5931b0f34f", "shasum": "" }, "require": { @@ -2704,6 +3676,7 @@ "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.0.1" }, + "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -2730,9 +3703,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.9.0" + "source": "https://github.com/utopia-php/storage/tree/0.10.0" }, - "time": "2022-05-19T11:05:45+00:00" + "time": "2022-06-03T10:47:29+00:00" }, { "name": "utopia-php/swoole", @@ -2905,21 +3878,21 @@ }, { "name": "webmozart/assert", - "version": "1.10.0", + "version": "1.11.0", "source": { "type": "git", "url": "https://github.com/webmozarts/assert.git", - "reference": "6964c76c7804814a842473e0c8fd15bab0f18e25" + "reference": "11cb2199493b2f8a3b53e7f19068fc6aac760991" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/webmozarts/assert/zipball/6964c76c7804814a842473e0c8fd15bab0f18e25", - "reference": "6964c76c7804814a842473e0c8fd15bab0f18e25", + "url": "https://api.github.com/repos/webmozarts/assert/zipball/11cb2199493b2f8a3b53e7f19068fc6aac760991", + "reference": "11cb2199493b2f8a3b53e7f19068fc6aac760991", "shasum": "" }, "require": { - "php": "^7.2 || ^8.0", - "symfony/polyfill-ctype": "^1.8" + "ext-ctype": "*", + "php": "^7.2 || ^8.0" }, "conflict": { "phpstan/phpstan": "<0.12.20", @@ -2957,9 +3930,9 @@ ], "support": { "issues": "https://github.com/webmozarts/assert/issues", - "source": "https://github.com/webmozarts/assert/tree/1.10.0" + "source": "https://github.com/webmozarts/assert/tree/1.11.0" }, - "time": "2021-03-09T10:59:23+00:00" + "time": "2022-06-03T18:03:27+00:00" } ], "packages-dev": [ @@ -5086,89 +6059,6 @@ ], "time": "2022-04-18T20:38:04+00:00" }, - { - "name": "symfony/polyfill-mbstring", - "version": "v1.26.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", - "shasum": "" - }, - "require": { - "php": ">=7.1" - }, - "provide": { - "ext-mbstring": "*" - }, - "suggest": { - "ext-mbstring": "For best performance" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-main": "1.26-dev" - }, - "thanks": { - "name": "symfony/polyfill", - "url": "https://github.com/symfony/polyfill" - } - }, - "autoload": { - "files": [ - "bootstrap.php" - ], - "psr-4": { - "Symfony\\Polyfill\\Mbstring\\": "" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Nicolas Grekas", - "email": "p@tchwork.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill for the Mbstring extension", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "mbstring", - "polyfill", - "portable", - "shim" - ], - "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.26.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2022-05-24T11:49:31+00:00" - }, { "name": "textalk/websocket", "version": "1.5.7", @@ -5347,7 +6237,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/storage": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -5363,11 +6255,12 @@ "ext-pdo": "*", "ext-openssl": "*", "ext-zlib": "*", - "ext-sockets": "*" + "ext-sockets": "*", + "ext-simplexml": "*" }, "platform-dev": [], "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/docker-compose.yml b/docker-compose.yml index b6e1ed68b2..f4001f53d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,38 +7,14 @@ x-logging: &x-logging logging: driver: 'json-file' options: - max-file: '5' - max-size: '10m' - -x-env-storage: &x-env-storage |- - _APP_STORAGE_DEVICE - _APP_STORAGE_S3_ACCESS_KEY - _APP_STORAGE_S3_SECRET - _APP_STORAGE_S3_REGION - _APP_STORAGE_S3_BUCKET - _APP_STORAGE_DO_SPACES_ACCESS_KEY - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION - _APP_STORAGE_DO_SPACES_BUCKET - _APP_STORAGE_BACKBLAZE_ACCESS_KEY - _APP_STORAGE_BACKBLAZE_SECRET - _APP_STORAGE_BACKBLAZE_REGION - _APP_STORAGE_BACKBLAZE_BUCKET - _APP_STORAGE_DO_SPACES_BUCKET - _APP_STORAGE_LINODE_ACCESS_KEY - _APP_STORAGE_LINODE_SECRET - _APP_STORAGE_LINODE_REGION - _APP_STORAGE_LINODE_BUCKET - _APP_STORAGE_WASABI_ACCESS_KEY - _APP_STORAGE_WASABI_SECRET - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET + max-file: "5" + max-size: 10m version: '3' services: traefik: - image: traefik:2.7 + image: traefik:2.5 <<: *x-logging container_name: appwrite-traefik command: @@ -96,6 +72,7 @@ services: - traefik.http.routers.appwrite_api_https.tls=true volumes: - appwrite-uploads:/storage/uploads:rw + - appwrite-video:/storage/video:rw - appwrite-cache:/storage/cache:rw - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -103,7 +80,7 @@ services: - ./phpunit.xml:/usr/src/code/phpunit.xml - ./tests:/usr/src/code/tests - ./app:/usr/src/code/app - # - ./vendor:/usr/src/code/vendor + - ./vendor:/usr/src/code/vendor #TODO remove when done!! - ./docs:/usr/src/code/docs - ./public:/usr/src/code/public - ./src:/usr/src/code/src @@ -152,11 +129,18 @@ services: - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT - _APP_STORAGE_LIMIT - - _APP_STORAGE_PREVIEW_LIMIT - _APP_STORAGE_ANTIVIRUS - _APP_STORAGE_ANTIVIRUS_HOST - _APP_STORAGE_ANTIVIRUS_PORT - - *x-env-storage + - _APP_STORAGE_DEVICE + - _APP_STORAGE_S3_ACCESS_KEY + - _APP_STORAGE_S3_SECRET + - _APP_STORAGE_S3_REGION + - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_DO_SPACES_ACCESS_KEY + - _APP_STORAGE_DO_SPACES_SECRET + - _APP_STORAGE_DO_SPACES_REGION + - _APP_STORAGE_DO_SPACES_BUCKET - _APP_FUNCTIONS_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT @@ -166,7 +150,6 @@ services: - _APP_FUNCTIONS_MEMORY_SWAP - _APP_FUNCTIONS_RUNTIMES - _APP_EXECUTOR_SECRET - - _APP_EXECUTOR_HOST - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_STATSD_HOST @@ -225,6 +208,40 @@ services: - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + appwrite-worker-transcoding: + entrypoint: worker-transcoding + <<: *x-logging + container_name: appwrite-worker-transcoding + build: + context: . + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + - appwrite-uploads:/storage/uploads:rw + - appwrite-video:/storage/video:rw + - ./tests:/usr/src/code/tests #TODO remove when done!! + - ./vendor:/usr/src/code/vendor #TODO remove when done!! + + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + appwrite-worker-audits: entrypoint: worker-audits <<: *x-logging @@ -293,6 +310,7 @@ services: - mariadb volumes: - appwrite-uploads:/storage/uploads:rw + - appwrite-video:/storage/video:rw - appwrite-cache:/storage/cache:rw - appwrite-functions:/storage/functions:rw - appwrite-builds:/storage/builds:rw @@ -311,11 +329,18 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - *x-env-storage + - _APP_STORAGE_DEVICE + - _APP_STORAGE_S3_ACCESS_KEY + - _APP_STORAGE_S3_SECRET + - _APP_STORAGE_S3_REGION + - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_DO_SPACES_ACCESS_KEY + - _APP_STORAGE_DO_SPACES_SECRET + - _APP_STORAGE_DO_SPACES_REGION + - _APP_STORAGE_DO_SPACES_BUCKET - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET - - _APP_EXECUTOR_HOST appwrite-worker-database: entrypoint: worker-database @@ -365,7 +390,6 @@ services: - _APP_ENV - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - - _APP_EXECUTOR_HOST - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -397,7 +421,6 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST @@ -441,7 +464,6 @@ services: - _APP_DB_PASS - _APP_FUNCTIONS_TIMEOUT - _APP_EXECUTOR_SECRET - - _APP_EXECUTOR_HOST - _APP_USAGE_STATS - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD @@ -488,7 +510,15 @@ services: - OPEN_RUNTIMES_NETWORK - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - - *x-env-storage + - _APP_STORAGE_DEVICE + - _APP_STORAGE_S3_ACCESS_KEY + - _APP_STORAGE_S3_SECRET + - _APP_STORAGE_S3_REGION + - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_DO_SPACES_ACCESS_KEY + - _APP_STORAGE_DO_SPACES_SECRET + - _APP_STORAGE_DO_SPACES_REGION + - _APP_STORAGE_DO_SPACES_BUCKET - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD @@ -539,18 +569,11 @@ services: - redis environment: - _APP_ENV - - _APP_DOMAIN - - _APP_DOMAIN_TARGET - _APP_OPENSSL_KEY_V1 - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_ABUSE @@ -779,6 +802,7 @@ volumes: appwrite-redis: appwrite-cache: appwrite-uploads: + appwrite-video: appwrite-certificates: appwrite-functions: appwrite-builds: diff --git a/src/Appwrite/Event/Transcoding.php b/src/Appwrite/Event/Transcoding.php new file mode 100644 index 0000000000..52a4c87db4 --- /dev/null +++ b/src/Appwrite/Event/Transcoding.php @@ -0,0 +1,81 @@ +bucketId = $bucketId; + + return $this; + } + + /** + * Returns bucketId. + * + * @return null|Document + */ + public function getBucketId(): ?string + { + return $this->bucketId; + } + + /** + * Sets fileId. + * + * @param $fileId string + * @return self + */ + public function setFileId(string $fileId): self + { + $this->fileId = $fileId; + + return $this; + } + + /** + * Returns fileId. + * + * @return null|Document + */ + public function getFileId(): ?string + { + return $this->fileId; + } + + + /** + * Executes the function event and sends it to the functions worker. + * + * @return string|bool + * @throws \InvalidArgumentException + */ + public function trigger(): string|bool + { + return Resque::enqueue($this->queue, $this->class, [ + 'project' => $this->project, + 'user' => $this->user, + 'bucketId' => $this->bucketId, + 'fileId' => $this->fileId, + ]); + } +} diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 8a876ecb4d..fc43c15d27 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -69,6 +69,7 @@ use Appwrite\Utopia\Response\Model\UsageFunctions; use Appwrite\Utopia\Response\Model\UsageProject; use Appwrite\Utopia\Response\Model\UsageStorage; use Appwrite\Utopia\Response\Model\UsageUsers; +use Appwrite\Utopia\Response\Model\FileRendition; /** * @method Response setStatusCode(int $code = 200) @@ -127,6 +128,8 @@ class Response extends SwooleResponse public const MODEL_FILE_LIST = 'fileList'; public const MODEL_BUCKET = 'bucket'; public const MODEL_BUCKET_LIST = 'bucketList'; + public const MODEL_FILE_RENDITION = 'fileRendition'; + public const MODEL_FILE_RENDITIONS_LIST = 'fileRenditionsList'; // Locale public const MODEL_LOCALE = 'locale'; @@ -218,6 +221,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Sessions List', self::MODEL_SESSION_LIST, 'sessions', self::MODEL_SESSION)) ->setModel(new BaseList('Logs List', self::MODEL_LOG_LIST, 'logs', self::MODEL_LOG)) ->setModel(new BaseList('Files List', self::MODEL_FILE_LIST, 'files', self::MODEL_FILE)) + ->setModel(new BaseList('File Renditions List', self::MODEL_FILE_RENDITIONS_LIST, 'renditions', self::MODEL_FILE_RENDITION)) ->setModel(new BaseList('Buckets List', self::MODEL_BUCKET_LIST, 'buckets', self::MODEL_BUCKET)) ->setModel(new BaseList('Teams List', self::MODEL_TEAM_LIST, 'teams', self::MODEL_TEAM)) ->setModel(new BaseList('Memberships List', self::MODEL_MEMBERSHIP_LIST, 'memberships', self::MODEL_MEMBERSHIP)) @@ -290,6 +294,8 @@ class Response extends SwooleResponse ->setModel(new UsageBuckets()) ->setModel(new UsageFunctions()) ->setModel(new UsageProject()) + ->setModel(new FileRendition()) + // Verification // Recovery // Tests (keep last) diff --git a/src/Appwrite/Utopia/Response/Model/FileRendition.php b/src/Appwrite/Utopia/Response/Model/FileRendition.php new file mode 100644 index 0000000000..2376f34453 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/FileRendition.php @@ -0,0 +1,89 @@ +addRule('$id', [ + 'type' => self::TYPE_STRING, + 'description' => 'ID.', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) + ->addRule('bucketId', [ + 'type' => self::TYPE_STRING, + 'description' => 'Bucket ID.', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) + ->addRule('fileId', [ + 'type' => self::TYPE_STRING, + 'description' => 'File ID.', + 'default' => '', + 'example' => 'd5fg5ehg1c168g7c', + ]) + ->addRule('renditionId', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Rendition ID.', + 'default' => '', + 'example' => 3, + ]) + ->addRule('renditionName', [ + 'type' => self::TYPE_STRING, + 'description' => 'Rendition name.', + 'default' => '', + 'example' => '720P', + ]) + ->addRule('startedAt', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Transcoding time started in Unix timestamp.', + 'default' => 0, + 'example' => 1592981220, + ]) + ->addRule('endedAt', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Transcoding time ended in Unix timestamp.', + 'default' => 0, + 'example' => 1592981290, + ]) + ->addRule('status', [ + 'type' => self::TYPE_STRING, + 'description' => 'Rendition transcoding status', + 'default' => '', + 'example' => 'ready', + ]) + ->addRule('progress', [ + 'type' => self::TYPE_STRING, + 'description' => 'Rendition trascoding progress', + 'default' => 0, + 'example' => 88, + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'File rendition'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_FILE_RENDITION; + } +} diff --git a/tests/e2e/Services/Video/VideoCustomServerTest.php b/tests/e2e/Services/Video/VideoCustomServerTest.php new file mode 100644 index 0000000000..9ce7d7ce28 --- /dev/null +++ b/tests/e2e/Services/Video/VideoCustomServerTest.php @@ -0,0 +1,133 @@ +client->call(Client::METHOD_POST, '/storage/buckets', [ +// 'content-type' => 'application/json', +// 'x-appwrite-project' => $this->getProject()['$id'], +// 'x-appwrite-key' => $this->getProject()['apiKey'], +// ], [ +// 'bucketId' => 'unique()', +// 'name' => 'Test Bucket 2', +// 'permission' => 'file', +// 'read' => ['role:all'], +// 'write' => ['role:all'] +// ]); +// +// $source = __DIR__ . "/../../../resources/disk-a/large-file.mp4"; +// $totalSize = \filesize($source); +// $chunkSize = 5 * 1024 * 1024; +// $handle = @fopen($source, "rb"); +// $fileId = 'unique()'; +// $mimeType = mime_content_type($source); +// $counter = 0; +// $size = filesize($source); +// $headers = [ +// 'content-type' => 'multipart/form-data', +// 'x-appwrite-project' => $this->getProject()['$id'] +// ]; +// $id = ''; +// +// while (!feof($handle)) { +// $curlFile = new \CURLFile('data:' . $mimeType . ';base64,' . base64_encode(@fread($handle, $chunkSize)), $mimeType, 'in1.mp4'); +// $headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . min(((($counter * $chunkSize) + $chunkSize) - 1), $size) . '/' . $size; +// +// if (!empty($id)) { +// $headers['x-appwrite-id'] = $id; +// } +// +// $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket['body']['$id'] . '/files', array_merge($headers, $this->getHeaders()), [ +// 'fileId' => $fileId, +// 'file' => $curlFile, +// 'read' => ['role:all'], +// 'write' => ['role:all'], +// ]); +// $counter++; +// $id = $file['body']['$id']; +// } +// @fclose($handle); +// +// $pid = $this->getProject()['$id']; +// $key = $this->getProject()['apiKey']; +// $fid = $id; +// $bid = $bucket['body']['$id']; +// +// var_dump($pid); +// var_dump($key); +// var_dump($fid); +// var_dump($bid); + + + $pid = '62aaf3408decb0f5a0b3'; + $key = '6e8bf2fd07e5206a9b90efbc3dfbf0794a8e35810838e6b906f0c651a510924b8cf0c535b3a4e84b0330344153a4d8d5e413a0d9314f6955a4b2693633ab120d4f674dd668820d4c195d3006bd814003de18dc2161d7ce639a03cd37fd6fa14151445eddb5c9a294ddf16276ec97d56ecb7275eddab3517254bc7201688d8f47'; + $fid = '62aaf3423b91f5660b04'; + $bid = '62aaf340e9c06064668d'; + + $transcoding = $this->client->call(Client::METHOD_POST, '/video/buckets/' . $bid . '/files/' . $fid, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $pid, + 'x-appwrite-key' => $key, + ], [ + 'read' => ['role:all'], + 'write' => ['role:all'] + ]); + + return [ + 'projectId' => $pid, + 'apiKey' => $key, + 'bucketId' => $bid, + 'fileId' => $fid, + ]; + } + + + public function testRenditions(): void + { + + $pid = '62aaf3408decb0f5a0b3'; + $key = '6e8bf2fd07e5206a9b90efbc3dfbf0794a8e35810838e6b906f0c651a510924b8cf0c535b3a4e84b0330344153a4d8d5e413a0d9314f6955a4b2693633ab120d4f674dd668820d4c195d3006bd814003de18dc2161d7ce639a03cd37fd6fa14151445eddb5c9a294ddf16276ec97d56ecb7275eddab3517254bc7201688d8f47'; + $fid = '62aaf3423b91f5660b04'; + $bid = '62aaf340e9c06064668d'; + + + $renditions = $this->client->call(Client::METHOD_GET, '/video/buckets/' . $bid . '/files/' . $fid . '/renditions', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $pid, + 'x-appwrite-key' => $key, + ]); + var_dump($renditions['body']); + } + + + public function testPlaylist(): void + { + + $pid = '62aaf3408decb0f5a0b3'; + $key = '6e8bf2fd07e5206a9b90efbc3dfbf0794a8e35810838e6b906f0c651a510924b8cf0c535b3a4e84b0330344153a4d8d5e413a0d9314f6955a4b2693633ab120d4f674dd668820d4c195d3006bd814003de18dc2161d7ce639a03cd37fd6fa14151445eddb5c9a294ddf16276ec97d56ecb7275eddab3517254bc7201688d8f47'; + $fid = '62aaf3423b91f5660b04'; + $bid = '62aaf340e9c06064668d'; + $stream = 'dash'; + + $renditions = $this->client->call(Client::METHOD_GET, '/video/buckets/' . $bid . '/files/' . $stream . '/' . $fid, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $pid, + 'x-appwrite-key' => $key, + ]); + var_dump($renditions['body']); + } +}