From 91a307490fa33a597187ac5fa9ab262358bdb368 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 17 Feb 2022 14:58:39 +0545 Subject: [PATCH 01/20] refcator storage usage in worker --- app/init.php | 3 +- app/workers/deletes.php | 23 +------------ src/Appwrite/Resque/Worker.php | 61 ++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 23 deletions(-) diff --git a/app/init.php b/app/init.php index d6feaf981a..876fbba5ea 100644 --- a/app/init.php +++ b/app/init.php @@ -49,6 +49,7 @@ use Swoole\Database\PDOPool; use Swoole\Database\RedisConfig; use Swoole\Database\RedisPool; use Utopia\Database\Query; +use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; @@ -815,7 +816,7 @@ App::setResource('deviceBuilds', function($project) { return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); }, ['project']); -function getDevice($root) { +function getDevice($root): Device { switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { case Storage::DEVICE_LOCAL:default: return new Local($root); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 9cb707f29d..a053be9b17 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -545,28 +545,7 @@ class DeletesV1 extends Worker $bucketId = $document->getId(); $dbForProject = $this->getProjectDB($projectId); $dbForProject->deleteCollection('bucket_' . $bucketId); - - $device = new Local(APP_STORAGE_UPLOADS.'/app-'.$projectId); - - switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { - case Storage::DEVICE_S3: - $s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', ''); - $s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', ''); - $s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', ''); - $s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', ''); - $s3Acl = 'private'; - $device = new S3(APP_STORAGE_UPLOADS . '/app-' . $projectId, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); - break; - case Storage::DEVICE_DO_SPACES: - $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', ''); - $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', ''); - $doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', ''); - $doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', ''); - $doSpacesAcl = 'private'; - $device = new DOSpaces(APP_STORAGE_UPLOADS . '/app-' . $projectId, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); - break; - } - + $device = $this->getFilesDevice($projectId); $device->deletePath($bucketId); } } diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index a8049c85dc..e48d6cfb2f 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -8,8 +8,14 @@ use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Adapter\MariaDB; +use Utopia\Storage\Device; +use Utopia\Storage\Storage; +use Utopia\Storage\Device\Local; +use Utopia\Storage\Device\DOSpaces; +use Utopia\Storage\Device\S3; use Exception; + abstract class Worker { /** @@ -219,4 +225,59 @@ abstract class Worker return $database; } + + /** + * Get Functions Storage Device + * @param string $projectId of the project + * @return Device + */ + protected function getFunctionsDevice($projectId): Device { + return $this->getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId); + } + + /** + * Get Files Storage Device + * @param string $projectId of the project + * @return Device + */ + protected function getFilesDevice($projectId): Device { + return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); + } + + + /** + * Get Builds Storage Device + * @param string $projectId of the project + * @return Device + */ + protected function getBuildsDevice($projectId): Device { + return $this->getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId); + } + + /** + * Get Device based on selected storage environment + * @param string $root path of the device + * @return Device + */ + private function getDevice($root): Device + { + switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { + case Storage::DEVICE_LOCAL:default: + return new Local($root); + case Storage::DEVICE_S3: + $s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', ''); + $s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', ''); + $s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', ''); + $s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', ''); + $s3Acl = 'private'; + return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + case Storage::DEVICE_DO_SPACES: + $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', ''); + $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', ''); + $doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', ''); + $doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', ''); + $doSpacesAcl = 'private'; + return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + } + } } From 0dac3a734629669d2a3da57db664be2b7c62f261 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 17 Feb 2022 17:25:02 +0545 Subject: [PATCH 02/20] get proper source type and device --- app/executor.php | 24 ++++++++++++++++++++++++ app/workers/builds.php | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/app/executor.php b/app/executor.php index c0d36a11cc..19e1e5a4a7 100644 --- a/app/executor.php +++ b/app/executor.php @@ -14,7 +14,10 @@ use Utopia\CLI\Console; use Utopia\Logger\Log; use Utopia\Orchestration\Adapter\DockerCLI; use Utopia\Orchestration\Orchestration; +use Utopia\Storage\Device; use Utopia\Storage\Device\Local; +use Utopia\Storage\Device\DOSpaces; +use Utopia\Storage\Device\S3; use Utopia\Storage\Storage; use Utopia\Swoole\Request; use Utopia\Swoole\Response; @@ -113,6 +116,27 @@ function logError(Throwable $error, string $action, Utopia\Route $route = null) Console::error('[Error] Line: ' . $error->getLine()); }; +function getDevice($root): Device { + switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { + case Storage::DEVICE_LOCAL:default: + return new Local($root); + case Storage::DEVICE_S3: + $s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', ''); + $s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', ''); + $s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', ''); + $s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', ''); + $s3Acl = 'private'; + return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + case Storage::DEVICE_DO_SPACES: + $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', ''); + $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', ''); + $doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', ''); + $doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', ''); + $doSpacesAcl = 'private'; + return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + } +} + App::post('/v1/runtimes') ->desc("Create a new runtime server") ->param('runtimeId', '', new Text(62), 'Unique runtime ID.') diff --git a/app/workers/builds.php b/app/workers/builds.php index 3e70c1978b..0eb6e60e69 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -91,7 +91,7 @@ class BuildsV1 extends Worker 'outputPath' => '', 'runtime' => $function->getAttribute('runtime'), 'source' => $deployment->getAttribute('path'), - 'sourceType' => Storage::DEVICE_LOCAL, + 'sourceType' => App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL), 'stdout' => '', 'stderr' => '', 'endTime' => 0, From 9d5b6ef096a9412d0234bdcf83faf5c07162dce6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 17 Feb 2022 12:47:26 +0000 Subject: [PATCH 03/20] fix permission and other fixes for chunked upload --- app/controllers/api/functions.php | 42 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 0b34f6ce10..bc85b2991e 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -543,14 +543,16 @@ App::post('/v1/functions/:functionId/deployments') $deployment = $dbForProject->getDocument('deployments', $deploymentId); - if(!$deployment->isEmpty()) { + $metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)]; + if (!$deployment->isEmpty()) { $chunks = $deployment->getAttribute('chunksTotal', 1); - if($chunk == -1) { + $metadata = $deployment->getAttribute('metadata', []); + if ($chunk === -1) { $chunk = $chunks; } } - $chunksUploaded = $deviceFunctions->upload($fileTmpName, $path, $chunk, $chunks); + $chunksUploaded = $deviceFunctions->upload($fileTmpName, $path, $chunk, $chunks, $metadata); if (empty($chunksUploaded)) { throw new Exception('Failed moving file', 500); @@ -585,16 +587,28 @@ App::post('/v1/functions/:functionId/deployments') 'size' => $fileSize, 'search' => implode(' ', [$deploymentId, $entrypoint]), 'activate' => ((bool) $activate === true), + 'metadata' => $metadata, ])); } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('size', $fileSize)); + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('size', $fileSize)->setAttribute('metadata', $metadata)); } + // Enqueue a message to start the build + Resque::enqueue(Event::BUILDS_QUEUE_NAME, Event::BUILDS_CLASS_NAME, [ + 'projectId' => $project->getId(), + 'resourceId' => $function->getId(), + 'deploymentId' => $deploymentId, + 'type' => BUILD_TYPE_DEPLOYMENT + ]); + + $usage + ->setParam('storage', $deployment->getAttribute('size', 0)) + ; } else { if($deployment->isEmpty()) { $deployment = $dbForProject->createDocument('deployments', new Document([ '$id' => $deploymentId, - '$read' => [], - '$write' => [], + '$read' => ['role:all'], + '$write' => ['role:all'], 'resourceId' => $function->getId(), 'dateCreated' => time(), 'entrypoint' => $entrypoint, @@ -604,24 +618,14 @@ App::post('/v1/functions/:functionId/deployments') 'chunksUploaded' => $chunksUploaded, 'search' => implode(' ', [$deploymentId, $entrypoint]), 'activate' => ((bool) $activate === true), + 'metadata' => $metadata, ])); } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('chunksUploaded', $chunksUploaded)); + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('chunksUploaded', $chunksUploaded)->setAttribute('metadata', $metadata)); } } - // Enqueue a message to start the build - Resque::enqueue(Event::BUILDS_QUEUE_NAME, Event::BUILDS_CLASS_NAME, [ - 'projectId' => $project->getId(), - 'resourceId' => $function->getId(), - 'deploymentId' => $deploymentId, - 'type' => BUILD_TYPE_DEPLOYMENT - ]); - - $usage - ->setParam('storage', $deployment->getAttribute('size', 0)) - ; - + $metadata = null; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($deployment, Response::MODEL_DEPLOYMENT); From 0b767c0b8acae965fbf102dec5c9f9c18fad6780 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 17 Feb 2022 12:48:00 +0000 Subject: [PATCH 04/20] metadata to deployments --- app/config/collections.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/config/collections.php b/app/config/collections.php index 6721837da8..21570a7a13 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -1844,6 +1844,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'metadata', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, // https://tools.ietf.org/html/rfc4288#section-4.2 + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['json'], + ], [ '$id' => 'search', 'type' => Database::VAR_STRING, From 77babbf6d8de257d51ae8e18caacbd9be1ac4f61 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 17 Feb 2022 13:09:33 +0000 Subject: [PATCH 05/20] trivial fixes --- app/executor.php | 2 +- app/workers/builds.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/executor.php b/app/executor.php index 19e1e5a4a7..aa7a32ab65 100644 --- a/app/executor.php +++ b/app/executor.php @@ -116,7 +116,7 @@ function logError(Throwable $error, string $action, Utopia\Route $route = null) Console::error('[Error] Line: ' . $error->getLine()); }; -function getDevice($root): Device { +function getStorageDevice($root): Device { switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) { case Storage::DEVICE_LOCAL:default: return new Local($root); diff --git a/app/workers/builds.php b/app/workers/builds.php index 0eb6e60e69..8e75f13449 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -39,7 +39,7 @@ class BuildsV1 extends Worker { $type = $this->args['type'] ?? ''; $projectId = $this->args['projectId'] ?? ''; - $functionId = $this->args['functionId'] ?? ''; + $functionId = $this->args['resourceId'] ?? ''; $deploymentId = $this->args['deploymentId'] ?? ''; switch ($type) { From 27a2e80ad1517f56a582638d686dd6ba2c13c34a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 18 Feb 2022 12:36:24 +0000 Subject: [PATCH 06/20] extend exception import --- app/controllers/api/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 1b07945b3f..c9fa410c14 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -3,6 +3,7 @@ use Ahc\Jwt\JWT; use Appwrite\Auth\Auth; use Appwrite\Event\Event; +use Appwrite\Extend\Exception; use Appwrite\Utopia\Database\Validator\CustomId; use Utopia\Database\Validator\UID; use Utopia\Storage\Validator\File; @@ -12,7 +13,6 @@ use Utopia\Storage\Validator\Upload; use Appwrite\Utopia\Response; use Appwrite\Task\Validator\Cron; use Utopia\App; -use Utopia\Exception; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; From 045ba04124bb04f052bf865b175554c26634a0f6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 18 Feb 2022 21:23:48 +0545 Subject: [PATCH 07/20] fix resource type --- app/controllers/api/functions.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index c9fa410c14..42a7748fdd 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -581,6 +581,7 @@ App::post('/v1/functions/:functionId/deployments') '$read' => ['role:all'], '$write' => ['role:all'], 'resourceId' => $function->getId(), + 'resourceType' => 'functions', 'dateCreated' => time(), 'entrypoint' => $entrypoint, 'path' => $path, @@ -610,6 +611,7 @@ App::post('/v1/functions/:functionId/deployments') '$read' => ['role:all'], '$write' => ['role:all'], 'resourceId' => $function->getId(), + 'resourceType' => 'functions', 'dateCreated' => time(), 'entrypoint' => $entrypoint, 'path' => $path, From a086559cb847c36e41ab8358c2320926cfe771ca Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 05:50:00 +0000 Subject: [PATCH 08/20] composer update --- composer.lock | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/composer.lock b/composer.lock index c45b9b7501..2c490269c9 100644 --- a/composer.lock +++ b/composer.lock @@ -3071,17 +3071,11 @@ }, { "name": "appwrite/sdk-generator", - "version": "0.17.2", + "version": "dev-feat-preps-for-0.13", "source": { "type": "git", - "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "37bc6fc1b4b4940c7659748d7d2d5110da5457a4" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/37bc6fc1b4b4940c7659748d7d2d5110da5457a4", - "reference": "37bc6fc1b4b4940c7659748d7d2d5110da5457a4", - "shasum": "" + "url": "https://github.com/appwrite/sdk-generator", + "reference": "b977fcf357a267f41299539ac9095aa7bbcc2600" }, "require": { "ext-curl": "*", @@ -3116,11 +3110,7 @@ } ], "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", - "support": { - "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.17.2" - }, - "time": "2022-01-28T08:25:10+00:00" + "time": "2022-02-15T11:09:40+00:00" }, { "name": "composer/pcre", From 6fb644a8e67d32be721b9571489befa55161ab60 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 05:56:50 +0000 Subject: [PATCH 09/20] update storage usage --- app/executor.php | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/executor.php b/app/executor.php index 4731c440b2..a1c798a3bb 100644 --- a/app/executor.php +++ b/app/executor.php @@ -172,9 +172,10 @@ App::post('/v1/runtimes') /** * Copy code files from source to a temporary location on the executor */ - $device = new Local(); - $buffer = $device->read($source); - if(!$device->write($tmpSource, $buffer)) { + $sourceDevice = getStorageDevice("/"); + $localDevice = new Local(); + $buffer = $sourceDevice->read($source); + if(!$localDevice->write($tmpSource, $buffer)) { throw new Exception('Failed to copy source code to temporary directory', 500); }; @@ -262,11 +263,12 @@ App::post('/v1/runtimes') throw new Exception('Something went wrong during the build process', 500); } - $device = new Local($destination); - $outputPath = $device->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); + $destinationDevice = getStorageDevice($destination); + $localDevice = new Local(); + $outputPath = $destinationDevice->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $buffer = $device->read($tmpBuild); - if(!$device->write($outputPath, $buffer)) { + $buffer = $localDevice->read($tmpBuild); + if(!$destinationDevice->upload($buffer, $outputPath)) { throw new Exception('Failed to move built code to storage', 500); }; From cf86289ca0549e526c4ba5621abb27d21e2666c0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 06:31:06 +0000 Subject: [PATCH 10/20] function deployment limit env --- .env | 1 + app/config/variables.php | 9 +++++++++ app/views/install/compose.phtml | 1 + docker-compose.yml | 1 + 4 files changed, 12 insertions(+) diff --git a/.env b/.env index f3ee0ddef5..755bcb0359 100644 --- a/.env +++ b/.env @@ -34,6 +34,7 @@ _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= _APP_STORAGE_LIMIT=30000000 +_APP_FUNCTIONS_DEPLOYMENT_LIMIT=30000000 _APP_FUNCTIONS_TIMEOUT=900 _APP_FUNCTIONS_BUILD_TIMEOUT=900 _APP_FUNCTIONS_CONTAINERS=10 diff --git a/app/config/variables.php b/app/config/variables.php index 771c080202..82f9ac32c0 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -498,6 +498,15 @@ return [ 'category' => 'Functions', 'description' => '', 'variables' => [ + [ + 'name' => '_APP_FUNCTIONS_DEPLOYMENT_LIMIT', + 'description' => 'The maximum size deployment in bytes.', + 'introduction' => '0.13.0', + 'default' => '30000000', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_FUNCTIONS_TIMEOUT', 'description' => 'The maximum number of seconds allowed as a timeout value when creating a new function. The default value is 900 seconds.', diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 039e1c875f..bd52ab95f9 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -100,6 +100,7 @@ services: - _APP_STORAGE_ANTIVIRUS - _APP_STORAGE_ANTIVIRUS_HOST - _APP_STORAGE_ANTIVIRUS_PORT + - _APP_FUNCTIONS_DEPLOYMENT_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT - _APP_FUNCTIONS_CONTAINERS diff --git a/docker-compose.yml b/docker-compose.yml index d5af584aa2..569112e921 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -127,6 +127,7 @@ services: - _APP_SMTP_PASSWORD - _APP_USAGE_STATS - _APP_STORAGE_LIMIT + - _APP_FUNCTIONS_DEPLOYMENT_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_BUILD_TIMEOUT - _APP_FUNCTIONS_CONTAINERS From 6c49811d14f3cc0ce5c62c4706c587d528e7638f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 06:32:31 +0000 Subject: [PATCH 11/20] use deployment limit instead --- app/controllers/api/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 47f8c47698..97a7fb1735 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -489,7 +489,7 @@ App::post('/v1/functions/:functionId/deployments') $file = $request->getFiles('code'); $fileExt = new FileExt([FileExt::TYPE_GZIP]); - $fileSizeValidator = new FileSize(App::getEnv('_APP_STORAGE_LIMIT', 0)); + $fileSizeValidator = new FileSize(App::getEnv('_APP_FUNCTIONS_DEPLOYMENT_LIMIT', 0)); $upload = new Upload(); if (empty($file)) { From 57c83f8b21e8ec207c03136e80cf02f5156a0c8d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 06:53:20 +0000 Subject: [PATCH 12/20] fix upload using write --- app/executor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/executor.php b/app/executor.php index a1c798a3bb..7b2fdf8f22 100644 --- a/app/executor.php +++ b/app/executor.php @@ -268,7 +268,7 @@ App::post('/v1/runtimes') $outputPath = $destinationDevice->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $buffer = $localDevice->read($tmpBuild); - if(!$destinationDevice->upload($buffer, $outputPath)) { + if(!$destinationDevice->write($outputPath, $buffer, $localDevice->getFileMimeType($tmpBuild))) { throw new Exception('Failed to move built code to storage', 500); }; From ca1b39513a192a1063b53ac2a7e81d5b6535b856 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 16:17:15 +0545 Subject: [PATCH 13/20] Update app/workers/builds.php Co-authored-by: Christy Jacob --- app/workers/builds.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/builds.php b/app/workers/builds.php index 2301e3d6d3..35ab5a5885 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -39,7 +39,7 @@ class BuildsV1 extends Worker { $type = $this->args['type'] ?? ''; $projectId = $this->args['projectId'] ?? ''; - $functionId = $this->args['resourceId'] ?? ''; + $functionId = $this->args['functionId'] ?? ''; $deploymentId = $this->args['deploymentId'] ?? ''; switch ($type) { From 10826daaa7e6acc9cae71f626a8f46632a65b3f5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 16:17:58 +0545 Subject: [PATCH 14/20] Update app/executor.php Co-authored-by: Christy Jacob --- app/executor.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/executor.php b/app/executor.php index 7b2fdf8f22..bc58783ce5 100644 --- a/app/executor.php +++ b/app/executor.php @@ -264,7 +264,6 @@ App::post('/v1/runtimes') } $destinationDevice = getStorageDevice($destination); - $localDevice = new Local(); $outputPath = $destinationDevice->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $buffer = $localDevice->read($tmpBuild); From b0096c94e631c62d8c8cb48be1d4de8821d39deb Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 16:19:51 +0545 Subject: [PATCH 15/20] Update app/controllers/api/functions.php Co-authored-by: Christy Jacob --- app/controllers/api/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 97a7fb1735..74eadb009f 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -375,7 +375,7 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId') } if ($deployment->isEmpty()) { - throw new Exception('Deployment not found', Exception::DEPLOYMENT_NOT_FOUND); + throw new Exception('Deployment not found', 404, Exception::DEPLOYMENT_NOT_FOUND); } if ($build->isEmpty()) { From 289cffa0bb0509aaf616cd7322728e21ce1aea03 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 16:24:07 +0545 Subject: [PATCH 16/20] Update .env Co-authored-by: Christy Jacob --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 755bcb0359..3164961cee 100644 --- a/.env +++ b/.env @@ -34,7 +34,7 @@ _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= _APP_STORAGE_LIMIT=30000000 -_APP_FUNCTIONS_DEPLOYMENT_LIMIT=30000000 +_APP_FUNCTIONS_DEPLOYMENT_LIMIT=30000000 // 30MB _APP_FUNCTIONS_TIMEOUT=900 _APP_FUNCTIONS_BUILD_TIMEOUT=900 _APP_FUNCTIONS_CONTAINERS=10 From 45e6845826a0ecf13cd5420eb35f479423e572a5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 16:24:36 +0545 Subject: [PATCH 17/20] Update app/config/variables.php Co-authored-by: Christy Jacob --- app/config/variables.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/variables.php b/app/config/variables.php index 82f9ac32c0..a105070c87 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -500,7 +500,7 @@ return [ 'variables' => [ [ 'name' => '_APP_FUNCTIONS_DEPLOYMENT_LIMIT', - 'description' => 'The maximum size deployment in bytes.', + 'description' => 'The maximum size deployment in bytes. The default value is 30MB.', 'introduction' => '0.13.0', 'default' => '30000000', 'required' => false, From 372baa51e95a13a548fa689ef1f99837a2921a73 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 11:18:18 +0000 Subject: [PATCH 18/20] fix redis queue --- app/controllers/api/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 74eadb009f..0741f09907 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -598,7 +598,7 @@ App::post('/v1/functions/:functionId/deployments') // Enqueue a message to start the build Resque::enqueue(Event::BUILDS_QUEUE_NAME, Event::BUILDS_CLASS_NAME, [ 'projectId' => $project->getId(), - 'resourceId' => $function->getId(), + 'functionId' => $function->getId(), 'deploymentId' => $deploymentId, 'type' => BUILD_TYPE_DEPLOYMENT ]); From 7b291711192aa761199fb2b7a9b3461e9ccb879e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 11:19:31 +0000 Subject: [PATCH 19/20] fix env --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 3164961cee..755bcb0359 100644 --- a/.env +++ b/.env @@ -34,7 +34,7 @@ _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= _APP_STORAGE_LIMIT=30000000 -_APP_FUNCTIONS_DEPLOYMENT_LIMIT=30000000 // 30MB +_APP_FUNCTIONS_DEPLOYMENT_LIMIT=30000000 _APP_FUNCTIONS_TIMEOUT=900 _APP_FUNCTIONS_BUILD_TIMEOUT=900 _APP_FUNCTIONS_CONTAINERS=10 From 5d9a9f53d82d464b4e1730d9883af97b562764d9 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 22 Feb 2022 11:28:26 +0000 Subject: [PATCH 20/20] update --- app/controllers/api/functions.php | 2 +- app/workers/builds.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 0741f09907..74eadb009f 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -598,7 +598,7 @@ App::post('/v1/functions/:functionId/deployments') // Enqueue a message to start the build Resque::enqueue(Event::BUILDS_QUEUE_NAME, Event::BUILDS_CLASS_NAME, [ 'projectId' => $project->getId(), - 'functionId' => $function->getId(), + 'resourceId' => $function->getId(), 'deploymentId' => $deploymentId, 'type' => BUILD_TYPE_DEPLOYMENT ]); diff --git a/app/workers/builds.php b/app/workers/builds.php index 35ab5a5885..2301e3d6d3 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -39,7 +39,7 @@ class BuildsV1 extends Worker { $type = $this->args['type'] ?? ''; $projectId = $this->args['projectId'] ?? ''; - $functionId = $this->args['functionId'] ?? ''; + $functionId = $this->args['resourceId'] ?? ''; $deploymentId = $this->args['deploymentId'] ?? ''; switch ($type) {