From 3f0857b36de5132db32ba101d9c227b30fbeabae Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 14 Apr 2025 10:28:29 +0000 Subject: [PATCH 1/9] Add plan to create deployment --- app/controllers/api/functions.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index c7f24f984a..0bb7523181 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1278,7 +1278,8 @@ App::post('/v1/functions/:functionId/deployments') ->inject('deviceForFunctions') ->inject('deviceForLocal') ->inject('queueForBuilds') - ->action(function (string $functionId, ?string $entrypoint, ?string $commands, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Event $queueForEvents, Document $project, Device $deviceForFunctions, Device $deviceForLocal, Build $queueForBuilds) { + ->inject('plan') + ->action(function (string $functionId, ?string $entrypoint, ?string $commands, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Event $queueForEvents, Document $project, Device $deviceForFunctions, Device $deviceForLocal, Build $queueForBuilds, array $plan) { $activate = \strval($activate) === 'true' || \strval($activate) === '1'; @@ -1311,8 +1312,9 @@ App::post('/v1/functions/:functionId/deployments') throw new Exception(Exception::STORAGE_FILE_EMPTY, 'No file sent'); } + $functionFileSize = isset($plan['functionSize']) ? $plan['functionSize'] * 1000 * 1000 : (System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT') ?: 30000000); $fileExt = new FileExt([FileExt::TYPE_GZIP]); - $fileSizeValidator = new FileSize(System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000')); + $fileSizeValidator = new FileSize($functionFileSize); $upload = new Upload(); // Make sure we handle a single file and multiple files the same way From c4509f5ce74f53088b64b203734a93ab5c43a775 Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 14 Apr 2025 10:31:08 +0000 Subject: [PATCH 2/9] Add plan to builds worker --- src/Appwrite/Platform/Workers/Builds.php | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 4057d4b190..5be7a0e526 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -60,8 +60,9 @@ class Builds extends Action ->inject('isResourceBlocked') ->inject('log') ->inject('executor') - ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log, Executor $executor) => - $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log, $executor)); + ->inject('plan') + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log, Executor $executor, array $plan) => + $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log, $executor, $plan)); } /** @@ -78,10 +79,11 @@ class Builds extends Action * @param Device $deviceForFunctions * @param Log $log * @param Executor $executor + * @param array $plan * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log, Executor $executor): void + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log, Executor $executor, array $plan): void { $payload = $message->getPayload() ?? []; @@ -102,7 +104,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($deviceForFunctions, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log, $executor); + $this->buildDeployment($deviceForFunctions, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log, $executor, $plan); break; default: @@ -126,11 +128,12 @@ class Builds extends Action * @param Document $template * @param Log $log * @param Executor $executor + * @param array $plan * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log, Executor $executor): void + protected function buildDeployment(Device $deviceForFunctions, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log, Executor $executor, array $plan): void { $functionId = $function->getId(); $log->addTag('functionId', $function->getId()); @@ -401,7 +404,11 @@ class Builds extends Action } $directorySize = $localDevice->getDirectorySize($tmpDirectory); - $functionsSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000'); + + $functionsSizeLimit = empty($plan['functionSize']) + ? (int) System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', 30000000) + : (int) $plan['functionSize'] * 1000 * 1000; + if ($directorySize > $functionsSizeLimit) { throw new \Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / 1048576, 2) . ' MBs.'); } From 0df1396d525e2faf0e9838ba03c9d199d2c20efb Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 14 Apr 2025 11:54:40 +0000 Subject: [PATCH 3/9] Add plan to worker.php --- app/worker.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/worker.php b/app/worker.php index 6a51ee55be..4e25d7f365 100644 --- a/app/worker.php +++ b/app/worker.php @@ -267,6 +267,10 @@ Server::setResource('timelimit', function (\Redis $redis) { Server::setResource('log', fn () => new Log()); +Server::setResource('plan', function (array $plan = []) { + return []; +}); + Server::setResource('publisher', function (Group $pools) { return $pools->get('publisher')->pop()->getResource(); }, ['pools']); From f02707627f5511e5d807ddcc85999fa29e85e3d8 Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 14 Apr 2025 12:16:50 +0000 Subject: [PATCH 4/9] Refactor --- app/controllers/api/functions.php | 7 ++++++- src/Appwrite/Platform/Workers/Builds.php | 8 +++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 0bb7523181..5849de9dd9 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1312,7 +1312,12 @@ App::post('/v1/functions/:functionId/deployments') throw new Exception(Exception::STORAGE_FILE_EMPTY, 'No file sent'); } - $functionFileSize = isset($plan['functionSize']) ? $plan['functionSize'] * 1000 * 1000 : (System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT') ?: 30000000); + $functionFileSize = (System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT') ?: 30000000); + + if (isset($plan['functionSize'])) { + $functionFileSize = $plan['functionSize'] * 1000 * 1000; + } + $fileExt = new FileExt([FileExt::TYPE_GZIP]); $fileSizeValidator = new FileSize($functionFileSize); $upload = new Upload(); diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 5be7a0e526..d838152892 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -405,9 +405,11 @@ class Builds extends Action $directorySize = $localDevice->getDirectorySize($tmpDirectory); - $functionsSizeLimit = empty($plan['functionSize']) - ? (int) System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', 30000000) - : (int) $plan['functionSize'] * 1000 * 1000; + $functionsSizeLimit = (int) (System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT') ?: 30000000); + + if (!empty($plan['functionSize'])) { + $functionsSizeLimit = (int) $plan['functionSize'] * 1000 * 1000; + } if ($directorySize > $functionsSizeLimit) { throw new \Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / 1048576, 2) . ' MBs.'); From 1acbff316e08f405ce39ce01823126528908eca7 Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 14 Apr 2025 13:46:52 +0000 Subject: [PATCH 5/9] Use 1000 instead of 1024 --- src/Appwrite/Platform/Workers/Builds.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index d838152892..f86f776ffb 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -412,7 +412,7 @@ class Builds extends Action } if ($directorySize > $functionsSizeLimit) { - throw new \Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / 1048576, 2) . ' MBs.'); + throw new \Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / (1000 * 1000), 2) . ' MBs.'); } Console::execute('find ' . \escapeshellarg($tmpDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr); @@ -630,7 +630,7 @@ class Builds extends Action $buildSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_BUILD_SIZE_LIMIT', '2000000000'); if ($response['size'] > $buildSizeLimit) { - throw new \Exception('Build size should be less than ' . number_format($buildSizeLimit / 1048576, 2) . ' MBs.'); + throw new \Exception('Build size should be less than ' . number_format($buildSizeLimit / (1000 * 1000), 2) . ' MBs.'); } /** Update the build document */ From 81c3d966297e6cd9582b0a4b360bc6b672277c82 Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 14 Apr 2025 14:00:44 +0000 Subject: [PATCH 6/9] Consistent naming --- app/controllers/api/functions.php | 6 +++--- src/Appwrite/Platform/Workers/Builds.php | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 5849de9dd9..abf7b58e59 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1312,14 +1312,14 @@ App::post('/v1/functions/:functionId/deployments') throw new Exception(Exception::STORAGE_FILE_EMPTY, 'No file sent'); } - $functionFileSize = (System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT') ?: 30000000); + $functionSizeLimit = System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000'); if (isset($plan['functionSize'])) { - $functionFileSize = $plan['functionSize'] * 1000 * 1000; + $functionSizeLimit = $plan['functionSize'] * 1000 * 1000; } $fileExt = new FileExt([FileExt::TYPE_GZIP]); - $fileSizeValidator = new FileSize($functionFileSize); + $fileSizeValidator = new FileSize($functionSizeLimit); $upload = new Upload(); // Make sure we handle a single file and multiple files the same way diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index f86f776ffb..f6e8726c07 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -405,9 +405,9 @@ class Builds extends Action $directorySize = $localDevice->getDirectorySize($tmpDirectory); - $functionsSizeLimit = (int) (System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT') ?: 30000000); + $functionsSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000'); - if (!empty($plan['functionSize'])) { + if (isset($plan['functionSize'])) { $functionsSizeLimit = (int) $plan['functionSize'] * 1000 * 1000; } From 6852ee1f0f325ea0953df5cda60dde4fd67a0436 Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 14 Apr 2025 14:07:23 +0000 Subject: [PATCH 7/9] Read buildSize from plan --- src/Appwrite/Platform/Workers/Builds.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index f6e8726c07..f7395a214a 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -629,6 +629,9 @@ class Builds extends Action $durationEnd = \microtime(true); $buildSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_BUILD_SIZE_LIMIT', '2000000000'); + if (isset($plan['buildSize'])) { + $buildSizeLimit = $plan['buildSize'] * 1000 * 1000 * 1000; + } if ($response['size'] > $buildSizeLimit) { throw new \Exception('Build size should be less than ' . number_format($buildSizeLimit / (1000 * 1000), 2) . ' MBs.'); } From 5d9bdd4cc7480b4d58275af1f71e4da959fdd85f Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 14 Apr 2025 19:53:39 +0530 Subject: [PATCH 8/9] Update app/controllers/api/functions.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matej Bačo --- app/controllers/api/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index abf7b58e59..3a866b5447 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1312,7 +1312,7 @@ App::post('/v1/functions/:functionId/deployments') throw new Exception(Exception::STORAGE_FILE_EMPTY, 'No file sent'); } - $functionSizeLimit = System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000'); + $functionSizeLimit = (int) System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000'); if (isset($plan['functionSize'])) { $functionSizeLimit = $plan['functionSize'] * 1000 * 1000; From 6e86527eb62597a7372255d0c4c4c5ca3d3da613 Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 14 Apr 2025 14:33:46 +0000 Subject: [PATCH 9/9] Keep build size in MBs --- src/Appwrite/Platform/Workers/Builds.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index f7395a214a..6392b9f3e5 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -630,7 +630,7 @@ class Builds extends Action $buildSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_BUILD_SIZE_LIMIT', '2000000000'); if (isset($plan['buildSize'])) { - $buildSizeLimit = $plan['buildSize'] * 1000 * 1000 * 1000; + $buildSizeLimit = $plan['buildSize'] * 1000 * 1000; } if ($response['size'] > $buildSizeLimit) { throw new \Exception('Build size should be less than ' . number_format($buildSizeLimit / (1000 * 1000), 2) . ' MBs.');