From fa3fc7c8582a15e9335b47416a35e5138173d5ce Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 11 Feb 2025 16:39:04 +0530 Subject: [PATCH] Replace deviceForFunctions with deviceForSites --- app/controllers/api/health.php | 5 +-- .../Modules/Functions/Workers/Builds.php | 31 ++++++++++++------- .../Sites/Http/Deployments/Builds/Create.php | 9 +++--- .../Modules/Sites/Http/Deployments/Create.php | 9 +++--- .../Modules/Sites/Http/Deployments/Delete.php | 5 ++- .../Sites/Http/Deployments/Download/Get.php | 13 ++++---- src/Appwrite/Platform/Workers/Deletes.php | 27 ++++++++++------ 7 files changed, 57 insertions(+), 42 deletions(-) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 305444abc5..f38cc0fc9f 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -825,9 +825,10 @@ App::get('/v1/health/storage') ->inject('response') ->inject('deviceForFiles') ->inject('deviceForFunctions') + ->inject('deviceForSites') ->inject('deviceForBuilds') - ->action(function (Response $response, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds) { - $devices = [$deviceForFiles, $deviceForFunctions, $deviceForBuilds]; + ->action(function (Response $response, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForSites, Device $deviceForBuilds) { + $devices = [$deviceForFiles, $deviceForFunctions, $deviceForSites, $deviceForBuilds]; $checkStart = \microtime(true); foreach ($devices as $device) { diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 41ab2fe02a..b7b21df865 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -54,9 +54,10 @@ class Builds extends Action ->inject('queueForUsage') ->inject('cache') ->inject('dbForProject') + ->inject('deviceForSites') ->inject('deviceForFunctions') ->inject('log') - ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log)); + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForSites, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForSites, $deviceForFunctions, $log)); } /** @@ -68,12 +69,13 @@ class Builds extends Action * @param Usage $queueForUsage * @param Cache $cache * @param Database $dbForProject + * @param Device $deviceForSites * @param Device $deviceForFunctions * @param Log $log * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForSites, Device $deviceForFunctions, Log $log): void { $payload = $message->getPayload() ?? []; @@ -94,7 +96,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log); + $this->buildDeployment($deviceForSites, $deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log); break; default: @@ -103,6 +105,7 @@ class Builds extends Action } /** + * @param Device $deviceForSites * @param Device $deviceForFunctions * @param Func $queueForFunctions * @param Event $queueForEvents @@ -120,7 +123,7 @@ class Builds extends Action * * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void + protected function buildDeployment(Device $deviceForSites, Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void { $resourceKey = match($resource->getCollection()) { 'functions' => 'functionId', @@ -159,6 +162,12 @@ class Builds extends Action 'deploymentId' => $deployment->getId() ]); + $device = match($resource->getCollection()) { + 'functions' => $deviceForFunctions, + 'sites' => $deviceForSites, + default => throw new \Exception('Invalid resource type') + }; + $startTime = DateTime::now(); $durationStart = \microtime(true); $buildId = $deployment->getAttribute('buildId', ''); @@ -176,7 +185,7 @@ class Builds extends Action 'path' => '', 'runtime' => $resource->getAttribute('runtime'), 'source' => $deployment->getAttribute('path', ''), - 'sourceType' => strtolower($deviceForFunctions->getType()), + 'sourceType' => strtolower($device->getType()), 'logs' => '', 'endTime' => null, 'duration' => 0, @@ -251,8 +260,8 @@ class Builds extends Action $tarParamDirectory = \escapeshellarg($tmpTemplateDirectory . (empty($templateRootDirectory) ? '' : '/' . $templateRootDirectory)); Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax - $source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $result = $localDevice->transfer($tmpPathFile, $source, $deviceForFunctions); + $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); + $result = $localDevice->transfer($tmpPathFile, $source, $device); if (!$result) { throw new \Exception("Unable to move file"); @@ -260,7 +269,7 @@ class Builds extends Action Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr); - $directorySize = $deviceForFunctions->getFileSize($source); + $directorySize = $device->getFileSize($source); $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source)); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize)); } @@ -408,8 +417,8 @@ class Builds extends Action $tarParamDirectory = '/tmp/builds/' . $buildId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory); Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax - $source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $result = $localDevice->transfer($tmpPathFile, $source, $deviceForFunctions); + $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); + $result = $localDevice->transfer($tmpPathFile, $source, $device); if (!$result) { throw new \Exception("Unable to move file"); @@ -419,7 +428,7 @@ class Builds extends Action $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source)); - $directorySize = $deviceForFunctions->getFileSize($source); + $directorySize = $device->getFileSize($source); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize)); $this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform); diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Builds/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Builds/Create.php index 0c5cfc34e4..16e9aab43b 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Builds/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Builds/Create.php @@ -57,11 +57,10 @@ class Create extends Action ->inject('queueForEvents') ->inject('queueForBuilds') ->inject('deviceForSites') - ->inject('deviceForFunctions') //TODO: remove it later ->callback([$this, 'action']); } - public function action(string $siteId, string $deploymentId, Response $response, Database $dbForProject, Event $queueForEvents, Build $queueForBuilds, Device $deviceForSites, Device $deviceForFunctions) + public function action(string $siteId, string $deploymentId, Response $response, Database $dbForProject, Event $queueForEvents, Build $queueForBuilds, Device $deviceForSites) { $site = $dbForProject->getDocument('sites', $siteId); @@ -75,14 +74,14 @@ class Create extends Action } $path = $deployment->getAttribute('path'); - if (empty($path) || !$deviceForFunctions->exists($path)) { + if (empty($path) || !$deviceForSites->exists($path)) { throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } $deploymentId = ID::unique(); - $destination = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $deviceForFunctions->transfer($path, $destination, $deviceForFunctions); + $destination = $deviceForSites->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); + $deviceForSites->transfer($path, $destination, $deviceForSites); $deployment->removeAttribute('$internalId'); $deployment = $dbForProject->createDocument('deployments', $deployment->setAttributes([ diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php index 38b9e2aeba..91555143c0 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php @@ -80,13 +80,12 @@ class Create extends Action ->inject('project') ->inject('queueForEvents') ->inject('deviceForSites') - ->inject('deviceForFunctions') // TODO: Remove this later once volume is added to executor ->inject('deviceForLocal') ->inject('queueForBuilds') ->callback([$this, 'action']); } - public function action(string $siteId, ?string $installCommand, ?string $buildCommand, ?string $outputDirectory, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Database $dbForPlatform, Document $project, Event $queueForEvents, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForLocal, Build $queueForBuilds) + public function action(string $siteId, ?string $installCommand, ?string $buildCommand, ?string $outputDirectory, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Database $dbForPlatform, Document $project, Event $queueForEvents, Device $deviceForSites, Device $deviceForLocal, Build $queueForBuilds) { $activate = \strval($activate) === 'true' || \strval($activate) === '1'; @@ -168,7 +167,7 @@ class Create extends Action // Save to storage $fileSize ??= $deviceForLocal->getFileSize($fileTmpName); - $path = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); + $path = $deviceForSites->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); $deployment = $dbForProject->getDocument('deployments', $deploymentId); $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; @@ -180,7 +179,7 @@ class Create extends Action } } - $chunksUploaded = $deviceForFunctions->upload($fileTmpName, $path, $chunk, $chunks, $metadata); + $chunksUploaded = $deviceForSites->upload($fileTmpName, $path, $chunk, $chunks, $metadata); if (empty($chunksUploaded)) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed moving file'); @@ -203,7 +202,7 @@ class Create extends Action } } - $fileSize = $deviceForFunctions->getFileSize($path); + $fileSize = $deviceForSites->getFileSize($path); if ($deployment->isEmpty()) { $deployment = $dbForProject->createDocument('deployments', new Document([ diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Delete.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Delete.php index 133e9a1906..efcdf88170 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Delete.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Delete.php @@ -59,11 +59,10 @@ class Delete extends Action ->inject('queueForDeletes') ->inject('queueForEvents') ->inject('deviceForSites') - ->inject('deviceForFunctions') //TODO: remove it later ->callback([$this, 'action']); } - public function action(string $siteId, string $deploymentId, Response $response, Database $dbForProject, DeleteEvent $queueForDeletes, Event $queueForEvents, Device $deviceForSites, Device $deviceForFunctions) + public function action(string $siteId, string $deploymentId, Response $response, Database $dbForProject, DeleteEvent $queueForDeletes, Event $queueForEvents, Device $deviceForSites) { $site = $dbForProject->getDocument('sites', $siteId); if ($site->isEmpty()) { @@ -84,7 +83,7 @@ class Delete extends Action } if (!empty($deployment->getAttribute('path', ''))) { - if (!($deviceForFunctions->delete($deployment->getAttribute('path', '')))) { + if (!($deviceForSites->delete($deployment->getAttribute('path', '')))) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove deployment from storage'); } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php index 8e1235c5ee..2d38e619a6 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php @@ -55,11 +55,10 @@ class Get extends Action ->inject('request') ->inject('dbForProject') ->inject('deviceForSites') - ->inject('deviceForFunctions') //TODO: Remove this later ->callback([$this, 'action']); } - public function action(string $siteId, string $deploymentId, Response $response, Request $request, Database $dbForProject, Device $deviceForSites, Device $deviceForFunctions) + public function action(string $siteId, string $deploymentId, Response $response, Request $request, Database $dbForProject, Device $deviceForSites) { $site = $dbForProject->getDocument('sites', $siteId); if ($site->isEmpty()) { @@ -76,7 +75,7 @@ class Get extends Action } $path = $deployment->getAttribute('path', ''); - if (!$deviceForFunctions->exists($path)) { + if (!$deviceForSites->exists($path)) { throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } @@ -86,7 +85,7 @@ class Get extends Action ->addHeader('X-Peak', \memory_get_peak_usage()) ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '.tar.gz"'); - $size = $deviceForFunctions->getFileSize($path); + $size = $deviceForSites->getFileSize($path); $rangeHeader = $request->getHeader('range'); if (!empty($rangeHeader)) { @@ -108,13 +107,13 @@ class Get extends Action ->addHeader('Content-Length', $end - $start + 1) ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); - $response->send($deviceForFunctions->read($path, $start, ($end - $start + 1))); + $response->send($deviceForSites->read($path, $start, ($end - $start + 1))); } if ($size > APP_STORAGE_READ_BUFFER) { for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { $response->chunk( - $deviceForFunctions->read( + $deviceForSites->read( $path, ($i * MAX_OUTPUT_CHUNK_SIZE), min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) @@ -123,7 +122,7 @@ class Get extends Action ); } } else { - $response->send($deviceForFunctions->read($path)); + $response->send($deviceForSites->read($path)); } } } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 09c71dc004..4843e2c477 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -91,13 +91,13 @@ class Deletes extends Action $this->deleteProject($dbForPlatform, $getProjectDB, $deviceForFiles, $deviceForSites, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document); break; case DELETE_TYPE_SITES: - $this->deleteSite($dbForPlatform, $getProjectDB, $deviceForSites, $deviceForFunctions, $deviceForBuilds, $document, $certificates, $project); + $this->deleteSite($dbForPlatform, $getProjectDB, $deviceForSites, $deviceForBuilds, $document, $certificates, $project); break; case DELETE_TYPE_FUNCTIONS: $this->deleteFunction($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project); break; case DELETE_TYPE_DEPLOYMENTS: - $this->deleteDeployment($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $certificates, $project); + $this->deleteDeployment($dbForPlatform, $getProjectDB, $deviceForSites, $deviceForFunctions, $deviceForBuilds, $document, $certificates, $project); break; case DELETE_TYPE_USERS: $this->deleteUser($getProjectDB, $document, $project); @@ -485,6 +485,7 @@ class Deletes extends Action * @param Database $dbForPlatform * @param callable $getProjectDB * @param Device $deviceForFiles + * @param Device $deviceForSites * @param Device $deviceForFunctions * @param Device $deviceForBuilds * @param Device $deviceForCache @@ -740,14 +741,13 @@ class Deletes extends Action /** * @param callable $getProjectDB * @param Device $deviceForSites - * @param Device $deviceForFunctions * @param Device $deviceForBuilds * @param Document $document function document * @param Document $project * @return void * @throws Exception */ - private function deleteSite(Database $dbForPlatform, callable $getProjectDB, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, CertificatesAdapter $certificates, Document $project): void + private function deleteSite(Database $dbForPlatform, callable $getProjectDB, Device $deviceForSites, Device $deviceForBuilds, Document $document, CertificatesAdapter $certificates, Document $project): void { $dbForProject = $getProjectDB($project); $siteId = $document->getId(); @@ -781,9 +781,9 @@ class Deletes extends Action $deploymentInternalIds = []; $this->deleteByGroup('deployments', [ Query::equal('resourceInternalId', [$siteInternalId]) - ], $dbForProject, function (Document $document) use ($deviceForFunctions, &$deploymentInternalIds) { + ], $dbForProject, function (Document $document) use ($deviceForSites, &$deploymentInternalIds) { $deploymentInternalIds[] = $document->getInternalId(); - $this->deleteDeploymentFiles($deviceForFunctions, $document); + $this->deleteDeploymentFiles($deviceForSites, $document); }); /** @@ -990,6 +990,7 @@ class Deletes extends Action /** * @param callable $getProjectDB + * @param Device $deviceForSites * @param Device $deviceForFunctions * @param Device $deviceForBuilds * @param Document $document @@ -997,17 +998,25 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteDeployment(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, CertificatesAdapter $certificates, Document $project): void + private function deleteDeployment(Database $dbForPlatform, callable $getProjectDB, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, CertificatesAdapter $certificates, Document $project): void { - $projectId = $project->getId(); $dbForProject = $getProjectDB($project); $deploymentId = $document->getId(); $deploymentInternalId = $document->getInternalId(); + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + $resourceType = $deployment->getAttribute('resourceType'); + + $device = match($resourceType) { + 'functions' => $deviceForFunctions, + 'sites' => $deviceForSites, + default => throw new \Exception('Invalid resource type') + }; + /** * Delete deployment files */ - $this->deleteDeploymentFiles($deviceForFunctions, $document); //TODO: For sites, this should be deviceForSites + $this->deleteDeploymentFiles($device, $document); /** * Delete builds