diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 5fd997bb47..597f02f7d6 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1154,6 +1154,7 @@ return [ ] ], ], + 'deployments' => [ '$collection' => ID::custom(Database::METADATA), '$id' => ID::custom('deployments'), @@ -1203,6 +1204,17 @@ return [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('buildId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ 'array' => false, '$id' => ID::custom('entrypoint'), @@ -1539,94 +1551,6 @@ return [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('startTime'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => ID::custom('endTime'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => ID::custom('buildTime'), - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('buildSize'), - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('runtime'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => true, - 'default' => '', - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('status'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 256, - 'signed' => true, - 'required' => true, - 'default' => 'processing', - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('buildPath'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => '', - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('buildLogs'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 1000000, - 'signed' => true, - 'required' => false, - 'default' => '', - 'array' => false, - 'filters' => [], - ] ], 'indexes' => [ [ @@ -1657,6 +1581,13 @@ return [ 'lengths' => [], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => ID::custom('_key_buildId'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['buildId'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], [ '$id' => ID::custom('_key_activate'), 'type' => Database::INDEX_KEY, @@ -1667,6 +1598,155 @@ return [ ], ], + 'builds' => [ + '$collection' => ID::custom(Database::METADATA), + '$id' => ID::custom('builds'), + 'name' => 'Builds', + 'attributes' => [ + [ + '$id' => ID::custom('startTime'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => ID::custom('endTime'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => ID::custom('duration'), + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('size'), + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('deploymentInternalId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('deploymentId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('runtime'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => true, + 'default' => '', + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('status'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => true, + 'default' => 'processing', + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('path'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => '', + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('logs'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 1000000, + 'signed' => true, + 'required' => false, + 'default' => '', + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('sourceType'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => true, + 'default' => 'local', + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('source'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => true, + 'default' => '', + 'array' => false, + 'filters' => [], + ] + ], + 'indexes' => [ + [ + '$id' => ID::custom('_key_deployment'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['deploymentId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ] + ], + ], + 'executions' => [ '$collection' => ID::custom(Database::METADATA), '$id' => ID::custom('executions'), diff --git a/app/controllers/general.php b/app/controllers/general.php index 5e3edca418..d4728fb5af 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -191,7 +191,13 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw throw new AppwriteException(AppwriteException::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); } - if ($deployment->getAttribute('status') !== 'ready') { + /** Check if build has completed */ + $build = Authorization::skip(fn () => $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''))); + if ($build->isEmpty()) { + throw new AppwriteException(AppwriteException::BUILD_NOT_FOUND); + } + + if ($build->getAttribute('status') !== 'ready') { throw new AppwriteException(AppwriteException::BUILD_NOT_READY); } @@ -382,7 +388,7 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw variables: $vars, timeout: $resource->getAttribute('timeout', 30), image: $runtime['image'], - source: $deployment->getAttribute('buildPath', ''), + source: $build->getAttribute('path', ''), entrypoint: $entrypoint, version: $version, path: $path, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php index d528bc9995..a63ac18354 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php @@ -81,7 +81,12 @@ class Get extends Action switch ($type) { case 'output': - $path = $deployment->getAttribute('buildPath', ''); + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); + if ($build->isEmpty()) { + throw new Exception(Exception::BUILD_NOT_FOUND); + } + + $path = $build->getAttribute('path', ''); $device = $deviceForBuilds; break; case 'source': diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Get.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Get.php index 8e58433e7c..1d074e9e2a 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Get.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Get.php @@ -69,6 +69,13 @@ class Get extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + $deployment->setAttribute('status', $build->getAttribute('status', 'waiting')); + $deployment->setAttribute('buildLogs', $build->getAttribute('logs', '')); + $deployment->setAttribute('buildTime', $build->getAttribute('duration', 0)); + $deployment->setAttribute('buildSize', $build->getAttribute('size', 0)); + $deployment->setAttribute('size', $deployment->getAttribute('size', 0)); + $response->dynamic($deployment, Response::MODEL_DEPLOYMENT); } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php index 5b58a972a3..e349292e4b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php @@ -13,6 +13,8 @@ use Utopia\App; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Helpers\ID; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; @@ -48,7 +50,7 @@ class Update extends Action responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_DEPLOYMENT, + model: Response::MODEL_BUILD, ) ] )) @@ -75,19 +77,46 @@ class Update extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } - if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'])) { - throw new Exception(Exception::BUILD_ALREADY_COMPLETED); + $build = Authorization::skip(fn () => $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''))); + + if ($build->isEmpty()) { + $buildId = ID::unique(); + $build = $dbForProject->createDocument('builds', new Document([ + '$id' => $buildId, + '$permissions' => [], + 'startTime' => DateTime::now(), + 'deploymentInternalId' => $deployment->getInternalId(), + 'deploymentId' => $deployment->getId(), + 'status' => 'canceled', + 'path' => '', + 'runtime' => $function->getAttribute('runtime'), + 'source' => $deployment->getAttribute('path', ''), + 'sourceType' => '', + 'logs' => '', + 'duration' => 0, + 'size' => 0 + ])); + + $deployment->setAttribute('buildId', $build->getId()); + $deployment->setAttribute('buildInternalId', $build->getInternalId()); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } else { + if (\in_array($build->getAttribute('status'), ['ready', 'failed'])) { + throw new Exception(Exception::BUILD_ALREADY_COMPLETED); + } + + $startTime = new \DateTime($build->getAttribute('startTime')); + $endTime = new \DateTime('now'); + $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); + + $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttributes([ + 'endTime' => DateTime::now(), + 'duration' => $duration, + 'status' => 'canceled' + ])); } - $startTime = new \DateTime($deployment->getAttribute('startTime')); - $endTime = new \DateTime('now'); - $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); - - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttributes([ - 'endTime' => DateTime::now(), - 'duration' => $duration, - 'status' => 'canceled' - ])); + $dbForProject->purgeCachedDocument('deployments', $deployment->getId()); try { $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); @@ -103,6 +132,6 @@ class Update extends Action ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()); - $response->dynamic($deployment, Response::MODEL_DEPLOYMENT); + $response->dynamic($build, Response::MODEL_BUILD); } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/XList.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/XList.php index a416001e2f..9e03a86bc7 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/XList.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/XList.php @@ -110,6 +110,15 @@ class XList extends Action $results = $dbForProject->find('deployments', $queries); $total = $dbForProject->count('deployments', $filterQueries, APP_LIMIT_COUNT); + foreach ($results as $result) { + $build = $dbForProject->getDocument('builds', $result->getAttribute('buildId', '')); + $result->setAttribute('status', $build->getAttribute('status', 'processing')); + $result->setAttribute('buildLogs', $build->getAttribute('logs', '')); + $result->setAttribute('buildTime', $build->getAttribute('duration', 0)); + $result->setAttribute('buildSize', $build->getAttribute('size', 0)); + $result->setAttribute('size', $result->getAttribute('size', 0)); + } + $response->dynamic(new Document([ 'deployments' => $results, 'total' => $total, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index 9678b1ef64..f522e2dc62 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -4,6 +4,8 @@ namespace Appwrite\Platform\Modules\Functions\Http\Executions; use Ahc\Jwt\JWT; use Appwrite\Auth\Auth; +use Appwrite\Event\Build; +use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\StatsUsage; @@ -156,7 +158,13 @@ class Create extends Base throw new Exception(Exception::DEPLOYMENT_NOT_FOUND, 'Deployment not found. Create a deployment before trying to execute a function'); } - if ($deployment->getAttribute('status') !== 'ready') { + /** Check if build has completed */ + $build = Authorization::skip(fn () => $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''))); + if ($build->isEmpty()) { + throw new Exception(Exception::BUILD_NOT_FOUND); + } + + if ($build->getAttribute('status') !== 'ready') { throw new Exception(Exception::BUILD_NOT_READY); } @@ -377,7 +385,7 @@ class Create extends Base variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $deployment->getAttribute('buildPath', ''), + source: $build->getAttribute('path', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), version: $version, path: $path, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Deployment/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Deployment/Update.php index cda0b8b6d9..f45aa0a108 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Deployment/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Deployment/Update.php @@ -68,6 +68,7 @@ class Update extends Base { $function = $dbForProject->getDocument('functions', $functionId); $deployment = $dbForProject->getDocument('deployments', $deploymentId); + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); if ($function->isEmpty()) { throw new Exception(Exception::FUNCTION_NOT_FOUND); @@ -77,7 +78,11 @@ class Update extends Base throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } - if ($deployment->getAttribute('status') !== 'ready') { + if ($build->isEmpty()) { + throw new Exception(Exception::BUILD_NOT_FOUND); + } + + if ($build->getAttribute('status') !== 'ready') { throw new Exception(Exception::BUILD_NOT_READY); } diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index f66c4b4638..53556f60ba 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -186,20 +186,38 @@ class Builds extends Action $startTime = DateTime::now(); $durationStart = \microtime(true); + $buildId = $deployment->getAttribute('buildId', ''); + $build = $dbForProject->getDocument('builds', $buildId); + $isNewBuild = empty($buildId); + if ($build->isEmpty()) { + $buildId = ID::unique(); + $build = $dbForProject->createDocument('builds', new Document([ + '$id' => $buildId, + '$permissions' => [], + 'startTime' => $startTime, + 'deploymentInternalId' => $deployment->getInternalId(), + 'deploymentId' => $deployment->getId(), + 'status' => 'processing', + 'path' => '', + 'runtime' => $resource->getAttribute('runtime'), + 'source' => $deployment->getAttribute('path', ''), + 'sourceType' => strtolower($deviceForFunctions->getType()), + 'logs' => '', + 'endTime' => null, + 'duration' => 0, + 'size' => 0 + ])); - if ($deployment->getAttribute('status') === 'canceled') { + $deployment->setAttribute('buildId', $build->getId()); + $deployment->setAttribute('buildInternalId', $build->getInternalId()); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } elseif ($build->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; + } else { + $build = $dbForProject->getDocument('builds', $buildId); } - $deploymentId = $deployment->getId(); - - $deployment->setAttribute('startTime', $startTime); - $deployment->setAttribute('status', 'processing'); - $deployment->setAttribute('runtime', $resource->getAttribute('runtime')); - $deployment->setAttribute('sourceType', \strtolower($deviceForFunctions->getType())); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - $source = $deployment->getAttribute('path', ''); $installationId = $deployment->getAttribute('installationId', ''); $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', ''); @@ -218,7 +236,7 @@ class Builds extends Action } try { - if (!$isVcsEnabled) { + if ($isNewBuild && !$isVcsEnabled) { // Non-VCS + Template $templateRepositoryName = $template->getAttribute('repositoryName', ''); $templateOwnerName = $template->getAttribute('ownerName', ''); @@ -234,7 +252,7 @@ class Builds extends Action $stderr = ''; // Clone template repo - $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '-template'; + $tmpTemplateDirectory = '/tmp/builds/' . $buildId . '-template'; $gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateVersion, GitHub::CLONE_TYPE_TAG, $tmpTemplateDirectory, $templateRootDirectory); $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); @@ -268,14 +286,12 @@ class Builds extends Action Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr); $directorySize = $device->getFileSize($source); - $deployment - ->setAttribute('path', $source) - ->setAttribute('size', $directorySize); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source)); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize)); } - } elseif ($isVcsEnabled) { + } elseif ($isNewBuild && $isVcsEnabled) { // VCS and VCS+Temaplte - $tmpDirectory = '/tmp/builds/' . $deploymentId . '/code'; + $tmpDirectory = '/tmp/builds/' . $buildId . '/code'; $rootDirectory = $resource->getAttribute('providerRootDirectory', ''); $rootDirectory = \rtrim($rootDirectory, '/'); $rootDirectory = \ltrim($rootDirectory, '.'); @@ -301,9 +317,9 @@ class Builds extends Action $stdout = ''; $stderr = ''; - Console::execute('mkdir -p ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); + Console::execute('mkdir -p ' . \escapeshellarg('/tmp/builds/' . $buildId), '', $stdout, $stderr); - if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { + if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; } @@ -340,7 +356,7 @@ class Builds extends Action if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateVersion)) { // Clone template repo - $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '/template'; + $tmpTemplateDirectory = '/tmp/builds/' . $buildId . '/template'; $gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateVersion, GitHub::CLONE_TYPE_TAG, $tmpTemplateDirectory, $templateRootDirectory); $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); @@ -385,19 +401,19 @@ class Builds extends Action $target = Realtime::fromPayload( // Pass first, most verbose event pattern event: $allEvents[0], - payload: $deployment, + payload: $build, project: $project ); Realtime::send( projectId: 'console', - payload: $deployment->getArrayCopy(), + payload: $build->getArrayCopy(), events: $allEvents, channels: $target['channels'], roles: $target['roles'] ); } - $tmpPath = '/tmp/builds/' . $deployment; + $tmpPath = '/tmp/builds/' . $buildId; $tmpPathFile = $tmpPath . '/code.tar.gz'; $localDevice = new Local(); @@ -414,7 +430,7 @@ class Builds extends Action Console::execute('find ' . \escapeshellarg($tmpDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr); - $tarParamDirectory = '/tmp/builds/' . $deploymentId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory); + $tarParamDirectory = '/tmp/builds/' . $buildId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory); Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); @@ -426,19 +442,17 @@ class Builds extends Action Console::execute('rm -rf ' . \escapeshellarg($tmpPath), '', $stdout, $stderr); - $directorySize = $device->getFileSize($source); + $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source)); - $deployment - ->setAttribute('path', $source) - ->setAttribute('size', $directorySize); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + $directorySize = $device->getFileSize($source); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize)); $this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform); } /** Request the executor to build the code... */ - $deployment->setAttribute('status', 'building'); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + $build->setAttribute('status', 'building'); + $build = $dbForProject->updateDocument('builds', $buildId, $build); if ($isVcsEnabled) { $this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform); @@ -467,13 +481,13 @@ class Builds extends Action $target = Realtime::fromPayload( // Pass first, most verbose event pattern event: $allEvents[0], - payload: $deployment, + payload: $build, project: $project ); Realtime::send( projectId: 'console', - payload: $deployment->getArrayCopy(), + payload: $build->getArrayCopy(), events: $allEvents, channels: $target['channels'], roles: $target['roles'] @@ -568,7 +582,7 @@ class Builds extends Action $response = null; $err = null; - if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { + if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; } @@ -601,22 +615,26 @@ class Builds extends Action $err = $error; } }), - Co\go(function () use ($executor, $project, &$deployment, &$response, $dbForProject, $allEvents, $timeout, &$err, &$isCanceled) { + Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, $timeout, &$err, &$isCanceled) { try { $executor->getLogs( deploymentId: $deployment->getId(), projectId: $project->getId(), timeout: $timeout, - callback: function ($logs) use (&$response, &$err, $dbForProject, $allEvents, $project, &$isCanceled, &$deployment) { + callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $allEvents, $project, &$isCanceled) { if ($isCanceled) { return; } // If we have response or error from concurrent coroutine, we already have latest logs if ($response === null && $err === null) { - $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); + $build = $dbForProject->getDocument('builds', $build->getId()); - if ($deployment->getAttribute('status') === 'canceled') { + if ($build->isEmpty()) { + throw new \Exception('Build not found'); + } + + if ($build->getAttribute('status') === 'canceled') { $isCanceled = true; Console::info('Ignoring realtime logs because build has been canceled'); return; @@ -625,7 +643,7 @@ class Builds extends Action // Get only valid UTF8 part - removes leftover half-multibytes causing SQL errors $logs = \mb_substr($logs, 0, null, 'UTF-8'); - $currentLogs = $deployment->getAttribute('logs', ''); + $currentLogs = $build->getAttribute('logs', ''); $streamLogs = \str_replace("\\n", "{APPWRITE_LINEBREAK_PLACEHOLDER}", $logs); foreach (\explode("\n", $streamLogs) as $streamLog) { @@ -640,8 +658,8 @@ class Builds extends Action $currentLogs .= $streamParts[1]; } - $deployment = $deployment->setAttribute('logs', $currentLogs); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + $build = $build->setAttribute('logs', $currentLogs); + $build = $dbForProject->updateDocument('builds', $build->getId(), $build); /** * Send realtime Event @@ -649,12 +667,12 @@ class Builds extends Action $target = Realtime::fromPayload( // Pass first, most verbose event pattern event: $allEvents[0], - payload: $deployment, + payload: $build, project: $project ); Realtime::send( projectId: 'console', - payload: $deployment->getArrayCopy(), + payload: $build->getArrayCopy(), events: $allEvents, channels: $target['channels'], roles: $target['roles'] @@ -670,7 +688,7 @@ class Builds extends Action }), ]); - if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { + if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; } @@ -688,26 +706,26 @@ class Builds extends Action } /** Update the build document */ - $deployment->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); - $deployment->setAttribute('endTime', $endTime); - $deployment->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); - $deployment->setAttribute('status', 'ready'); - $deployment->setAttribute('buildPath', $response['path']); - $deployment->setAttribute('buildSize', $response['size']); + $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); + $build->setAttribute('endTime', $endTime); + $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); + $build->setAttribute('status', 'ready'); + $build->setAttribute('path', $response['path']); + $build->setAttribute('size', $response['size']); $logs = ''; foreach ($response['output'] as $log) { $logs .= $log['content']; } - $deployment->setAttribute('logs', $logs); + $build->setAttribute('logs', $logs); - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); + $build = $dbForProject->updateDocument('builds', $buildId, $build); if ($isVcsEnabled) { $this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform); } - Console::success("Build id: $deploymentId created"); + Console::success("Build id: $buildId created"); if ($resource->getCollection() === 'sites') { try { @@ -866,7 +884,7 @@ class Builds extends Action } - if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { + if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; } @@ -883,19 +901,20 @@ class Builds extends Action Authorization::skip(fn () => $dbForPlatform->updateDocument('schedules', $schedule->getId(), $schedule)); } } catch (\Throwable $th) { - if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { + if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; } $endTime = DateTime::now(); $durationEnd = \microtime(true); - $deployment->setAttribute('endTime', $endTime); - $deployment->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); - $deployment->setAttribute('status', 'failed'); - $deployment->setAttribute('logs', "" . $th->getMessage()); + $build->setAttribute('endTime', $endTime); + $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); + $build->setAttribute('status', 'failed'); - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); + $build->setAttribute('logs', "[31m" . $th->getMessage() . "[0m"); + + $build = $dbForProject->updateDocument('builds', $buildId, $build); if ($isVcsEnabled) { $this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform); @@ -907,26 +926,26 @@ class Builds extends Action $target = Realtime::fromPayload( // Pass first, most verbose event pattern event: $allEvents[0], - payload: $deployment, + payload: $build, project: $project ); Realtime::send( projectId: 'console', - payload: $deployment->getArrayCopy(), + payload: $build->getArrayCopy(), events: $allEvents, channels: $target['channels'], roles: $target['roles'] ); $this->sendUsage( resource:$resource, - deployment: $deployment, + build: $build, project: $project, queue: $queueForStatsUsage ); } } - protected function sendUsage(Document $resource, Document $deployment, Document $project, StatsUsage $queue): void + protected function sendUsage(Document $resource, Document $build, Document $project, StatsUsage $queue): void { $key = match($resource->getCollection()) { 'functions' => 'functionInternalId', @@ -957,32 +976,32 @@ class Builds extends Action ] }; - switch ($deployment->getAttribute('status')) { + switch ($build->getAttribute('status')) { case 'ready': $queue ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$deployment->getAttribute('duration', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000) ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$deployment->getAttribute('duration', 0) * 1000); + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000); break; case 'failed': $queue ->addMetric(METRIC_BUILDS_FAILED, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$deployment->getAttribute('duration', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000) ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsFailed']), 1) // per function - ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeFailed']), (int)$deployment->getAttribute('duration', 0) * 1000); + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeFailed']), (int)$build->getAttribute('duration', 0) * 1000); break; } $queue ->addMetric(METRIC_BUILDS, 1) // per project - ->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0)) - ->addMetric(METRIC_BUILDS_COMPUTE, (int)$deployment->getAttribute('duration', 0) * 1000) - ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) + ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0)) - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$deployment->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) ->setProject($project) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php index 1d84288203..f4b6b73784 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php @@ -80,7 +80,12 @@ class Get extends Action switch ($type) { case 'output': - $path = $deployment->getAttribute('buildPath', ''); + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); + if ($build->isEmpty()) { + throw new Exception(Exception::BUILD_NOT_FOUND); + } + + $path = $build->getAttribute('path', ''); $device = $deviceForBuilds; break; case 'source': diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Get.php index 8c3beaf2c0..c7929b2e7a 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Get.php @@ -69,6 +69,13 @@ class Get extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + $deployment->setAttribute('status', $build->getAttribute('status', 'waiting')); + $deployment->setAttribute('buildLogs', $build->getAttribute('logs', '')); + $deployment->setAttribute('buildTime', $build->getAttribute('duration', 0)); + $deployment->setAttribute('buildSize', $build->getAttribute('size', 0)); + $deployment->setAttribute('size', $deployment->getAttribute('size', 0)); + $response->dynamic($deployment, Response::MODEL_DEPLOYMENT); } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php index d4d836af7d..0a15053df1 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php @@ -13,6 +13,8 @@ use Utopia\App; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Helpers\ID; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; @@ -46,7 +48,7 @@ class Update extends Action responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_DEPLOYMENT, + model: Response::MODEL_BUILD, ) ] )) @@ -73,19 +75,46 @@ class Update extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } - if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'])) { - throw new Exception(Exception::BUILD_ALREADY_COMPLETED); + $build = Authorization::skip(fn () => $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''))); + + if ($build->isEmpty()) { + $buildId = ID::unique(); + $build = $dbForProject->createDocument('builds', new Document([ + '$id' => $buildId, + '$permissions' => [], + 'startTime' => DateTime::now(), + 'deploymentInternalId' => $deployment->getInternalId(), + 'deploymentId' => $deployment->getId(), + 'status' => 'canceled', + 'path' => '', + 'runtime' => $site->getAttribute('framework'), + 'source' => $deployment->getAttribute('path', ''), + 'sourceType' => '', + 'logs' => '', + 'duration' => 0, + 'size' => 0 + ])); + + $deployment->setAttribute('buildId', $build->getId()); + $deployment->setAttribute('buildInternalId', $build->getInternalId()); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } else { + if (\in_array($build->getAttribute('status'), ['ready', 'failed'])) { + throw new Exception(Exception::BUILD_ALREADY_COMPLETED); + } + + $startTime = new \DateTime($build->getAttribute('startTime')); + $endTime = new \DateTime('now'); + $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); + + $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttributes([ + 'endTime' => DateTime::now(), + 'duration' => $duration, + 'status' => 'canceled' + ])); } - $startTime = new \DateTime($deployment->getAttribute('startTime')); - $endTime = new \DateTime('now'); - $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); - - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttributes([ - 'endTime' => DateTime::now(), - 'duration' => $duration, - 'status' => 'canceled' - ])); + $dbForProject->purgeCachedDocument('deployments', $deployment->getId()); try { $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); @@ -101,6 +130,6 @@ class Update extends Action ->setParam('siteId', $site->getId()) ->setParam('deploymentId', $deployment->getId()); - $response->dynamic($deployment, Response::MODEL_DEPLOYMENT); + $response->dynamic($build, Response::MODEL_BUILD); } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/XList.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/XList.php index e041554132..c29b14b840 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/XList.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/XList.php @@ -110,6 +110,15 @@ class XList extends Action $results = $dbForProject->find('deployments', $queries); $total = $dbForProject->count('deployments', $filterQueries, APP_LIMIT_COUNT); + foreach ($results as $result) { + $build = $dbForProject->getDocument('builds', $result->getAttribute('buildId', '')); + $result->setAttribute('status', $build->getAttribute('status', 'processing')); + $result->setAttribute('buildLogs', $build->getAttribute('logs', '')); + $result->setAttribute('buildTime', $build->getAttribute('duration', 0)); + $result->setAttribute('buildSize', $build->getAttribute('size', 0)); + $result->setAttribute('size', $result->getAttribute('size', 0)); + } + $response->dynamic(new Document([ 'deployments' => $results, 'total' => $total, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Deployment/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Deployment/Update.php index 611cdcb104..fcb0532929 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Deployment/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Deployment/Update.php @@ -65,6 +65,7 @@ class Update extends Base { $site = $dbForProject->getDocument('sites', $siteId); $deployment = $dbForProject->getDocument('deployments', $deploymentId); + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); if ($site->isEmpty()) { throw new Exception(Exception::SITE_NOT_FOUND); @@ -74,7 +75,11 @@ class Update extends Base throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } - if ($deployment->getAttribute('status') !== 'ready') { + if ($build->isEmpty()) { + throw new Exception(Exception::BUILD_NOT_FOUND); + } + + if ($build->getAttribute('status') !== 'ready') { throw new Exception(Exception::BUILD_NOT_READY); } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index fe12c90c4a..09fe98dd7f 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -784,10 +784,9 @@ class Deletes extends Action $deploymentIds = []; $this->deleteByGroup('deployments', [ Query::equal('resourceInternalId', [$siteInternalId]) - ], $dbForProject, function (Document $document) use ($project, $certificates, $deviceForSites, $deviceForBuilds, $deviceForFiles, $dbForPlatform, &$deploymentInternalIds) { + ], $dbForProject, function (Document $document) use ($project, $certificates, $deviceForSites, $deviceForFiles, $dbForPlatform, &$deploymentInternalIds) { $deploymentInternalIds[] = $document->getInternalId(); $deploymentIds[] = $document->getId(); - $this->deleteBuildFiles($deviceForBuilds, $document); $this->deleteDeploymentFiles($deviceForSites, $document); $this->deleteDeploymentScreenshots($deviceForFiles, $dbForPlatform, $document); $this->deleteDeploymentRules($dbForPlatform, $document, $project, $certificates); @@ -807,6 +806,18 @@ class Deletes extends Action }); } + /** + * Delete builds + */ + Console::info("Deleting builds for site " . $siteId); + foreach ($deploymentInternalIds as $deploymentInternalId) { + $this->deleteByGroup('builds', [ + Query::equal('deploymentInternalId', [$deploymentInternalId]) + ], $dbForProject, function (Document $document) use ($deviceForBuilds) { + $this->deleteBuildFiles($deviceForBuilds, $document); + }); + } + /** * Delete VCS Repositories and VCS Comments */ @@ -870,13 +881,25 @@ class Deletes extends Action $deploymentInternalIds = []; $this->deleteByGroup('deployments', [ Query::equal('resourceInternalId', [$functionInternalId]) - ], $dbForProject, function (Document $document) use ($dbForPlatform, $project, $certificates, $deviceForFunctions, $deviceForBuilds, &$deploymentInternalIds) { + ], $dbForProject, function (Document $document) use ($dbForPlatform, $project, $certificates, $deviceForFunctions, &$deploymentInternalIds) { $deploymentInternalIds[] = $document->getInternalId(); $this->deleteDeploymentFiles($deviceForFunctions, $document); - $this->deleteBuildFiles($deviceForBuilds, $document); $this->deleteDeploymentRules($dbForPlatform, $document, $project, $certificates); }); + /** + * Delete builds + */ + Console::info("Deleting builds for function " . $functionId); + + foreach ($deploymentInternalIds as $deploymentInternalId) { + $this->deleteByGroup('builds', [ + Query::equal('deploymentInternalId', [$deploymentInternalId]) + ], $dbForProject, function (Document $document) use ($deviceForBuilds) { + $this->deleteBuildFiles($deviceForBuilds, $document); + }); + } + /** * Delete Executions */ @@ -1005,13 +1028,13 @@ class Deletes extends Action * @param Document $build * @return void */ - private function deleteBuildFiles(Device $device, Document $deployment): void + private function deleteBuildFiles(Device $device, Document $build): void { - $deploymentId = $deployment->getId(); - $buildPath = $deployment->getAttribute('buildPath', ''); + $buildId = $build->getId(); + $buildPath = $build->getAttribute('path', ''); if (empty($buildPath)) { - Console::info("No build files for deployment " . $deploymentId); + Console::info("No build files for build " . $buildId); return; } @@ -1062,9 +1085,15 @@ class Deletes extends Action $this->deleteDeploymentScreenshots($deviceForFiles, $dbForPlatform, $document); /** - * Delete deployment build + * Delete builds */ - $this->deleteBuildFiles($deviceForBuilds, $document); + Console::info("Deleting builds for deployment " . $deploymentId); + + $this->deleteByGroup('builds', [ + Query::equal('deploymentInternalId', [$deploymentInternalId]) + ], $dbForProject, function (Document $document) use ($deviceForBuilds) { + $this->deleteBuildFiles($deviceForBuilds, $document); + }); /** * Delete rules associated with the deployment diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index 008bd0c8ec..f6d441ef3c 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -346,7 +346,19 @@ class Functions extends Action return; } - if ($deployment->getAttribute('status') !== 'ready') { + $buildId = $deployment->getAttribute('buildId', ''); + + $log->addTag('buildId', $buildId); + + /** Check if build has exists */ + $build = $dbForProject->getDocument('builds', $buildId); + if ($build->isEmpty()) { + $errorMessage = 'The execution could not be completed because a corresponding deployment was not found. A function deployment needs to be created before it can be executed. Please create a deployment for your function and try again.'; + $this->fail($errorMessage, $dbForProject, $function, $trigger, $path, $method, $user, $jwt, $event); + return; + } + + if ($build->getAttribute('status') !== 'ready') { $errorMessage = 'The execution could not be completed because the build is not ready. Please wait for the build to complete and try again.'; $this->fail($errorMessage, $dbForProject, $function, $trigger, $path, $method, $user, $jwt, $event); return; @@ -501,7 +513,7 @@ class Functions extends Action variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $deployment->getAttribute('buildPath', ''), + source: $build->getAttribute('path', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), version: $version, path: $path, diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index fa8d7490ea..a6101522fb 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -268,13 +268,15 @@ class StatsResources extends Action protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region) { $deploymentsStorage = $dbForProject->sum('deployments', 'size'); - $buildsStorage = $dbForProject->sum('deployments', 'buildSize'); + $buildsStorage = $dbForProject->sum('builds', 'size'); $this->createStatsDocuments($region, METRIC_DEPLOYMENTS_STORAGE, $deploymentsStorage); $this->createStatsDocuments($region, METRIC_BUILDS_STORAGE, $buildsStorage); $deployments = $dbForProject->count('deployments'); + $builds = $dbForProject->count('builds'); $this->createStatsDocuments($region, METRIC_DEPLOYMENTS, $deployments); - $this->createStatsDocuments($region, METRIC_BUILDS, $deployments); + $this->createStatsDocuments($region, METRIC_BUILDS, $builds); + $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForLogs, $region) { $functionDeploymentsStorage = $dbForProject->sum('deployments', 'size', [ @@ -300,8 +302,9 @@ class StatsResources extends Action $this->foreachDocument($dbForProject, 'deployments', [ Query::equal('resourceInternalId', [$function->getInternalId()]), Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), - ], function (Document $deployment) use (&$functionBuildsStorage): void { - $functionBuildsStorage += $deployment->getAttribute('buildSize', 0); + ], function (Document $deployment) use ($dbForProject, &$functionBuildsStorage): void { + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + $functionBuildsStorage += $build->getAttribute('size', 0); }); $this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionBuildsStorage); diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index d93c2178d8..da222822e0 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -30,6 +30,7 @@ use Appwrite\Utopia\Response\Model\AuthProvider; use Appwrite\Utopia\Response\Model\BaseList; use Appwrite\Utopia\Response\Model\Branch; use Appwrite\Utopia\Response\Model\Bucket; +use Appwrite\Utopia\Response\Model\Build; use Appwrite\Utopia\Response\Model\Collection; use Appwrite\Utopia\Response\Model\ConsoleVariables; use Appwrite\Utopia\Response\Model\Continent; @@ -271,6 +272,8 @@ class Response extends SwooleResponse public const MODEL_DEPLOYMENT_LIST = 'deploymentList'; public const MODEL_EXECUTION = 'execution'; public const MODEL_EXECUTION_LIST = 'executionList'; + public const MODEL_BUILD = 'build'; + public const MODEL_BUILD_LIST = 'buildList'; // Not used anywhere yet public const MODEL_FUNC_PERMISSIONS = 'funcPermissions'; public const MODEL_HEADERS = 'headers'; public const MODEL_SPECIFICATION = 'specification'; @@ -378,6 +381,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Runtimes List', self::MODEL_RUNTIME_LIST, 'runtimes', self::MODEL_RUNTIME)) ->setModel(new BaseList('Deployments List', self::MODEL_DEPLOYMENT_LIST, 'deployments', self::MODEL_DEPLOYMENT)) ->setModel(new BaseList('Executions List', self::MODEL_EXECUTION_LIST, 'executions', self::MODEL_EXECUTION)) + ->setModel(new BaseList('Builds List', self::MODEL_BUILD_LIST, 'builds', self::MODEL_BUILD)) // Not used anywhere yet ->setModel(new BaseList('Projects List', self::MODEL_PROJECT_LIST, 'projects', self::MODEL_PROJECT, true, false)) ->setModel(new BaseList('Webhooks List', self::MODEL_WEBHOOK_LIST, 'webhooks', self::MODEL_WEBHOOK, true, false)) ->setModel(new BaseList('API Keys List', self::MODEL_KEY_LIST, 'keys', self::MODEL_KEY, true, false)) @@ -457,6 +461,7 @@ class Response extends SwooleResponse ->setModel(new FrameworkAdapter()) ->setModel(new Deployment()) ->setModel(new Execution()) + ->setModel(new Build()) ->setModel(new Project()) ->setModel(new Webhook()) ->setModel(new Key()) diff --git a/src/Appwrite/Utopia/Response/Model/Build.php b/src/Appwrite/Utopia/Response/Model/Build.php new file mode 100644 index 0000000000..d80c17645a --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/Build.php @@ -0,0 +1,94 @@ +addRule('$id', [ + 'type' => self::TYPE_STRING, + 'description' => 'Build ID.', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) + ->addRule('deploymentId', [ + 'type' => self::TYPE_STRING, + 'description' => 'The deployment that created this build.', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) + // Build Status + // Failed - The deployment build has failed. More details can usually be found in buildStderr + // Ready - The deployment build was successful and the deployment is ready to be deployed + // Processing - The deployment is currently waiting to have a build triggered + // Building - The deployment is currently being built + ->addRule('status', [ + 'type' => self::TYPE_STRING, + 'description' => 'The build status. There are a few different types and each one means something different. \nFailed - The deployment build has failed. More details can usually be found in buildStderr\nReady - The deployment build was successful and the deployment is ready to be deployed\nProcessing - The deployment is currently waiting to have a build triggered\nBuilding - The deployment is currently being built', + 'default' => '', + 'example' => 'ready', + ]) + ->addRule('stdout', [ + 'type' => self::TYPE_STRING, + 'description' => 'The stdout of the build.', + 'default' => '', + 'example' => '', + ]) + ->addRule('stderr', [ + 'type' => self::TYPE_STRING, + 'description' => 'The stderr of the build.', + 'default' => '', + 'example' => '', + ]) + ->addRule('startTime', [ + 'type' => self::TYPE_DATETIME, + 'description' => 'The deployment creation date in ISO 8601 format.', + 'default' => '', + 'example' => self::TYPE_DATETIME_EXAMPLE, + ]) + ->addRule('endTime', [ + 'type' => self::TYPE_DATETIME, + 'description' => 'The time the build was finished in ISO 8601 format.', + 'default' => '', + 'example' => self::TYPE_DATETIME_EXAMPLE, + ]) + ->addRule('duration', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The build duration in seconds.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('size', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The code size in bytes.', + 'default' => 0, + 'example' => 128, + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'Build'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_BUILD; + } +}