From a10246a9bd13cc1b9a08dfa3d375c8a9bc175036 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 28 Sep 2023 13:45:15 +0300 Subject: [PATCH] build && audit && usage workers --- app/worker.php | 7 +- src/Appwrite/Platform/Workers/Audits.php | 6 +- src/Appwrite/Platform/Workers/Builds.php | 749 +++++++++++++++++--- src/Appwrite/Platform/Workers/UsageHook.php | 6 - 4 files changed, 636 insertions(+), 132 deletions(-) diff --git a/app/worker.php b/app/worker.php index 2b1e6bb2d5..ed499eca9b 100644 --- a/app/worker.php +++ b/app/worker.php @@ -218,8 +218,6 @@ if (isset($args[0])) { Console::error('Missing worker name'); } -if (empty(App::getEnv('QUEUE'))) { - throw new Exception('Please configure "QUEUE" environment variable.'); try { $platform->init(Service::TYPE_WORKER, [ 'workersNum' => swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)), @@ -251,9 +249,6 @@ $worker throw $error; } - if (($error->getCode() >= 500 || $error->getCode() === 0) && !empty($logger)) { - $log = new Log(); - if ($logger && ($error->getCode() >= 500 || $error->getCode() === 0)) { $log->setNamespace("appwrite-worker"); $log->setServer(\gethostname()); @@ -291,4 +286,4 @@ try { }); } -$worker->start(); +$worker->start(); \ No newline at end of file diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index f889f5e939..ac6388855e 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -3,7 +3,7 @@ namespace Appwrite\Platform\Workers; use Exception; -use Utopia\App; +use Throwable; use Utopia\Audit\Audit; use Utopia\Database\Database; use Utopia\Database\Document; @@ -32,6 +32,7 @@ class Audits extends Action /** * @throws Exception + * @throws Throwable */ public function action(Message $message, Database $dbForProject): void { @@ -55,8 +56,7 @@ class Audits extends Action $audit = new Audit($dbForProject); $audit->log( - userInternalId: $user->getInternalId(), - userId: $user->getId(), + userId: $user->getInternalId(), // Pass first, most verbose event pattern event: $event, resource: $resource, diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 7dbdc9b67f..6dbcac637e 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -7,21 +7,26 @@ use Appwrite\Event\Func; use Appwrite\Event\Usage; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Utopia\Response\Model\Deployment; +use Appwrite\Vcs\Comment; use Exception; +use Swoole\Coroutine as Co; use Executor\Executor; use Utopia\App; +use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Exception\Authorization; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; use Utopia\DSN\DSN; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; +use Utopia\VCS\Adapter\Git\GitHub; class Builds extends Action { @@ -43,13 +48,15 @@ class Builds extends Action ->inject('queueForEvents') ->inject('queueForFunctions') ->inject('queueForUsage') - ->callback(fn($message, Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage) => $this->action($message, $dbForConsole, $dbForProject, $queueForEvents, $queueForFunctions, $queueForUsage)); + ->inject('cache') + ->inject('getProjectDB') + ->callback(fn($message, Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, callable $getProjectDB) => $this->action($message, $dbForConsole, $dbForProject, $queueForEvents, $queueForFunctions, $queueForUsage, $cache, $getProjectDB)); } /** * @throws Exception|\Throwable */ - public function action(Message $message, Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage): void + public function action(Message $message, Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, callable $getProjectDB): void { $payload = $message->getPayload() ?? []; @@ -61,21 +68,14 @@ class Builds extends Action $project = new Document($payload['project'] ?? []); $resource = new Document($payload['resource'] ?? []); $deployment = new Document($payload['deployment'] ?? []); + $template = new Document($payload['template'] ?? []); switch ($type) { case BUILD_TYPE_DEPLOYMENT: case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); - $this->buildDeployment( - dbForConsole: $dbForConsole, - dbForProject: $dbForProject, - queueForEvents: $queueForEvents, - queueForFunctions: $queueForFunctions, - queueForUsage: $queueForUsage, - deployment: $deployment, - project: $project, - function: $resource - ); + $github = new GitHub($cache); + $this->buildDeployment($queueForEvents, $queueForUsage, $getProjectDB, $dbForConsole, $github, $project, $resource, $deployment, $template); break; default: @@ -88,11 +88,13 @@ class Builds extends Action * @throws \Throwable * @throws Structure */ - private function buildDeployment(Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Document $deployment, Document $project, Document $function): void + protected function buildDeployment(Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, callable $getProjectDB, GitHub $github, Document $project, Document $function, Document $deployment, Document $template) { $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); - $function = $dbForProject->getDocument('functions', $function->getId()); + $dbForProject = $getProjectDB($project); + + $function = $dbForProject->getDocument('functions', $function->getId()); if ($function->isEmpty()) { throw new Exception('Function not found', 404); } @@ -102,27 +104,33 @@ class Builds extends Action throw new Exception('Deployment not found', 404); } - $runtimes = Config::getParam('runtimes', []); + if (empty($deployment->getAttribute('entrypoint', ''))) { + throw new Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500); + } + + $version = $function->getAttribute('version', 'v2'); + $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); $key = $function->getAttribute('runtime'); $runtime = $runtimes[$key] ?? null; if (\is_null($runtime)) { throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } - $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); - /** @TODO : move this to the registry or someplace else */ - $device = Storage::DEVICE_LOCAL; - try { - $dsn = new DSN($connection); - $device = $dsn->getScheme(); - } catch (\Exception $e) { - Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.'); - } + + // Realtime preparation + $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [ + 'functionId' => $function->getId(), + 'deploymentId' => $deployment->getId() + ]); + + $startTime = DateTime::now(); + $durationStart = \microtime(true); $buildId = $deployment->getAttribute('buildId', ''); - $startTime = DateTime::now(); - if (empty($buildId)) { + $isNewBuild = empty($buildId); + + if ($isNewBuild) { $buildId = ID::unique(); $build = $dbForProject->createDocument('builds', new Document([ '$id' => $buildId, @@ -131,106 +139,323 @@ class Builds extends Action 'deploymentInternalId' => $deployment->getInternalId(), 'deploymentId' => $deployment->getId(), 'status' => 'processing', - 'outputPath' => '', + 'path' => '', 'runtime' => $function->getAttribute('runtime'), - 'source' => $deployment->getAttribute('path'), - 'sourceType' => $device, - 'stdout' => '', - 'stderr' => '', - 'duration' => 0 + 'source' => $deployment->getAttribute('path', ''), + 'sourceType' => strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)), + 'logs' => '', + 'endTime' => null, + 'duration' => 0, + 'size' => 0 ])); - $deployment->setAttribute('buildId', $buildId); + $deployment->setAttribute('buildId', $build->getId()); $deployment->setAttribute('buildInternalId', $build->getInternalId()); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); } else { $build = $dbForProject->getDocument('builds', $buildId); } - /** Request the executor to build the code... */ - $build->setAttribute('status', 'building'); - $build = $dbForProject->updateDocument('builds', $buildId, $build); + $source = $deployment->getAttribute('path', ''); + $installationId = $deployment->getAttribute('installationId', ''); + $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', ''); + $providerCommitHash = $deployment->getAttribute('providerCommitHash', ''); + $isVcsEnabled = $providerRepositoryId ? true : false; + $owner = ''; + $repositoryName = ''; - /** Trigger Webhook */ - $deploymentUpdate = $queueForEvents - ->setQueue(Event::WEBHOOK_QUEUE_NAME) - ->setClass(Event::WEBHOOK_CLASS_NAME) - ->setProject($project) - ->setEvent('functions.[functionId].deployments.[deploymentId].update') - ->setParam('functionId', $function->getId()) - ->setParam('deploymentId', $deployment->getId()) - ->setPayload($deployment->getArrayCopy( - array_keys( - (new Deployment())->getRules() - ) - )); + if ($isVcsEnabled) { + $installation = $dbForConsole->getDocument('installations', $installationId); + $providerInstallationId = $installation->getAttribute('providerInstallationId'); + $privateKey = App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'); + $githubAppId = App::getEnv('_APP_VCS_GITHUB_APP_ID'); - $deploymentUpdate->trigger(); - - /** Trigger Functions */ - $queueForFunctions - ->from($deploymentUpdate) - ->trigger(); - - - /** Trigger Realtime */ - $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [ - 'functionId' => $function->getId(), - 'deploymentId' => $deployment->getId() - ]); - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - - $source = $deployment->getAttribute('path'); - - $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) { - $carry[$var->getAttribute('key')] = $var->getAttribute('value'); - return $carry; - }, []); + $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); + } try { - $response = $executor->createRuntime( - deploymentId: $deployment->getId(), - projectId: $project->getId(), - source: $source, - image: $runtime['image'], - remove: true, - entrypoint: $deployment->getAttribute('entrypoint'), - workdir: '/usr/code', - destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", - variables: $vars, - commands: [ - 'sh', '-c', - 'tar -zxf /tmp/code.tar.gz -C /usr/code && \ - cd /usr/local/src/ && ./build.sh' - ] + if ($isNewBuild && $isVcsEnabled) { + $tmpDirectory = '/tmp/builds/' . $buildId . '/code'; + $rootDirectory = $function->getAttribute('providerRootDirectory', ''); + $rootDirectory = \rtrim($rootDirectory, '/'); + $rootDirectory = \ltrim($rootDirectory, '.'); + $rootDirectory = \ltrim($rootDirectory, '/'); + + $owner = $github->getOwnerName($providerInstallationId); + $repositoryName = $github->getRepositoryName($providerRepositoryId); + + $cloneOwner = $deployment->getAttribute('providerRepositoryOwner', $owner); + $cloneRepository = $deployment->getAttribute('providerRepositoryName', $repositoryName); + + $branchName = $deployment->getAttribute('providerBranch'); + $commitHash = $deployment->getAttribute('providerCommitHash', ''); + $gitCloneCommand = $github->generateCloneCommand($cloneOwner, $cloneRepository, $branchName, $tmpDirectory, $rootDirectory, $commitHash); + $stdout = ''; + $stderr = ''; + Console::execute('mkdir -p /tmp/builds/' . \escapeshellcmd($buildId), '', $stdout, $stderr); + $exit = Console::execute($gitCloneCommand, '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to clone code repository: ' . $stderr); + } + + // Build from template + $templateRepositoryName = $template->getAttribute('repositoryName', ''); + $templateOwnerName = $template->getAttribute('ownerName', ''); + $templateBranch = $template->getAttribute('branch', ''); + + $templateRootDirectory = $template->getAttribute('rootDirectory', ''); + $templateRootDirectory = \rtrim($templateRootDirectory, '/'); + $templateRootDirectory = \ltrim($templateRootDirectory, '.'); + $templateRootDirectory = \ltrim($templateRootDirectory, '/'); + + if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateBranch)) { + // Clone template repo + $tmpTemplateDirectory = '/tmp/builds/' . \escapeshellcmd($buildId) . '/template'; + $gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateBranch, $tmpTemplateDirectory, $templateRootDirectory); + $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to clone code repository: ' . $stderr); + } + + // Ensure directories + Console::execute('mkdir -p ' . $tmpTemplateDirectory . '/' . $templateRootDirectory, '', $stdout, $stderr); + Console::execute('mkdir -p ' . $tmpDirectory . '/' . $rootDirectory, '', $stdout, $stderr); + + // Merge template into user repo + Console::execute('cp -rfn ' . $tmpTemplateDirectory . '/' . $templateRootDirectory . '/* ' . $tmpDirectory . '/' . $rootDirectory, '', $stdout, $stderr); + + // Commit and push + $exit = Console::execute('git config --global user.email "team@appwrite.io" && git config --global user.name "Appwrite" && cd ' . $tmpDirectory . ' && git add . && git commit -m "Create \'' . \escapeshellcmd($function->getAttribute('name', '')) . '\' function" && git push origin ' . \escapeshellcmd($branchName), '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to push code repository: ' . $stderr); + } + + $exit = Console::execute('cd ' . $tmpDirectory . ' && git rev-parse HEAD', '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to get vcs commit SHA: ' . $stderr); + } + + $providerCommitHash = \trim($stdout); + $authorUrl = "https://github.com/$cloneOwner"; + + $deployment->setAttribute('providerCommitHash', $providerCommitHash ?? ''); + $deployment->setAttribute('providerCommitAuthorUrl', $authorUrl); + $deployment->setAttribute('providerCommitAuthor', 'Appwrite'); + $deployment->setAttribute('providerCommitMessage', "Create '" . $function->getAttribute('name', '') . "' function"); + $deployment->setAttribute('providerCommitUrl', "https://github.com/$cloneOwner/$cloneRepository/commit/$providerCommitHash"); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + + /** + * Send realtime Event + */ + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $build, + project: $project + ); + Realtime::send( + projectId: 'console', + payload: $build->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + } + + $tmpPath = '/tmp/builds/' . \escapeshellcmd($buildId); + $tmpPathFile = $tmpPath . '/code.tar.gz'; + + Console::execute('tar --exclude code.tar.gz -czf ' . $tmpPathFile . ' -C /tmp/builds/' . \escapeshellcmd($buildId) . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory) . ' .', '', $stdout, $stderr); + + $deviceFunctions = $this->getFunctionsDevice($project->getId()); + + $localDevice = new Local(); + $buffer = $localDevice->read($tmpPathFile); + $mimeType = $localDevice->getFileMimeType($tmpPathFile); + + $path = $deviceFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); + $result = $deviceFunctions->write($path, $buffer, $mimeType); + + if (!$result) { + throw new \Exception("Unable to move file"); + } + + Console::execute('rm -rf ' . $tmpPath, '', $stdout, $stderr); + + $source = $path; + + $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source)); + + $this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole); + } + + /** Request the executor to build the code... */ + $build->setAttribute('status', 'building'); + $build = $dbForProject->updateDocument('builds', $buildId, $build); + + if ($isVcsEnabled) { + $this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole); + } + + /** Trigger Webhook */ + $deploymentUpdate = + $queueForEvents + ->setQueue(Event::WEBHOOK_QUEUE_NAME) + ->setClass(Event::WEBHOOK_CLASS_NAME) + ->setProject($project) + ->setEvent('functions.[functionId].deployments.[deploymentId].update') + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($deployment->getArrayCopy( + array_keys( + (new Deployment())->getRules() + ) + )); + + $deploymentUpdate->trigger(); + + /** Trigger Functions */ + $queueForFunctions + ->from($deploymentUpdate) + ->trigger(); + + /** Trigger Realtime */ + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $build, + project: $project ); - $endTime = new \DateTime(); - $endTime->setTimestamp($response['endTimeUnix']); + Realtime::send( + projectId: 'console', + payload: $build->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + + $vars = []; + + // Shared vars + foreach ($function->getAttribute('varsProject', []) as $var) { + $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); + } + + // Function vars + foreach ($function->getAttribute('vars', []) as $var) { + $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); + } + + // Appwrite vars + $vars = \array_merge($vars, [ + 'APPWRITE_FUNCTION_ID' => $function->getId(), + 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), + 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), + 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), + 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', + 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', + ]); + + $command = $deployment->getAttribute('commands', ''); + $command = \str_replace('"', '\\"', $command); + + $response = null; + + $err = null; + + // TODO: Remove run() wrapper when switching to new utopia queue. That should be done on Swoole adapter in the libary + Co\run(function () use ($executor, $project, $deployment, &$response, $source, $function, $runtime, $vars, $command, &$build, $dbForProject, $allEvents, &$err) { + Co::join([ + Co\go(function () use ($executor, &$response, $project, $deployment, $source, $function, $runtime, $vars, $command, &$err) { + try { + $version = $function->getAttribute('version', 'v2'); + $command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . $command . '"'; + + $response = $executor->createRuntime( + deploymentId: $deployment->getId(), + projectId: $project->getId(), + source: $source, + image: $runtime['image'], + version: $version, + remove: true, + entrypoint: $deployment->getAttribute('entrypoint'), + destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", + variables: $vars, + command: $command + ); + } catch (Exception $error) { + $err = $error; + } + }), + Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err) { + try { + $executor->getLogs( + deploymentId: $deployment->getId(), + projectId: $project->getId(), + callback: function ($logs) use (&$response, &$build, $dbForProject, $allEvents, $project) { + if ($response === null) { + $build = $dbForProject->getDocument('builds', $build->getId()); + + if ($build->isEmpty()) { + throw new Exception('Build not found', 404); + } + + $build = $build->setAttribute('logs', $build->getAttribute('logs', '') . $logs); + $build = $dbForProject->updateDocument('builds', $build->getId(), $build); + + /** + * Send realtime Event + */ + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $build, + project: $project + ); + Realtime::send( + projectId: 'console', + payload: $build->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + } + } + ); + } catch (Exception $error) { + if (empty($err)) { + $err = $error; + } + } + }), + ]); + }); + + if ($err) { + throw $err; + } + + $endTime = DateTime::now(); + $durationEnd = \microtime(true); /** Update the build document */ - $build->setAttribute('endTime', DateTime::format($endTime)); - $build->setAttribute('duration', \intval($response['duration'])); - $build->setAttribute('status', $response['status']); - $build->setAttribute('outputPath', $response['outputPath']); - $build->setAttribute('stderr', $response['stderr']); - $build->setAttribute('stdout', $response['stdout']); + $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime']))); + $build->setAttribute('endTime', $endTime); + $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); + $build->setAttribute('status', 'ready'); + $build->setAttribute('path', $response['path']); + $build->setAttribute('size', $response['size']); + $build->setAttribute('logs', $response['output']); - /* Also update the deployment buildTime */ - $deployment->setAttribute('buildTime', $response['duration']); + if ($isVcsEnabled) { + $this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole); + } Console::success("Build id: $buildId created"); @@ -238,27 +463,33 @@ class Builds extends Action if ($deployment->getAttribute('activate') === true) { $function->setAttribute('deploymentInternalId', $deployment->getInternalId()); $function->setAttribute('deployment', $deployment->getId()); + $function->setAttribute('live', true); $function = $dbForProject->updateDocument('functions', $function->getId(), $function); } /** Update function schedule */ - $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); - $schedule->setAttribute('resourceUpdatedAt', DateTime::now()); + // Inform scheduler if function is still active + $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) ->setAttribute('schedule', $function->getAttribute('schedule')) ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - - - \Utopia\Database\Validator\Authorization::skip(fn() => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); + Authorization::skip(fn() => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); } catch (\Throwable $th) { $endTime = DateTime::now(); - $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime)); + $durationEnd = \microtime(true); $build->setAttribute('endTime', $endTime); - $build->setAttribute('duration', $interval->format('%s') + 0); + $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); $build->setAttribute('status', 'failed'); - $build->setAttribute('stderr', $th->getMessage()); + $build->setAttribute('logs', $th->getMessage()); Console::error($th->getMessage()); + Console::error($th->getFile() . ':' . $th->getLine()); + Console::error($th->getTraceAsString()); + + if ($isVcsEnabled) { + $this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole); + } } finally { $build = $dbForProject->updateDocument('builds', $buildId, $build); @@ -291,4 +522,288 @@ class Builds extends Action ->trigger(); } } + + protected function runGitAction(string $status, GitHub $github, string $providerCommitHash, string $owner, string $repositoryName, Document $project, Document $function, string $deploymentId, Database $dbForProject, Database $dbForConsole): void + { + if ($function->getAttribute('providerSilentMode', false) === true) { + return; + } + + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + $commentId = $deployment->getAttribute('providerCommentId', ''); + + if (!empty($providerCommitHash)) { + $message = match ($status) { + 'ready' => 'Build succeeded.', + 'failed' => 'Build failed.', + 'processing' => 'Building...', + default => $status + }; + + $state = match ($status) { + 'ready' => 'success', + 'failed' => 'failure', + 'processing' => 'pending', + default => $status + }; + + $functionName = $function->getAttribute('name'); + $projectName = $project->getAttribute('name'); + + $name = "{$functionName} ({$projectName})"; + + $protocol = App::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; + $hostname = App::getEnv('_APP_DOMAIN'); + $functionId = $function->getId(); + $projectId = $project->getId(); + $providerTargetUrl = $protocol . '://' . $hostname . "/console/project-$projectId/functions/function-$functionId"; + + $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, $state, $message, $providerTargetUrl, $name); + } + + if (!empty($commentId)) { + $retries = 0; + + while (true) { + $retries++; + + try { + $dbForConsole->createDocument('vcsCommentLocks', new Document([ + '$id' => $commentId + ])); + break; + } catch (Exception $err) { + if ($retries >= 9) { + throw $err; + } + + \sleep(1); + } + } + + // Wrap in try/finally to ensure lock file gets deleted + try { + $comment = new Comment(); + $comment->parseComment($github->getComment($owner, $repositoryName, $commentId)); + $comment->addBuild($project, $function, $status, $deployment->getId(), ['type' => 'logs']); + $github->updateComment($owner, $repositoryName, $commentId, $comment->generateComment()); + } finally { + $dbForConsole->deleteDocument('vcsCommentLocks', $commentId); + } + } + } } + +// // Realtime preparation +// $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [ +// 'functionId' => $function->getId(), +// 'deploymentId' => $deployment->getId() +// ]); +// +// $startTime = DateTime::now(); +// $durationStart = \microtime(true); +// +// +// +// $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); +// $function = $dbForProject->getDocument('functions', $function->getId()); +// +// if ($function->isEmpty()) { +// throw new Exception('Function not found', 404); +// } +// +// $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); +// if ($deployment->isEmpty()) { +// throw new Exception('Deployment not found', 404); +// } +// +// $runtimes = Config::getParam('runtimes', []); +// $key = $function->getAttribute('runtime'); +// $runtime = $runtimes[$key] ?? null; +// if (\is_null($runtime)) { +// throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); +// } +// +// $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); +// /** @TODO : move this to the registry or someplace else */ +// $device = Storage::DEVICE_LOCAL; +// try { +// $dsn = new DSN($connection); +// $device = $dsn->getScheme(); +// } catch (\Exception $e) { +// Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.'); +// } +// +// $buildId = $deployment->getAttribute('buildId', ''); +// $startTime = DateTime::now(); +// +// if (empty($buildId)) { +// $buildId = ID::unique(); +// $build = $dbForProject->createDocument('builds', new Document([ +// '$id' => $buildId, +// '$permissions' => [], +// 'startTime' => $startTime, +// 'deploymentInternalId' => $deployment->getInternalId(), +// 'deploymentId' => $deployment->getId(), +// 'status' => 'processing', +// 'outputPath' => '', +// 'runtime' => $function->getAttribute('runtime'), +// 'source' => $deployment->getAttribute('path'), +// 'sourceType' => $device, +// 'stdout' => '', +// 'stderr' => '', +// 'duration' => 0 +// ])); +// +// $deployment->setAttribute('buildId', $buildId); +// $deployment->setAttribute('buildInternalId', $build->getInternalId()); +// $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); +// } else { +// $build = $dbForProject->getDocument('builds', $buildId); +// } +// +// /** Request the executor to build the code... */ +// $build->setAttribute('status', 'building'); +// $build = $dbForProject->updateDocument('builds', $buildId, $build); +// +// /** Trigger Webhook */ +// $deploymentUpdate = $queueForEvents +// ->setQueue(Event::WEBHOOK_QUEUE_NAME) +// ->setClass(Event::WEBHOOK_CLASS_NAME) +// ->setProject($project) +// ->setEvent('functions.[functionId].deployments.[deploymentId].update') +// ->setParam('functionId', $function->getId()) +// ->setParam('deploymentId', $deployment->getId()) +// ->setPayload($deployment->getArrayCopy( +// array_keys( +// (new Deployment())->getRules() +// ) +// )); +// +// $deploymentUpdate->trigger(); +// +// /** Trigger Functions */ +// $queueForFunctions +// ->from($deploymentUpdate) +// ->trigger(); +// +// +// /** Trigger Realtime */ +// $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [ +// 'functionId' => $function->getId(), +// 'deploymentId' => $deployment->getId() +// ]); +// $target = Realtime::fromPayload( +// // Pass first, most verbose event pattern +// event: $allEvents[0], +// payload: $build, +// project: $project +// ); +// +// Realtime::send( +// projectId: 'console', +// payload: $build->getArrayCopy(), +// events: $allEvents, +// channels: $target['channels'], +// roles: $target['roles'] +// ); +// +// $source = $deployment->getAttribute('path'); +// +// $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) { +// $carry[$var->getAttribute('key')] = $var->getAttribute('value'); +// return $carry; +// }, []); +// +// try { +// $response = $executor->createRuntime( +// deploymentId: $deployment->getId(), +// projectId: $project->getId(), +// source: $source, +// image: $runtime['image'], +// remove: true, +// entrypoint: $deployment->getAttribute('entrypoint'), +// workdir: '/usr/code', +// destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", +// variables: $vars, +// commands: [ +// 'sh', '-c', +// 'tar -zxf /tmp/code.tar.gz -C /usr/code && \ +// cd /usr/local/src/ && ./build.sh' +// ] +// ); +// +// $endTime = new \DateTime(); +// $endTime->setTimestamp($response['endTimeUnix']); +// +// /** Update the build document */ +// $build->setAttribute('endTime', DateTime::format($endTime)); +// $build->setAttribute('duration', \intval($response['duration'])); +// $build->setAttribute('status', $response['status']); +// $build->setAttribute('outputPath', $response['outputPath']); +// $build->setAttribute('stderr', $response['stderr']); +// $build->setAttribute('stdout', $response['stdout']); +// +// /* Also update the deployment buildTime */ +// $deployment->setAttribute('buildTime', $response['duration']); +// +// Console::success("Build id: $buildId created"); +// +// /** Set auto deploy */ +// if ($deployment->getAttribute('activate') === true) { +// $function->setAttribute('deploymentInternalId', $deployment->getInternalId()); +// $function->setAttribute('deployment', $deployment->getId()); +// $function = $dbForProject->updateDocument('functions', $function->getId(), $function); +// } +// +// /** Update function schedule */ +// $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId')); +// $schedule->setAttribute('resourceUpdatedAt', DateTime::now()); +// +// $schedule +// ->setAttribute('schedule', $function->getAttribute('schedule')) +// ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); +// +// +// \Utopia\Database\Validator\Authorization::skip(fn() => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule)); +// } catch (\Throwable $th) { +// $endTime = DateTime::now(); +// $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime)); +// $build->setAttribute('endTime', $endTime); +// $build->setAttribute('duration', $interval->format('%s') + 0); +// $build->setAttribute('status', 'failed'); +// $build->setAttribute('stderr', $th->getMessage()); +// Console::error($th->getMessage()); +// } finally { +// $build = $dbForProject->updateDocument('builds', $buildId, $build); +// +// /** +// * Send realtime Event +// */ +// $target = Realtime::fromPayload( +// // Pass first, most verbose event pattern +// event: $allEvents[0], +// payload: $build, +// project: $project +// ); +// Realtime::send( +// projectId: 'console', +// payload: $build->getArrayCopy(), +// events: $allEvents, +// channels: $target['channels'], +// roles: $target['roles'] +// ); +// +// /** Trigger usage queue */ +// $queueForUsage +// ->setProject($project) +// ->addMetric(METRIC_BUILDS, 1) // per project +// ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) +// ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) +// ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function +// ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) +// ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) +// ->trigger(); +// } +// } +//} diff --git a/src/Appwrite/Platform/Workers/UsageHook.php b/src/Appwrite/Platform/Workers/UsageHook.php index 06d5cb9feb..f0bd0ac2c9 100644 --- a/src/Appwrite/Platform/Workers/UsageHook.php +++ b/src/Appwrite/Platform/Workers/UsageHook.php @@ -89,12 +89,6 @@ class UsageHook extends Usage } } } - if (!empty($project['keys'])) { - $dbForProject->createDocument('statsLogger', new Document([ - 'time' => DateTime::now(), - 'metrics' => $project['keys'], - ])); - } } catch (\Exception $e) { console::error("[logger] " . " {DateTime::now()} " . " {$projectInternalId} " . " {$e->getMessage()}"); } finally {