diff --git a/app/cli.php b/app/cli.php index 0f8426afd9..052643f004 100644 --- a/app/cli.php +++ b/app/cli.php @@ -318,6 +318,10 @@ $setResource('logError', function (Registry $register) { $setResource('executor', fn () => new Executor(), []); +$setResource('bus', function (Registry $register) use ($cli) { + return $register->get('bus')->setResolver(fn (string $name) => $cli->getResource($name)); +}, ['register']); + $setResource('telemetry', fn () => new NoTelemetry(), []); $cli diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index b24c6dd75d..7b4774ff6a 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -2199,13 +2199,6 @@ return [ 'lengths' => [], 'orders' => [Database::ORDER_ASC], ], - [ - '$id' => ID::custom('_key_function_internal_id'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['resourceInternalId'], - 'lengths' => [], - 'orders' => [], - ], [ '$id' => ID::custom('_key_resourceType'), 'type' => Database::INDEX_KEY, diff --git a/app/config/services.php b/app/config/services.php index 0bbba96032..a99501c530 100644 --- a/app/config/services.php +++ b/app/config/services.php @@ -188,7 +188,7 @@ return [ 'name' => 'VCS', 'subtitle' => 'The VCS service allows you to interact with providers like GitHub, GitLab etc.', 'description' => '', - 'controller' => 'api/vcs.php', + 'controller' => '', // Uses modules 'sdk' => false, 'docs' => false, 'docsUrl' => '', diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php deleted file mode 100644 index 6dd18ed0f3..0000000000 --- a/app/controllers/api/vcs.php +++ /dev/null @@ -1,705 +0,0 @@ -getAttribute('resourceType'); - - if ($resourceType !== "function" && $resourceType !== "site") { - continue; - } - - $projectId = $repository->getAttribute('projectId'); - $project = $authorization->skip(fn () => $dbForPlatform->getDocument('projects', $projectId)); - if ($project->isEmpty()) { - throw new Exception(Exception::PROJECT_NOT_FOUND, 'Repository references non-existent project'); - } - $dbForProject = $getProjectDB($project); - - $resourceCollection = $resourceType === "function" ? 'functions' : 'sites'; - $resourceId = $repository->getAttribute('resourceId'); - $resource = $authorization->skip(fn () => $dbForProject->getDocument($resourceCollection, $resourceId)); - $resourceInternalId = $resource->getSequence(); - - $deploymentId = ID::unique(); - $repositoryId = $repository->getId(); - $repositoryInternalId = $repository->getSequence(); - $providerRepositoryId = $repository->getAttribute('providerRepositoryId'); - $installationId = $repository->getAttribute('installationId'); - $installationInternalId = $repository->getAttribute('installationInternalId'); - $productionBranch = $resource->getAttribute('providerBranch'); - $activate = false; - - if ($providerBranch == $productionBranch && $external === false) { - $activate = true; - } - - $owner = $github->getOwnerName($providerInstallationId) ?? ''; - try { - $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; - if (empty($repositoryName)) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - } catch (RepositoryNotFound $e) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - - if (empty($repositoryName)) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - - $isAuthorized = !$external; - - if (!$isAuthorized && !empty($providerPullRequestId)) { - if (\in_array($providerPullRequestId, $repository->getAttribute('providerPullRequestIds', []))) { - $isAuthorized = true; - } - } - - $commentStatus = $isAuthorized ? 'waiting' : 'failed'; - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; - $hostname = $platform['consoleHostname'] ?? ''; - - $authorizeUrl = $protocol . '://' . $hostname . "/console/git/authorize-contributor?projectId={$projectId}&installationId={$installationId}&repositoryId={$repositoryId}&providerPullRequestId={$providerPullRequestId}"; - - $action = $isAuthorized ? ['type' => 'logs'] : ['type' => 'authorize', 'url' => $authorizeUrl]; - - $latestCommentId = ''; - - if (!empty($providerPullRequestId) && $resource->getAttribute('providerSilentMode', false) === false) { - $latestComment = $authorization->skip(fn () => $dbForPlatform->findOne('vcsComments', [ - Query::equal('providerRepositoryId', [$providerRepositoryId]), - Query::equal('providerPullRequestId', [$providerPullRequestId]), - Query::orderDesc('$createdAt'), - ])); - - if (!$latestComment->isEmpty()) { - $latestCommentId = $latestComment->getAttribute('providerCommentId', ''); - - $retries = 0; - $lockAcquired = false; - - while ($retries < 9) { - $retries++; - - try { - $dbForPlatform->createDocument('vcsCommentLocks', new Document([ - '$id' => $latestCommentId - ])); - $lockAcquired = true; - break; - } catch (\Throwable $err) { - if ($retries >= 9) { - Console::warning("Error creating vcs comment lock for " . $latestCommentId . ": " . $err->getMessage()); - } - - \sleep(1); - } - } - - if ($lockAcquired) { - // Wrap in try/finally to ensure lock file gets deleted - try { - $comment = new Comment($platform); - $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); - $comment->addBuild($project, $resource, $resourceType, $commentStatus, $deploymentId, $action, ''); - - $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); - } finally { - $authorization->skip(fn () => $dbForPlatform->deleteDocument('vcsCommentLocks', $latestCommentId)); - } - } - } else { - $comment = new Comment($platform); - $comment->addBuild($project, $resource, $resourceType, $commentStatus, $deploymentId, $action, ''); - $latestCommentId = \strval($github->createComment($owner, $repositoryName, $providerPullRequestId, $comment->generateComment())); - - if (!empty($latestCommentId)) { - $teamId = $project->getAttribute('teamId', ''); - - $latestComment = $authorization->skip(fn () => $dbForPlatform->createDocument('vcsComments', new Document([ - '$id' => ID::unique(), - '$permissions' => [ - Permission::read(Role::team(ID::custom($teamId))), - Permission::update(Role::team(ID::custom($teamId), 'owner')), - Permission::update(Role::team(ID::custom($teamId), 'developer')), - Permission::delete(Role::team(ID::custom($teamId), 'owner')), - Permission::delete(Role::team(ID::custom($teamId), 'developer')), - ], - 'installationInternalId' => $installationInternalId, - 'installationId' => $installationId, - 'projectInternalId' => $project->getSequence(), - 'projectId' => $project->getId(), - 'providerRepositoryId' => $providerRepositoryId, - 'providerBranch' => $providerBranch, - 'providerPullRequestId' => $providerPullRequestId, - 'providerCommentId' => $latestCommentId - ]))); - } - } - } elseif (!empty($providerBranch)) { - $latestComments = $authorization->skip(fn () => $dbForPlatform->find('vcsComments', [ - Query::equal('providerRepositoryId', [$providerRepositoryId]), - Query::equal('providerBranch', [$providerBranch]), - Query::orderDesc('$createdAt'), - ])); - - foreach ($latestComments as $comment) { - $latestCommentId = $comment->getAttribute('providerCommentId', ''); - - $retries = 0; - $lockAcquired = false; - - while ($retries < 9) { - $retries++; - - try { - $dbForPlatform->createDocument('vcsCommentLocks', new Document([ - '$id' => $latestCommentId - ])); - $lockAcquired = true; - break; - } catch (\Throwable $err) { - if ($retries >= 9) { - Console::warning("Error creating vcs comment lock for " . $latestCommentId . ": " . $err->getMessage()); - } - - \sleep(1); - } - } - - if ($lockAcquired) { - // Wrap in try/finally to ensure lock file gets deleted - try { - $comment = new Comment($platform); - $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); - $comment->addBuild($project, $resource, $resourceType, $commentStatus, $deploymentId, $action, ''); - - $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); - } finally { - $authorization->skip(fn () => $dbForPlatform->deleteDocument('vcsCommentLocks', $latestCommentId)); - } - } - } - } - - if (!$isAuthorized) { - $resourceName = $resource->getAttribute('name'); - $projectName = $project->getAttribute('name'); - $name = "{$resourceName} ({$projectName})"; - $message = 'Authorization required for external contributor.'; - - $providerRepositoryId = $repository->getAttribute('providerRepositoryId'); - try { - $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; - if (empty($repositoryName)) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - } catch (RepositoryNotFound $e) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - $owner = $github->getOwnerName($providerInstallationId); - $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'failure', $message, $authorizeUrl, $name); - continue; - } - - if ($external) { - $pullRequestResponse = $github->getPullRequest($owner, $repositoryName, $providerPullRequestId); - $providerRepositoryName = $pullRequestResponse['head']['repo']['owner']['login']; - $providerRepositoryOwner = $pullRequestResponse['head']['repo']['name']; - } - - $commands = []; - if (!empty($resource->getAttribute('installCommand', ''))) { - $commands[] = $resource->getAttribute('installCommand', ''); - } - if (!empty($resource->getAttribute('buildCommand', ''))) { - $commands[] = $resource->getAttribute('buildCommand', ''); - } - if (!empty($resource->getAttribute('commands', ''))) { - $commands[] = $resource->getAttribute('commands', ''); - } - - $deployment = $authorization->skip(fn () => $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceId' => $resourceId, - 'resourceInternalId' => $resourceInternalId, - 'resourceType' => $resourceCollection, - 'entrypoint' => $resource->getAttribute('entrypoint', ''), - 'buildCommands' => \implode(' && ', $commands), - 'startCommand' => $resource->getAttribute('startCommand', ''), - 'buildOutput' => $resource->getAttribute('outputDirectory', ''), - 'adapter' => $resource->getAttribute('adapter', ''), - 'fallbackFile' => $resource->getAttribute('fallbackFile', ''), - 'type' => 'vcs', - 'installationId' => $installationId, - 'installationInternalId' => $installationInternalId, - 'providerRepositoryId' => $providerRepositoryId, - 'repositoryId' => $repositoryId, - 'repositoryInternalId' => $repositoryInternalId, - 'providerBranchUrl' => $providerBranchUrl, - 'providerRepositoryName' => $providerRepositoryName, - 'providerRepositoryOwner' => $providerRepositoryOwner, - 'providerRepositoryUrl' => $providerRepositoryUrl, - 'providerCommitHash' => $providerCommitHash, - 'providerCommitAuthorUrl' => $providerCommitAuthorUrl, - 'providerCommitAuthor' => $providerCommitAuthor, - 'providerCommitMessage' => mb_strimwidth($providerCommitMessage, 0, 255, '...'), - 'providerCommitUrl' => $providerCommitUrl, - 'providerCommentId' => \strval($latestCommentId), - 'providerBranch' => $providerBranch, - 'activate' => $activate, - ]))); - - $resource = $resource - ->setAttribute('latestDeploymentId', $deployment->getId()) - ->setAttribute('latestDeploymentInternalId', $deployment->getSequence()) - ->setAttribute('latestDeploymentCreatedAt', $deployment->getCreatedAt()) - ->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', '')); - $authorization->skip(fn () => $dbForProject->updateDocument($resource->getCollection(), $resource->getId(), $resource)); - - if ($resource->getCollection() === 'sites') { - $projectId = $project->getId(); - - // Deployment preview - $sitesDomain = $platform['sitesDomain']; - $domain = ID::unique() . "." . $sitesDomain; - $ruleId = md5($domain); - $previewRuleId = $ruleId; - $authorization->skip( - fn () => $dbForPlatform->createDocument('rules', new Document([ - '$id' => $ruleId, - 'projectId' => $project->getId(), - 'projectInternalId' => $project->getSequence(), - 'domain' => $domain, - 'type' => 'deployment', - 'trigger' => 'deployment', - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - 'deploymentResourceType' => 'site', - 'deploymentResourceId' => $resourceId, - 'deploymentResourceInternalId' => $resourceInternalId, - 'deploymentVcsProviderBranch' => $providerBranch, - 'status' => 'verified', - 'certificateId' => '', - 'search' => implode(' ', [$ruleId, $domain]), - 'owner' => 'Appwrite', - 'region' => $project->getAttribute('region') - ])) - ); - - // VCS branch preview - if (!empty($providerBranch)) { - $domain = (new BranchDomainFilter())->apply([ - 'branch' => $providerBranch, - 'resourceId' => $resource->getId(), - 'projectId' => $project->getId(), - 'sitesDomain' => $sitesDomain, - ]); - $ruleId = md5($domain); - try { - $authorization->skip( - fn () => $dbForPlatform->createDocument('rules', new Document([ - '$id' => $ruleId, - 'projectId' => $project->getId(), - 'projectInternalId' => $project->getSequence(), - 'domain' => $domain, - 'type' => 'deployment', - 'trigger' => 'deployment', - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - 'deploymentResourceType' => 'site', - 'deploymentResourceId' => $resourceId, - 'deploymentResourceInternalId' => $resourceInternalId, - 'deploymentVcsProviderBranch' => $providerBranch, - 'status' => 'verified', - 'certificateId' => '', - 'search' => implode(' ', [$ruleId, $domain]), - 'owner' => 'Appwrite', - 'region' => $project->getAttribute('region') - ])) - ); - } catch (Duplicate $err) { - // Ignore, rule already exists; will be updated by builds worker - } - } - - // VCS commit preview - if (!empty($providerCommitHash)) { - $domain = "commit-" . substr($providerCommitHash, 0, 16) . ".{$sitesDomain}"; - $ruleId = md5($domain); - try { - $authorization->skip( - fn () => $dbForPlatform->createDocument('rules', new Document([ - '$id' => $ruleId, - 'projectId' => $project->getId(), - 'projectInternalId' => $project->getSequence(), - 'domain' => $domain, - 'type' => 'deployment', - 'trigger' => 'deployment', - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - 'deploymentResourceType' => 'site', - 'deploymentResourceId' => $resourceId, - 'deploymentResourceInternalId' => $resourceInternalId, - 'deploymentVcsProviderBranch' => $providerBranch, - 'status' => 'verified', - 'certificateId' => '', - 'search' => implode(' ', [$ruleId, $domain]), - 'owner' => 'Appwrite', - 'region' => $project->getAttribute('region') - ])) - ); - } catch (Duplicate $err) { - // Ignore, rule already exists; will be updated by builds worker - } - } - } - - if ($resource->getCollection() === 'sites' && !empty($latestCommentId) && !empty($previewRuleId)) { - $retries = 0; - $lockAcquired = false; - - while ($retries < 9) { - $retries++; - - try { - $dbForPlatform->createDocument('vcsCommentLocks', new Document([ - '$id' => $latestCommentId - ])); - $lockAcquired = true; - break; - } catch (\Throwable $err) { - if ($retries >= 9) { - Console::warning("Error creating vcs comment lock for " . $latestCommentId . ": " . $err->getMessage()); - } - - \sleep(1); - } - } - - if ($lockAcquired) { - // Wrap in try/finally to ensure lock file gets deleted - try { - $rule = $authorization->skip(fn () => $dbForPlatform->getDocument('rules', $previewRuleId)); - - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; - $previewUrl = !empty($rule) ? ("{$protocol}://" . $rule->getAttribute('domain', '')) : ''; - - if (!empty($previewUrl)) { - $comment = new Comment($platform); - $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); - $comment->addBuild($project, $resource, $resourceType, $commentStatus, $deploymentId, $action, $previewUrl); - $github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment()); - } - } finally { - $authorization->skip(fn () => $dbForPlatform->deleteDocument('vcsCommentLocks', $latestCommentId)); - } - } - } - - if (!empty($providerCommitHash) && $resource->getAttribute('providerSilentMode', false) === false) { - $resourceName = $resource->getAttribute('name'); - $projectName = $project->getAttribute('name'); - $region = $project->getAttribute('region', 'default'); - $name = "{$resourceName} ({$projectName})"; - $message = 'Starting...'; - - $providerRepositoryId = $repository->getAttribute('providerRepositoryId'); - try { - $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; - if (empty($repositoryName)) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - } catch (RepositoryNotFound $e) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - $owner = $github->getOwnerName($providerInstallationId); - - $providerTargetUrl = $protocol . '://' . $hostname . "/console/project-$region-$projectId/$resourceCollection/$resourceType-$resourceId"; - $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name); - } - - $queueForBuilds - ->setType(BUILD_TYPE_DEPLOYMENT) - ->setResource($resource) - ->setDeployment($deployment) - ->setProject($project); // set the project because it won't be set for git deployments - - $queueForBuilds->trigger(); // must trigger here so that we create a build for each function/site - - //TODO: Add event? - } catch (Throwable $e) { - $errors[] = $e->getMessage(); - } - } - - $queueForBuilds->reset(); // prevent shutdown hook from triggering again - - if (!empty($errors)) { - throw new Exception(Exception::GENERAL_UNKNOWN, \implode("\n", $errors)); - } -}; - -Http::post('/v1/vcs/github/events') - ->desc('Create event') - ->groups(['api', 'vcs']) - ->label('scope', 'public') - ->inject('gitHub') - ->inject('request') - ->inject('response') - ->inject('dbForPlatform') - ->inject('authorization') - ->inject('getProjectDB') - ->inject('queueForBuilds') - ->inject('platform') - ->action( - function (GitHub $github, Request $request, Response $response, Database $dbForPlatform, Authorization $authorization, callable $getProjectDB, Build $queueForBuilds, array $platform) use ($createGitDeployments) { - $payload = $request->getRawPayload(); - $signatureRemote = $request->getHeader('x-hub-signature-256', ''); - $signatureLocal = System::getEnv('_APP_VCS_GITHUB_WEBHOOK_SECRET', ''); - - $valid = empty($signatureRemote) ? true : $github->validateWebhookEvent($payload, $signatureRemote, $signatureLocal); - - if (!$valid) { - throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, "Invalid webhook payload signature. Please make sure the webhook secret has same value in your GitHub app and in the _APP_VCS_GITHUB_WEBHOOK_SECRET environment variable"); - } - - $event = $request->getHeader('x-github-event', ''); - $privateKey = System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'); - $githubAppId = System::getEnv('_APP_VCS_GITHUB_APP_ID'); - $parsedPayload = $github->getEvent($event, $payload); - - if ($event == $github::EVENT_PUSH) { - $providerBranchCreated = $parsedPayload["branchCreated"] ?? false; - $providerBranchDeleted = $parsedPayload["branchDeleted"] ?? false; - $providerBranch = $parsedPayload["branch"] ?? ''; - $providerBranchUrl = $parsedPayload["branchUrl"] ?? ''; - $providerRepositoryId = $parsedPayload["repositoryId"] ?? ''; - $providerRepositoryName = $parsedPayload["repositoryName"] ?? ''; - $providerInstallationId = $parsedPayload["installationId"] ?? ''; - $providerRepositoryUrl = $parsedPayload["repositoryUrl"] ?? ''; - $providerCommitHash = $parsedPayload["commitHash"] ?? ''; - $providerRepositoryOwner = $parsedPayload["owner"] ?? ''; - $providerCommitAuthorName = $parsedPayload["headCommitAuthorName"] ?? ''; - $providerCommitAuthorEmail = $parsedPayload["headCommitAuthorEmail"] ?? ''; - $providerCommitAuthorUrl = $parsedPayload["authorUrl"] ?? ''; - $providerCommitMessage = $parsedPayload["headCommitMessage"] ?? ''; - $providerCommitUrl = $parsedPayload["headCommitUrl"] ?? ''; - - $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); - - //find resourceId from relevant resources table - $repositories = $authorization->skip(fn () => $dbForPlatform->find('repositories', [ - Query::equal('providerRepositoryId', [$providerRepositoryId]), - Query::limit(100), - ])); - - // create new deployment only on push (not committed by us) and not when branch is created or deleted - if ($providerCommitAuthorEmail !== APP_VCS_GITHUB_EMAIL && !$providerBranchCreated && !$providerBranchDeleted) { - $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', false, $dbForPlatform, $authorization, $queueForBuilds, $getProjectDB, $request, $platform); - } - } elseif ($event == $github::EVENT_INSTALLATION) { - if ($parsedPayload["action"] == "deleted") { - // TODO: Use worker for this job instead (update function/site as well) - $providerInstallationId = $parsedPayload["installationId"]; - - $installations = $dbForPlatform->find('installations', [ - Query::equal('providerInstallationId', [$providerInstallationId]), - Query::limit(1000) - ]); - - foreach ($installations as $installation) { - $repositories = $authorization->skip(fn () => $dbForPlatform->find('repositories', [ - Query::equal('installationInternalId', [$installation->getSequence()]), - Query::limit(1000) - ])); - - foreach ($repositories as $repository) { - $authorization->skip(fn () => $dbForPlatform->deleteDocument('repositories', $repository->getId())); - } - - $authorization->skip(fn () => $dbForPlatform->deleteDocument('installations', $installation->getId())); - } - } - } elseif ($event == $github::EVENT_PULL_REQUEST) { - if ($parsedPayload["action"] == "opened" || $parsedPayload["action"] == "reopened" || $parsedPayload["action"] == "synchronize") { - $providerBranch = $parsedPayload["branch"] ?? ''; - $providerBranchUrl = $parsedPayload["branchUrl"] ?? ''; - $providerRepositoryId = $parsedPayload["repositoryId"] ?? ''; - $providerRepositoryName = $parsedPayload["repositoryName"] ?? ''; - $providerInstallationId = $parsedPayload["installationId"] ?? ''; - $providerRepositoryUrl = $parsedPayload["repositoryUrl"] ?? ''; - $providerPullRequestId = $parsedPayload["pullRequestNumber"] ?? ''; - $providerCommitHash = $parsedPayload["commitHash"] ?? ''; - $providerRepositoryOwner = $parsedPayload["owner"] ?? ''; - $external = $parsedPayload["external"] ?? true; - $providerCommitUrl = $parsedPayload["headCommitUrl"] ?? ''; - $providerCommitAuthorUrl = $parsedPayload["authorUrl"] ?? ''; - - // Ignore sync for non-external. We handle it in push webhook - if (!$external && $parsedPayload["action"] == "synchronize") { - return $response->json($parsedPayload); - } - - $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); - - $commitDetails = $github->getCommit($providerRepositoryOwner, $providerRepositoryName, $providerCommitHash); - $providerCommitAuthor = $commitDetails["commitAuthor"] ?? ''; - $providerCommitMessage = $commitDetails["commitMessage"] ?? ''; - - $repositories = $authorization->skip(fn () => $dbForPlatform->find('repositories', [ - Query::equal('providerRepositoryId', [$providerRepositoryId]), - Query::orderDesc('$createdAt') - ])); - - $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $external, $dbForPlatform, $authorization, $queueForBuilds, $getProjectDB, $request, $platform); - } elseif ($parsedPayload["action"] == "closed") { - // Allowed external contributions cleanup - - $providerRepositoryId = $parsedPayload["repositoryId"] ?? ''; - $providerPullRequestId = $parsedPayload["pullRequestNumber"] ?? ''; - $external = $parsedPayload["external"] ?? true; - - if ($external) { - $repositories = $authorization->skip(fn () => $dbForPlatform->find('repositories', [ - Query::equal('providerRepositoryId', [$providerRepositoryId]), - Query::orderDesc('$createdAt') - ])); - - foreach ($repositories as $repository) { - $providerPullRequestIds = $repository->getAttribute('providerPullRequestIds', []); - - if (\in_array($providerPullRequestId, $providerPullRequestIds)) { - $providerPullRequestIds = \array_diff($providerPullRequestIds, [$providerPullRequestId]); - $repository = $repository->setAttribute('providerPullRequestIds', $providerPullRequestIds); - $repository = $authorization->skip(fn () => $dbForPlatform->updateDocument('repositories', $repository->getId(), $repository)); - } - } - } - } - } - - $response->json($parsedPayload); - } - ); - -Http::patch('/v1/vcs/github/installations/:installationId/repositories/:repositoryId') - ->desc('Update external deployment (authorize)') - ->groups(['api', 'vcs']) - ->label('scope', 'vcs.write') - ->label('sdk', new Method( - namespace: 'vcs', - group: 'repositories', - name: 'updateExternalDeployments', - description: '/docs/references/vcs/update-external-deployments.md', - auth: [AuthType::ADMIN], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_NOCONTENT, - model: Response::MODEL_NONE, - ) - ] - )) - ->param('installationId', '', new Text(256), 'Installation Id') - ->param('repositoryId', '', new Text(256), 'VCS Repository Id') - ->param('providerPullRequestId', '', new Text(256), 'GitHub Pull Request Id') - ->inject('gitHub') - ->inject('response') - ->inject('project') - ->inject('dbForPlatform') - ->inject('authorization') - ->inject('getProjectDB') - ->inject('queueForBuilds') - ->inject('platform') - ->action(function (string $installationId, string $repositoryId, string $providerPullRequestId, GitHub $github, Request $request, Response $response, Document $project, Database $dbForPlatform, Authorization $authorization, callable $getProjectDB, Build $queueForBuilds, array $platform) use ($createGitDeployments) { - $installation = $dbForPlatform->getDocument('installations', $installationId); - - if ($installation->isEmpty()) { - throw new Exception(Exception::INSTALLATION_NOT_FOUND); - } - - $repository = $authorization->skip(fn () => $dbForPlatform->findOne('repositories', [ - Query::equal('$id', [$repositoryId]), - Query::equal('projectInternalId', [$project->getSequence()]) - ])); - - if ($repository->isEmpty()) { - throw new Exception(Exception::REPOSITORY_NOT_FOUND); - } - - if (\in_array($providerPullRequestId, $repository->getAttribute('providerPullRequestIds', []))) { - throw new Exception(Exception::PROVIDER_CONTRIBUTION_CONFLICT); - } - - $providerPullRequestIds = \array_unique(\array_merge($repository->getAttribute('providerPullRequestIds', []), [$providerPullRequestId])); - $repository = $repository->setAttribute('providerPullRequestIds', $providerPullRequestIds); - - // TODO: Delete from array when PR is closed - - $repository = $authorization->skip(fn () => $dbForPlatform->updateDocument('repositories', $repository->getId(), $repository)); - - $privateKey = System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'); - $githubAppId = System::getEnv('_APP_VCS_GITHUB_APP_ID'); - $providerInstallationId = $installation->getAttribute('providerInstallationId'); - $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); - - $repositories = [$repository]; - $providerRepositoryId = $repository->getAttribute('providerRepositoryId'); - - $owner = $github->getOwnerName($providerInstallationId); - try { - $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; - if (empty($repositoryName)) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - } catch (RepositoryNotFound $e) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - $pullRequestResponse = $github->getPullRequest($owner, $repositoryName, $providerPullRequestId); - - $providerBranch = \explode(':', $pullRequestResponse['head']['label'])[1] ?? ''; - $providerCommitHash = $pullRequestResponse['head']['sha'] ?? ''; - $providerBranchUrl = $pullRequestResponse['head']['repo']['html_url'] ?? ''; - $providerRepositoryName = $pullRequestResponse['head']['repo']['name'] ?? ''; - $providerRepositoryUrl = $pullRequestResponse['head']['repo']['html_url'] ?? ''; - $providerRepositoryOwner = $pullRequestResponse['head']['repo']['owner']['login'] ?? ''; - $providerCommitAuthor = $pullRequestResponse['head']['user']['login'] ?? ''; - $providerCommitAuthorUrl = $pullRequestResponse['head']['user']['html_url'] ?? ''; - $providerCommitMessage = $pullRequestResponse['title'] ?? ''; - $providerCommitUrl = $pullRequestResponse['html_url'] ?? ''; - - $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, '', '', '', '', $providerCommitHash, '', '', '', '', $providerPullRequestId, true, $dbForPlatform, $authorization, $queueForBuilds, $getProjectDB, $request, $platform); - - $response->noContent(); - }); diff --git a/app/controllers/general.php b/app/controllers/general.php index 2ac03368df..43c7e47ca6 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -5,11 +5,11 @@ require_once __DIR__ . '/../init.php'; use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Key; +use Appwrite\Bus\Events\ExecutionCompleted; +use Appwrite\Bus\Events\RequestCompleted; use Appwrite\Event\Certificate; use Appwrite\Event\Delete as DeleteEvent; use Appwrite\Event\Event; -use Appwrite\Event\Execution; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Network\Cors; use Appwrite\Platform\Appwrite; @@ -35,6 +35,7 @@ use Executor\Executor; use MaxMind\Db\Reader; use Swoole\Http\Request as SwooleRequest; use Swoole\Table; +use Utopia\Bus\Bus; use Utopia\Config\Config; use Utopia\Console; use Utopia\Database\Database; @@ -62,7 +63,7 @@ Config::setParam('domainVerification', false); Config::setParam('cookieDomain', 'localhost'); Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, StatsUsage $queueForStatsUsage, Execution $queueForExecutions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeleteEvent $queueForDeletes, int $executionsRetentionCount) +function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { $host = $request->getHostname() ?? ''; if (!empty($previewHostname)) { @@ -706,10 +707,12 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S } } finally { if ($type === 'function' || $type === 'site') { - $queueForExecutions - ->setExecution($execution) - ->setProject($project) - ->trigger(); + $bus->dispatch(new ExecutionCompleted( + execution: $execution->getArrayCopy(), + project: $project->getArrayCopy(), + spec: $spec, + resource: $resource->getArrayCopy(), + )); } } @@ -754,70 +757,12 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S ->setStatusCode($execution['responseStatusCode'] ?? 200) ->send($body); - $fileSize = 0; - $file = $request->getFiles('file'); - if (!empty($file)) { - $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; - } - - if (!empty($apiKey) && !empty($apiKey->getDisabledMetrics())) { - foreach ($apiKey->getDisabledMetrics() as $key) { - $queueForStatsUsage->disableMetric($key); - } - } - - $metricTypeExecutions = str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_EXECUTIONS); - $metricTypeIdExecutions = str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_EXECUTIONS); - $metricTypeExecutionsCompute = str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_EXECUTIONS_COMPUTE); - $metricTypeIdExecutionsCompute = str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_COMPUTE); - $metricTypeExecutionsMbSeconds = str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_EXECUTIONS_MB_SECONDS); - $metricTypeIdExecutionsMBSeconds = str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_MB_SECONDS); - if ($deployment->getAttribute('resourceType') === 'sites') { - $queueForStatsUsage - ->disableMetric(METRIC_NETWORK_REQUESTS) - ->disableMetric(METRIC_NETWORK_INBOUND) - ->disableMetric(METRIC_NETWORK_OUTBOUND); - if ($resource->getAttribute('adapter') !== 'ssr') { - $queueForStatsUsage - ->disableMetric(METRIC_EXECUTIONS) - ->disableMetric(METRIC_EXECUTIONS_COMPUTE) - ->disableMetric(METRIC_EXECUTIONS_MB_SECONDS) - ->disableMetric($metricTypeExecutions) - ->disableMetric($metricTypeIdExecutions) - ->disableMetric($metricTypeExecutionsCompute) - ->disableMetric($metricTypeIdExecutionsCompute) - ->disableMetric($metricTypeExecutionsMbSeconds) - ->disableMetric($metricTypeIdExecutionsMBSeconds); - } - - $queueForStatsUsage - ->addMetric(METRIC_SITES_REQUESTS, 1) - ->addMetric(METRIC_SITES_INBOUND, $request->getSize() + $fileSize) - ->addMetric(METRIC_SITES_OUTBOUND, $response->getSize()) - ->addMetric(str_replace('{siteInternalId}', $resource->getSequence(), METRIC_SITES_ID_REQUESTS), 1) - ->addMetric(str_replace('{siteInternalId}', $resource->getSequence(), METRIC_SITES_ID_INBOUND), $request->getSize() + $fileSize) - ->addMetric(str_replace('{siteInternalId}', $resource->getSequence(), METRIC_SITES_ID_OUTBOUND), $response->getSize()) - ; - } - - - $compute = (int)($execution->getAttribute('duration') * 1000); - $mbSeconds = (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT)); - $queueForStatsUsage - ->addMetric(METRIC_NETWORK_REQUESTS, 1) - ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) - ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) - ->addMetric(METRIC_EXECUTIONS, 1) - ->addMetric($metricTypeExecutions, 1) - ->addMetric($metricTypeIdExecutions, 1) - ->addMetric(METRIC_EXECUTIONS_COMPUTE, $compute) // per project - ->addMetric($metricTypeExecutionsCompute, $compute) // per function - ->addMetric($metricTypeIdExecutionsCompute, $compute) // per function - ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, $mbSeconds) - ->addMetric($metricTypeExecutionsMbSeconds, $mbSeconds) - ->addMetric($metricTypeIdExecutionsMBSeconds, $mbSeconds) - ->setProject($project) - ->trigger(); + $bus->dispatch(new RequestCompleted( + project: $project->getArrayCopy(), + request: $request, + response: $response, + deployment: $deployment->getArrayCopy(), + )); /* cleanup */ if ($executionsRetentionCount > 0 && ENABLE_EXECUTIONS_LIMIT_ON_ROUTE) { @@ -881,9 +826,8 @@ Http::init() ->inject('locale') ->inject('localeCodes') ->inject('geodb') - ->inject('queueForStatsUsage') ->inject('queueForEvents') - ->inject('queueForExecutions') + ->inject('bus') ->inject('executor') ->inject('platform') ->inject('isResourceBlocked') @@ -894,7 +838,7 @@ Http::init() ->inject('authorization') ->inject('queueForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, StatsUsage $queueForStatsUsage, Event $queueForEvents, Execution $queueForExecutions, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { /* * Appwrite Router */ @@ -902,7 +846,7 @@ Http::init() $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!\in_array($hostname, $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForExecutions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) { $utopia->getRoute()?->label('router', true); } } @@ -1178,8 +1122,7 @@ Http::options() ->inject('dbForPlatform') ->inject('getProjectDB') ->inject('queueForEvents') - ->inject('queueForStatsUsage') - ->inject('queueForExecutions') + ->inject('bus') ->inject('executor') ->inject('geodb') ->inject('isResourceBlocked') @@ -1192,14 +1135,14 @@ Http::options() ->inject('authorization') ->inject('queueForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Execution $queueForExecutions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { /* * Appwrite Router */ $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForExecutions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) { $utopia->getRoute()?->label('router', true); } } @@ -1215,12 +1158,11 @@ Http::options() /** OPTIONS requests in utopia do not execute shutdown handlers, as a result we need to track the OPTIONS requests explicitly * @see https://github.com/utopia-php/http/blob/0.33.16/src/App.php#L825-L855 */ - $queueForStatsUsage - ->addMetric(METRIC_NETWORK_REQUESTS, 1) - ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize()) - ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()) - ->setProject($project) - ->trigger(); + $bus->dispatch(new RequestCompleted( + project: $project->getArrayCopy(), + request: $request, + response: $response, + )); }); Http::error() @@ -1231,10 +1173,10 @@ Http::error() ->inject('project') ->inject('logger') ->inject('log') - ->inject('queueForStatsUsage') + ->inject('bus') ->inject('devKey') ->inject('authorization') - ->action(function (Throwable $error, Http $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, StatsUsage $queueForStatsUsage, Document $devKey, Authorization $authorization) { + ->action(function (Throwable $error, Http $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Bus $bus, Document $devKey, Authorization $authorization) { $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); @@ -1307,21 +1249,12 @@ Http::error() */ if (!$publish && $project->getId() !== 'console') { if (!DBUser::isPrivileged($authorization->getRoles())) { - $fileSize = 0; - $file = $request->getFiles('file'); - if (!empty($file)) { - $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; - } - - $queueForStatsUsage - ->addMetric(METRIC_NETWORK_REQUESTS, 1) - ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) - ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); + $bus->dispatch(new RequestCompleted( + project: $project->getArrayCopy(), + request: $request, + response: $response, + )); } - - $queueForStatsUsage - ->setProject($project) - ->trigger(); } if ($logger && $publish) { @@ -1568,8 +1501,7 @@ Http::get('/robots.txt') ->inject('dbForPlatform') ->inject('getProjectDB') ->inject('queueForEvents') - ->inject('queueForStatsUsage') - ->inject('queueForExecutions') + ->inject('bus') ->inject('executor') ->inject('geodb') ->inject('isResourceBlocked') @@ -1579,13 +1511,13 @@ Http::get('/robots.txt') ->inject('authorization') ->inject('queueForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Execution $queueForExecutions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForExecutions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) { $utopia->getRoute()?->label('router', true); } } @@ -1603,8 +1535,7 @@ Http::get('/humans.txt') ->inject('dbForPlatform') ->inject('getProjectDB') ->inject('queueForEvents') - ->inject('queueForStatsUsage') - ->inject('queueForExecutions') + ->inject('bus') ->inject('executor') ->inject('geodb') ->inject('isResourceBlocked') @@ -1614,13 +1545,13 @@ Http::get('/humans.txt') ->inject('authorization') ->inject('queueForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Execution $queueForExecutions, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeleteEvent $queueForDeletes, int $executionsRetentionCount) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $queueForStatsUsage, $queueForExecutions, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $queueForDeletes, $executionsRetentionCount)) { $utopia->getRoute()?->label('router', true); } } diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 56f8ed30a1..641cf691ee 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -2,6 +2,7 @@ use Appwrite\Auth\Key; use Appwrite\Auth\MFA\Type\TOTP; +use Appwrite\Bus\Events\RequestCompleted; use Appwrite\Event\Audit; use Appwrite\Event\Build; use Appwrite\Event\Database as EventDatabase; @@ -21,6 +22,7 @@ use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Utopia\Abuse\Abuse; +use Utopia\Bus\Bus; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; use Utopia\Config\Config; @@ -752,7 +754,8 @@ Http::shutdown() ->inject('authorization') ->inject('timelimit') ->inject('eventProcessor') - ->action(function (Http $utopia, Request $request, Response $response, Document $project, User $user, Event $queueForEvents, Audit $queueForAudits, StatsUsage $queueForStatsUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, Authorization $authorization, callable $timelimit, EventProcessor $eventProcessor) use ($parseLabel) { + ->inject('bus') + ->action(function (Http $utopia, Request $request, Response $response, Document $project, User $user, Event $queueForEvents, Audit $queueForAudits, StatsUsage $queueForStatsUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, Authorization $authorization, callable $timelimit, EventProcessor $eventProcessor, Bus $bus) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -964,16 +967,11 @@ Http::shutdown() if ($project->getId() !== 'console') { if (!User::isPrivileged($authorization->getRoles())) { - $fileSize = 0; - $file = $request->getFiles('file'); - if (!empty($file)) { - $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; - } - - $queueForStatsUsage - ->addMetric(METRIC_NETWORK_REQUESTS, 1) - ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) - ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); + $bus->dispatch(new RequestCompleted( + project: $project->getArrayCopy(), + request: $request, + response: $response, + )); } $queueForStatsUsage diff --git a/app/http.php b/app/http.php index 9f5015d26c..77f9bb2881 100644 --- a/app/http.php +++ b/app/http.php @@ -188,6 +188,10 @@ $http->on(Constant::EVENT_AFTER_RELOAD, function ($server) { Console::success('Reload completed...'); }); +Http::setResource('bus', function ($register, $utopia) { + return $register->get('bus')->setResolver(fn (string $name) => $utopia->getResource($name)); +}, ['register', 'utopia']); + include __DIR__ . '/controllers/general.php'; function createDatabase(Http $app, string $resourceKey, string $dbName, array $collections, mixed $pools, ?callable $extraSetup = null): void diff --git a/app/init/registers.php b/app/init/registers.php index 413991540b..50ae954dd9 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -480,3 +480,11 @@ $register->set('promiseAdapter', function () { $register->set('hooks', function () { return new Hooks(); }); +$listeners = require __DIR__ . '/../listeners.php'; +$register->set('bus', function () use ($listeners) { + $bus = new \Utopia\Bus\Bus(); + foreach ($listeners as $listener) { + $bus->subscribe($listener); + } + return $bus; +}); diff --git a/app/init/resources.php b/app/init/resources.php index ac8a90dbfe..807f86ced6 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -10,7 +10,6 @@ use Appwrite\Event\Certificate; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Event; -use Appwrite\Event\Execution; use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; @@ -162,9 +161,6 @@ Http::setResource('queueForAudits', function (Publisher $publisher) { Http::setResource('queueForFunctions', function (Publisher $publisher) { return new Func($publisher); }, ['publisher']); -Http::setResource('queueForExecutions', function (Publisher $publisher) { - return new Execution($publisher); -}, ['publisher']); Http::setResource('eventProcessor', function () { return new EventProcessor(); }, []); diff --git a/app/listeners.php b/app/listeners.php new file mode 100644 index 0000000000..714c255974 --- /dev/null +++ b/app/listeners.php @@ -0,0 +1,9 @@ +getWorker(); +Server::setResource('bus', function ($register) use ($worker) { + return $register->get('bus')->setResolver(fn (string $name) => $worker->getResource($name)); +}, ['register']); + $worker ->error() ->inject('error') diff --git a/composer.json b/composer.json index f016b505fa..4b5e1eeec6 100644 --- a/composer.json +++ b/composer.json @@ -19,7 +19,8 @@ "autoload": { "psr-4": { "Appwrite\\": "src/Appwrite", - "Executor\\": "src/Executor" + "Executor\\": "src/Executor", + "Utopia\\Bus\\": "src/Utopia/Bus" } }, "autoload-dev": { diff --git a/phpstan.neon b/phpstan.neon index 153b3be21c..90f28e7539 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -1,7 +1,11 @@ parameters: level: 8 paths: + - src/Utopia/Bus + - src/Appwrite/Bus - src/Appwrite/Transformation + bootstrapFiles: + - app/init/constants.php scanDirectories: - vendor/swoole/ide-helper excludePaths: diff --git a/src/Appwrite/Bus/Events/ExecutionCompleted.php b/src/Appwrite/Bus/Events/ExecutionCompleted.php new file mode 100644 index 0000000000..58c84c82f0 --- /dev/null +++ b/src/Appwrite/Bus/Events/ExecutionCompleted.php @@ -0,0 +1,22 @@ + $execution + * @param array $project + * @param array $spec + * @param array $resource + */ + public function __construct( + public readonly array $execution, + public readonly array $project, + public readonly array $spec = [], + public readonly array $resource = [], + ) { + } +} diff --git a/src/Appwrite/Bus/Events/RequestCompleted.php b/src/Appwrite/Bus/Events/RequestCompleted.php new file mode 100644 index 0000000000..d6f9f1c90b --- /dev/null +++ b/src/Appwrite/Bus/Events/RequestCompleted.php @@ -0,0 +1,22 @@ + $project + * @param array $deployment + */ + public function __construct( + public readonly array $project, + public readonly Request $request, + public readonly Response $response, + public readonly array $deployment = [], + ) { + } +} diff --git a/src/Appwrite/Bus/Listeners/Log.php b/src/Appwrite/Bus/Listeners/Log.php new file mode 100644 index 0000000000..9bd539d5fe --- /dev/null +++ b/src/Appwrite/Bus/Listeners/Log.php @@ -0,0 +1,39 @@ +desc('Persists execution logs to database via queue') + ->inject('publisher') + ->callback($this->handle(...)); + } + + public function handle(ExecutionCompleted $event, Publisher $publisher): void + { + $queueForExecutions = new Execution($publisher); + $queueForExecutions + ->setExecution(new Document($event->execution)) + ->setProject(new Document($event->project)) + ->trigger(); + } +} diff --git a/src/Appwrite/Bus/Listeners/Usage.php b/src/Appwrite/Bus/Listeners/Usage.php new file mode 100644 index 0000000000..219287033d --- /dev/null +++ b/src/Appwrite/Bus/Listeners/Usage.php @@ -0,0 +1,114 @@ +desc('Records usage metrics') + ->inject('publisherStatsUsage') + ->callback($this->handle(...)); + } + + public function handle(Event $event, Publisher $publisher): void + { + match (true) { + $event instanceof ExecutionCompleted => $this->handleExecutionCompleted($event, $publisher), + $event instanceof RequestCompleted => $this->handleRequestCompleted($event, $publisher), + default => null, + }; + } + + private function handleExecutionCompleted(ExecutionCompleted $event, Publisher $publisher): void + { + $execution = new Document($event->execution); + $resource = new Document($event->resource); + + // Non-SSR sites don't record execution metrics + if ($execution->getAttribute('resourceType') === 'sites' && $resource->getAttribute('adapter') !== 'ssr') { + return; + } + $project = new Document($event->project); + $spec = $event->spec; + + $resourceType = $execution->getAttribute('resourceType', ''); + $resourceInternalId = $execution->getAttribute('resourceInternalId', ''); + $duration = $execution->getAttribute('duration', 0); + + $compute = (int)($duration * 1000); + $mbSeconds = (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $duration * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT)); + + $queueForStatsUsage = new StatsUsage($publisher); + $queueForStatsUsage + ->setProject($project) + ->addMetric(METRIC_EXECUTIONS, 1) + ->addMetric(str_replace(['{resourceType}'], [$resourceType], METRIC_RESOURCE_TYPE_EXECUTIONS), 1) + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS), 1) + ->addMetric(METRIC_EXECUTIONS_COMPUTE, $compute) + ->addMetric(str_replace(['{resourceType}'], [$resourceType], METRIC_RESOURCE_TYPE_EXECUTIONS_COMPUTE), $compute) + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_COMPUTE), $compute) + ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, $mbSeconds) + ->addMetric(str_replace(['{resourceType}'], [$resourceType], METRIC_RESOURCE_TYPE_EXECUTIONS_MB_SECONDS), $mbSeconds) + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_MB_SECONDS), $mbSeconds) + ->trigger(); + } + + private function handleRequestCompleted(RequestCompleted $event, Publisher $publisher): void + { + $fileSize = 0; + $file = $event->request->getFiles('file'); + if (!empty($file)) { + $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + } + + $project = new Document($event->project); + $deployment = new Document($event->deployment); + $queueForStatsUsage = new StatsUsage($publisher); + + $inbound = $event->request->getSize() + $fileSize; + $outbound = $event->response->getSize(); + + $queueForStatsUsage->setProject($project); + + if ($deployment->getAttribute('resourceType') === 'sites') { + $siteInternalId = $deployment->getAttribute('resourceInternalId', ''); + $queueForStatsUsage + ->addMetric(METRIC_SITES_REQUESTS, 1) + ->addMetric(METRIC_SITES_INBOUND, $inbound) + ->addMetric(METRIC_SITES_OUTBOUND, $outbound) + ->addMetric(str_replace('{siteInternalId}', $siteInternalId, METRIC_SITES_ID_REQUESTS), 1) + ->addMetric(str_replace('{siteInternalId}', $siteInternalId, METRIC_SITES_ID_INBOUND), $inbound) + ->addMetric(str_replace('{siteInternalId}', $siteInternalId, METRIC_SITES_ID_OUTBOUND), $outbound); + } else { + $queueForStatsUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $inbound) + ->addMetric(METRIC_NETWORK_OUTBOUND, $outbound); + } + + $queueForStatsUsage->trigger(); + } +} diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 06b6714206..3d76d58dd2 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -725,7 +725,9 @@ class Event $events = $pairedEvents; } // mirrored events can have duplicates in case of smaller events - return array_unique($events); + // array unique can turns list to hasmap in case duplicates present + // so forcing array value will turn this to array list always + return array_values(array_unique($events)); } /** diff --git a/src/Appwrite/Messaging/Adapter/Realtime.php b/src/Appwrite/Messaging/Adapter/Realtime.php index 1ca1bff53d..2d93ed48ce 100644 --- a/src/Appwrite/Messaging/Adapter/Realtime.php +++ b/src/Appwrite/Messaging/Adapter/Realtime.php @@ -512,23 +512,19 @@ class Realtime extends MessagingAdapter $collectionId = $payload->getAttribute('$collectionId', ''); $resourceId = $tableId ?: $collectionId; $channels = []; - // backward compat(tablesdb will have databases channels + tablesdb prefixed channels) - if ($parts[0] === 'databases' || $parts[0] === 'tablesdb') { - $prefix = 'databases'; - $channels = self::getDatabaseChannels('legacy', $database->getId(), $resourceId, $payload->getId(), $prefix); - - $channels = array_unique([ - ...$channels, - ...self::getDatabaseChannels('tablesdb', $database->getId(), $resourceId, $payload->getId(), $prefix) - ]); - } - // prefixed channels -> tablesdb, documentsdb,etc - if ($parts[0] !== 'databases') { - $channels = array_unique([ - ...$channels, - ...self::getDatabaseChannels($parts[0], $database->getId(), $resourceId, $payload->getId()), - ]); + switch($parts[0]){ + case 'databases': + case 'tablesdb': + // sending legacy + tablesdb events to both legacy and tablesdb + $channels = array_values(array_unique(array_merge( + self::getDatabaseChannels('legacy', $database->getId(), $resourceId, $payload->getId(), 'databases'), + self::getDatabaseChannels('tablesdb', $database->getId(), $resourceId, $payload->getId(), 'databases'), + self::getDatabaseChannels('tablesdb', $database->getId(), $resourceId, $payload->getId()) + ))); + default: + // only prefixed events + $channels = array_values(self::getDatabaseChannels($parts[0], $database->getId(), $resourceId, $payload->getId())); } $roles = $collection->getAttribute('documentSecurity', false) @@ -622,6 +618,7 @@ class Realtime extends MessagingAdapter $channels[] = "{$basePrefix}.{$databaseId}.collections.{$resourceId}.documents"; $channels[] = "{$basePrefix}.{$databaseId}.collections.{$resourceId}.documents.{$payloadId}"; break; + case 'tablesdb': $channels[] = 'rows'; $channels[] = "{$basePrefix}.{$databaseId}.tables.{$resourceId}.rows"; diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php new file mode 100644 index 0000000000..6b521e56d0 --- /dev/null +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php @@ -0,0 +1,141 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_PATCH) + ->setHttpPath('/v1/vcs/github/installations/:installationId/repositories/:repositoryId') + ->desc('Update external deployment (authorize)') + ->groups(['api', 'vcs']) + ->label('scope', 'vcs.write') + ->label('sdk', new Method( + namespace: 'vcs', + group: 'repositories', + name: 'updateExternalDeployments', + description: '/docs/references/vcs/update-external-deployments.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_NOCONTENT, + model: Response::MODEL_NONE, + ) + ] + )) + ->param('installationId', '', new Text(256), 'Installation Id') + ->param('repositoryId', '', new Text(256), 'VCS Repository Id') + ->param('providerPullRequestId', '', new Text(256), 'GitHub Pull Request Id') + ->inject('gitHub') + ->inject('response') + ->inject('project') + ->inject('dbForPlatform') + ->inject('authorization') + ->inject('getProjectDB') + ->inject('queueForBuilds') + ->inject('platform') + ->callback($this->action(...)); + } + + public function action( + string $installationId, + string $repositoryId, + string $providerPullRequestId, + GitHub $github, + Response $response, + Document $project, + Database $dbForPlatform, + Authorization $authorization, + callable $getProjectDB, + Build $queueForBuilds, + array $platform + ) { + $installation = $dbForPlatform->getDocument('installations', $installationId); + + if ($installation->isEmpty()) { + throw new Exception(Exception::INSTALLATION_NOT_FOUND); + } + + $repository = $authorization->skip(fn () => $dbForPlatform->findOne('repositories', [ + Query::equal('$id', [$repositoryId]), + Query::equal('projectInternalId', [$project->getSequence()]) + ])); + + if ($repository->isEmpty()) { + throw new Exception(Exception::REPOSITORY_NOT_FOUND); + } + + if (\in_array($providerPullRequestId, $repository->getAttribute('providerPullRequestIds', []))) { + throw new Exception(Exception::PROVIDER_CONTRIBUTION_CONFLICT); + } + + $providerPullRequestIds = \array_unique(\array_merge($repository->getAttribute('providerPullRequestIds', []), [$providerPullRequestId])); + $repository = $repository->setAttribute('providerPullRequestIds', $providerPullRequestIds); + + $repository = $authorization->skip(fn () => $dbForPlatform->updateDocument('repositories', $repository->getId(), $repository)); + + $privateKey = System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'); + $githubAppId = System::getEnv('_APP_VCS_GITHUB_APP_ID'); + $providerInstallationId = $installation->getAttribute('providerInstallationId'); + $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); + + $repositories = [$repository]; + $providerRepositoryId = $repository->getAttribute('providerRepositoryId'); + + try { + $providerRepositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($providerRepositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + + $owner = $github->getOwnerName($providerInstallationId); + $pullRequestResponse = $github->getPullRequest($owner, $providerRepositoryName, $providerPullRequestId); + + $providerRepositoryUrl = $pullRequestResponse['head']['repo']['html_url'] ?? ''; + $providerRepositoryOwner = $pullRequestResponse['head']['repo']['owner']['login'] ?? ''; + $providerBranch = \explode(':', $pullRequestResponse['head']['label'])[1] ?? ''; + $providerBranchUrl = "$providerRepositoryUrl/tree/$providerBranch"; + $providerCommitHash = $pullRequestResponse['head']['sha'] ?? ''; + + $commitDetails = $github->getCommit($providerRepositoryOwner, $providerRepositoryName, $providerCommitHash); + $providerCommitMessage = $commitDetails["commitMessage"] ?? ''; + $providerCommitUrl = $commitDetails["commitUrl"] ?? ''; + $providerCommitAuthor = $commitDetails["commitAuthor"] ?? ''; + $providerCommitAuthorUrl = $commitDetails["commitAuthorUrl"] ?? ''; + + $this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, true, $dbForPlatform, $authorization, $queueForBuilds, $getProjectDB, $platform); + + $response->noContent(); + } +} diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Callback/Get.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Callback/Get.php index 1212c06a72..cb45efbf39 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Callback/Get.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Callback/Get.php @@ -4,13 +4,12 @@ namespace Appwrite\Platform\Modules\VCS\Http\GitHub\Callback; use Appwrite\Auth\OAuth2\Github as OAuth2Github; use Appwrite\Extend\Exception; +use Appwrite\Platform\Permission as AppwritePermission; use Appwrite\Utopia\Response; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; @@ -21,6 +20,7 @@ use Utopia\VCS\Adapter\Git\GitHub; class Get extends Action { use HTTP; + use AppwritePermission; public static function getName() { @@ -132,13 +132,7 @@ class Get extends Action $installation = new Document([ '$id' => ID::unique(), - '$permissions' => [ - Permission::read(Role::team(ID::custom($teamId))), - Permission::update(Role::team(ID::custom($teamId), 'owner')), - Permission::update(Role::team(ID::custom($teamId), 'developer')), - Permission::delete(Role::team(ID::custom($teamId), 'owner')), - Permission::delete(Role::team(ID::custom($teamId), 'developer')), - ], + '$permissions' => $this->getPermissions($teamId, $projectId), 'providerInstallationId' => $providerInstallationId, 'projectId' => $projectId, 'projectInternalId' => $projectInternalId, diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php new file mode 100644 index 0000000000..6d493e2cdb --- /dev/null +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php @@ -0,0 +1,523 @@ +getId(); + $projectId = $repository->getAttribute('projectId'); + $resourceId = $repository->getAttribute('resourceId'); + $resourceType = $repository->getAttribute('resourceType'); + + $logBase = "vcs.github.event.repo.{$repositoryId}"; + Span::add("{$logBase}.projectId", $projectId); + Span::add("{$logBase}.resourceId", $resourceId); + Span::add("{$logBase}.resourceType", $resourceType); + + if ($resourceType !== "function" && $resourceType !== "site") { + continue; + } + + $project = $authorization->skip(fn () => $dbForPlatform->getDocument('projects', $projectId)); + + if ($project->isEmpty()) { + throw new Exception(Exception::PROJECT_NOT_FOUND, 'Repository references non-existent project'); + } + + try { + $dsn = new DSN($project->getAttribute('database')); + $databaseName = $dsn->getHost(); + } catch (\InvalidArgumentException) { + $databaseName = $project->getAttribute('database'); + } + + $databases = Config::getParam('pools-database', []); + $index = in_array($databaseName, $databases); + + if ($index === false) { + Console::error("Database: '{$databaseName}' is not part of region: " . System::getEnv('_APP_REGION')); + continue; + } + + $dbForProject = $getProjectDB($project); + $resourceCollection = $resourceType === "function" ? 'functions' : 'sites'; + $resource = $authorization->skip(fn () => $dbForProject->getDocument($resourceCollection, $resourceId)); + $resourceInternalId = $resource->getSequence(); + + $deploymentId = ID::unique(); + $repositoryId = $repository->getId(); + $repositoryInternalId = $repository->getSequence(); + $providerRepositoryId = $repository->getAttribute('providerRepositoryId'); + $installationId = $repository->getAttribute('installationId'); + $installationInternalId = $repository->getAttribute('installationInternalId'); + $productionBranch = $resource->getAttribute('providerBranch'); + $activate = false; + + if ($providerBranch == $productionBranch && $external === false) { + $activate = true; + } + + $owner = $github->getOwnerName($providerInstallationId) ?? ''; + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + + $isAuthorized = !$external; + + if (!$isAuthorized && !empty($providerPullRequestId)) { + if (\in_array($providerPullRequestId, $repository->getAttribute('providerPullRequestIds', []))) { + $isAuthorized = true; + } + } + + Span::add("{$logBase}.authorized", $isAuthorized); + + $commentStatus = 'waiting'; + $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; + $hostname = $platform['consoleHostname'] ?? ''; + + $authorizeUrl = $protocol . '://' . $hostname . "/console/git/authorize-contributor?projectId={$projectId}&installationId={$installationId}&repositoryId={$repositoryId}&providerPullRequestId={$providerPullRequestId}"; + + $action = $isAuthorized ? ['type' => 'logs'] : ['type' => 'authorize', 'url' => $authorizeUrl]; + + $latestCommentId = ''; + + if (!empty($providerPullRequestId) && $resource->getAttribute('providerSilentMode', false) === false) { + $latestComment = $authorization->skip(fn () => $dbForPlatform->findOne('vcsComments', [ + Query::equal('providerRepositoryId', [$providerRepositoryId]), + Query::equal('providerPullRequestId', [$providerPullRequestId]), + Query::orderDesc('$createdAt'), + ])); + + if (!$latestComment->isEmpty()) { + $latestCommentId = $latestComment->getAttribute('providerCommentId', ''); + + $retries = 0; + $lockAcquired = false; + + while ($retries < 9) { + $retries++; + + try { + $dbForPlatform->createDocument('vcsCommentLocks', new Document([ + '$id' => $latestCommentId + ])); + $lockAcquired = true; + break; + } catch (\Throwable $err) { + if ($retries >= 9) { + Console::warning("Error creating vcs comment lock for " . $latestCommentId . ": " . $err->getMessage()); + } + + \sleep(1); + } + } + + if ($lockAcquired) { + // Wrap in try/finally to ensure lock file gets deleted + try { + $comment = new Comment($platform); + $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); + $comment->addBuild($project, $resource, $resourceType, $commentStatus, $deploymentId, $action, ''); + + $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); + } finally { + $authorization->skip(fn () => $dbForPlatform->deleteDocument('vcsCommentLocks', $latestCommentId)); + } + } + } else { + $comment = new Comment($platform); + $comment->addBuild($project, $resource, $resourceType, $commentStatus, $deploymentId, $action, ''); + $latestCommentId = \strval($github->createComment($owner, $repositoryName, $providerPullRequestId, $comment->generateComment())); + + if (!empty($latestCommentId)) { + $teamId = $project->getAttribute('teamId', ''); + + $latestComment = $authorization->skip(fn () => $dbForPlatform->createDocument('vcsComments', new Document([ + '$id' => ID::unique(), + '$permissions' => [ + Permission::read(Role::team(ID::custom($teamId))), + Permission::update(Role::team(ID::custom($teamId), 'owner')), + Permission::update(Role::team(ID::custom($teamId), 'developer')), + Permission::delete(Role::team(ID::custom($teamId), 'owner')), + Permission::delete(Role::team(ID::custom($teamId), 'developer')), + ], + 'installationInternalId' => $installationInternalId, + 'installationId' => $installationId, + 'projectInternalId' => $project->getSequence(), + 'projectId' => $project->getId(), + 'providerRepositoryId' => $providerRepositoryId, + 'providerBranch' => $providerBranch, + 'providerPullRequestId' => $providerPullRequestId, + 'providerCommentId' => $latestCommentId + ]))); + } + } + } elseif (!empty($providerBranch)) { + $latestComments = $authorization->skip(fn () => $dbForPlatform->find('vcsComments', [ + Query::equal('providerRepositoryId', [$providerRepositoryId]), + Query::equal('providerBranch', [$providerBranch]), + Query::orderDesc('$createdAt'), + ])); + + foreach ($latestComments as $comment) { + $latestCommentId = $comment->getAttribute('providerCommentId', ''); + + $retries = 0; + $lockAcquired = false; + + while ($retries < 9) { + $retries++; + + try { + $dbForPlatform->createDocument('vcsCommentLocks', new Document([ + '$id' => $latestCommentId + ])); + $lockAcquired = true; + break; + } catch (\Throwable $err) { + if ($retries >= 9) { + Console::warning("Error creating vcs comment lock for " . $latestCommentId . ": " . $err->getMessage()); + } + + \sleep(1); + } + } + + if ($lockAcquired) { + // Wrap in try/finally to ensure lock file gets deleted + try { + $comment = new Comment($platform); + $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); + $comment->addBuild($project, $resource, $resourceType, $commentStatus, $deploymentId, $action, ''); + + $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); + } finally { + $authorization->skip(fn () => $dbForPlatform->deleteDocument('vcsCommentLocks', $latestCommentId)); + } + } + } + } + + if (!$isAuthorized) { + $resourceName = $resource->getAttribute('name'); + $projectName = $project->getAttribute('name'); + $name = "{$resourceName} ({$projectName})"; + $message = 'Authorization required for external contributor.'; + + $providerRepositoryId = $repository->getAttribute('providerRepositoryId'); + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + $owner = $github->getOwnerName($providerInstallationId); + $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $authorizeUrl, $name); + continue; + } + + $commands = []; + if (!empty($resource->getAttribute('installCommand', ''))) { + $commands[] = $resource->getAttribute('installCommand', ''); + } + if (!empty($resource->getAttribute('buildCommand', ''))) { + $commands[] = $resource->getAttribute('buildCommand', ''); + } + if (!empty($resource->getAttribute('commands', ''))) { + $commands[] = $resource->getAttribute('commands', ''); + } + + $deployment = $authorization->skip(fn () => $dbForProject->createDocument('deployments', new Document([ + '$id' => $deploymentId, + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'resourceId' => $resourceId, + 'resourceInternalId' => $resourceInternalId, + 'resourceType' => $resourceCollection, + 'entrypoint' => $resource->getAttribute('entrypoint', ''), + 'buildCommands' => \implode(' && ', $commands), + 'startCommand' => $resource->getAttribute('startCommand', ''), + 'buildOutput' => $resource->getAttribute('outputDirectory', ''), + 'adapter' => $resource->getAttribute('adapter', ''), + 'fallbackFile' => $resource->getAttribute('fallbackFile', ''), + 'type' => 'vcs', + 'installationId' => $installationId, + 'installationInternalId' => $installationInternalId, + 'providerRepositoryId' => $providerRepositoryId, + 'repositoryId' => $repositoryId, + 'repositoryInternalId' => $repositoryInternalId, + 'providerBranchUrl' => $providerBranchUrl, + 'providerRepositoryName' => $providerRepositoryName, + 'providerRepositoryOwner' => $providerRepositoryOwner, + 'providerRepositoryUrl' => $providerRepositoryUrl, + 'providerCommitHash' => $providerCommitHash, + 'providerCommitAuthorUrl' => $providerCommitAuthorUrl, + 'providerCommitAuthor' => $providerCommitAuthor, + 'providerCommitMessage' => mb_strimwidth($providerCommitMessage, 0, 255, '...'), + 'providerCommitUrl' => $providerCommitUrl, + 'providerCommentId' => \strval($latestCommentId), + 'providerBranch' => $providerBranch, + 'activate' => $activate, + ]))); + + $resource = $resource + ->setAttribute('latestDeploymentId', $deployment->getId()) + ->setAttribute('latestDeploymentInternalId', $deployment->getSequence()) + ->setAttribute('latestDeploymentCreatedAt', $deployment->getCreatedAt()) + ->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', '')); + $authorization->skip(fn () => $dbForProject->updateDocument($resource->getCollection(), $resource->getId(), $resource)); + + if ($resource->getCollection() === 'sites') { + $projectId = $project->getId(); + + // Deployment preview + $sitesDomain = $platform['sitesDomain']; + $domain = ID::unique() . "." . $sitesDomain; + $ruleId = md5($domain); + $previewRuleId = $ruleId; + $authorization->skip( + fn () => $dbForPlatform->createDocument('rules', new Document([ + '$id' => $ruleId, + 'projectId' => $project->getId(), + 'projectInternalId' => $project->getSequence(), + 'domain' => $domain, + 'type' => 'deployment', + 'trigger' => 'deployment', + 'deploymentId' => $deployment->getId(), + 'deploymentInternalId' => $deployment->getSequence(), + 'deploymentResourceType' => 'site', + 'deploymentResourceId' => $resourceId, + 'deploymentResourceInternalId' => $resourceInternalId, + 'deploymentVcsProviderBranch' => $providerBranch, + 'status' => 'verified', + 'certificateId' => '', + 'search' => implode(' ', [$ruleId, $domain]), + 'owner' => 'Appwrite', + 'region' => $project->getAttribute('region') + ])) + ); + + // VCS branch preview + if (!empty($providerBranch)) { + $domain = (new BranchDomainFilter())->apply([ + 'branch' => $providerBranch, + 'resourceId' => $resource->getId(), + 'projectId' => $project->getId(), + 'sitesDomain' => $sitesDomain, + ]); + $ruleId = md5($domain); + try { + $authorization->skip( + fn () => $dbForPlatform->createDocument('rules', new Document([ + '$id' => $ruleId, + 'projectId' => $project->getId(), + 'projectInternalId' => $project->getSequence(), + 'domain' => $domain, + 'type' => 'deployment', + 'trigger' => 'deployment', + 'deploymentId' => $deployment->getId(), + 'deploymentInternalId' => $deployment->getSequence(), + 'deploymentResourceType' => 'site', + 'deploymentResourceId' => $resourceId, + 'deploymentResourceInternalId' => $resourceInternalId, + 'deploymentVcsProviderBranch' => $providerBranch, + 'status' => 'verified', + 'certificateId' => '', + 'search' => implode(' ', [$ruleId, $domain]), + 'owner' => 'Appwrite', + 'region' => $project->getAttribute('region') + ])) + ); + } catch (Duplicate $err) { + // Ignore, rule already exists; will be updated by builds worker + } + } + + // VCS commit preview + if (!empty($providerCommitHash)) { + $domain = "commit-" . substr($providerCommitHash, 0, 16) . ".{$sitesDomain}"; + $ruleId = md5($domain); + try { + $authorization->skip( + fn () => $dbForPlatform->createDocument('rules', new Document([ + '$id' => $ruleId, + 'projectId' => $project->getId(), + 'projectInternalId' => $project->getSequence(), + 'domain' => $domain, + 'type' => 'deployment', + 'trigger' => 'deployment', + 'deploymentId' => $deployment->getId(), + 'deploymentInternalId' => $deployment->getSequence(), + 'deploymentResourceType' => 'site', + 'deploymentResourceId' => $resourceId, + 'deploymentResourceInternalId' => $resourceInternalId, + 'deploymentVcsProviderBranch' => $providerBranch, + 'status' => 'verified', + 'certificateId' => '', + 'search' => implode(' ', [$ruleId, $domain]), + 'owner' => 'Appwrite', + 'region' => $project->getAttribute('region') + ])) + ); + } catch (Duplicate $err) { + // Ignore, rule already exists; will be updated by builds worker + } + } + } + + if ($resource->getCollection() === 'sites' && !empty($latestCommentId) && !empty($previewRuleId)) { + $retries = 0; + $lockAcquired = false; + + while ($retries < 9) { + $retries++; + + try { + $dbForPlatform->createDocument('vcsCommentLocks', new Document([ + '$id' => $latestCommentId + ])); + $lockAcquired = true; + break; + } catch (\Throwable $err) { + if ($retries >= 9) { + Console::warning("Error creating vcs comment lock for " . $latestCommentId . ": " . $err->getMessage()); + } + + \sleep(1); + } + } + + if ($lockAcquired) { + // Wrap in try/finally to ensure lock file gets deleted + try { + $rule = $authorization->skip(fn () => $dbForPlatform->getDocument('rules', $previewRuleId)); + + $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; + $previewUrl = !empty($rule) ? ("{$protocol}://" . $rule->getAttribute('domain', '')) : ''; + + if (!empty($previewUrl)) { + $comment = new Comment($platform); + $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); + $comment->addBuild($project, $resource, $resourceType, $commentStatus, $deploymentId, $action, $previewUrl); + $github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment()); + } + } finally { + $authorization->skip(fn () => $dbForPlatform->deleteDocument('vcsCommentLocks', $latestCommentId)); + } + } + } + + if (!empty($providerCommitHash) && $resource->getAttribute('providerSilentMode', false) === false) { + $resourceName = $resource->getAttribute('name'); + $projectName = $project->getAttribute('name'); + $region = $project->getAttribute('region', 'default'); + $name = "{$resourceName} ({$projectName})"; + $message = 'Starting...'; + + $providerRepositoryId = $repository->getAttribute('providerRepositoryId'); + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + $owner = $github->getOwnerName($providerInstallationId); + + $providerTargetUrl = $protocol . '://' . $hostname . "/console/project-$region-$projectId/$resourceCollection/$resourceType-$resourceId"; + $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name); + } + + $queueName = $this->getBuildQueueName($project, $dbForPlatform, $authorization); + + $queueForBuilds + ->setQueue($queueName) + ->setType(BUILD_TYPE_DEPLOYMENT) + ->setResource($resource) + ->setDeployment($deployment) + ->setProject($project); // set the project because it won't be set for git deployments + + $queueForBuilds->trigger(); // must trigger here so that we create a build for each function/site + + Span::add("{$logBase}.build.triggered", 'true'); + //TODO: Add event? + } catch (\Throwable $e) { + Span::add("{$logBase}.error", $e->getMessage()); + $errors[] = $e->getMessage(); + } + } + + $queueForBuilds->reset(); // prevent shutdown hook from triggering again + + if (!empty($errors)) { + throw new Exception(Exception::GENERAL_UNKNOWN, \implode("\n", $errors)); + } + } + + protected function getBuildQueueName(Document $project, Database $dbForPlatform, Authorization $authorization): string + { + return System::getEnv('_APP_BUILDS_QUEUE_NAME', Event::BUILDS_QUEUE_NAME); + } +} diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php new file mode 100644 index 0000000000..de6750fe64 --- /dev/null +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php @@ -0,0 +1,244 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vcs/github/events') + ->desc('Create event') + ->groups(['api', 'vcs']) + ->label('scope', 'public') + ->inject('gitHub') + ->inject('request') + ->inject('response') + ->inject('dbForPlatform') + ->inject('authorization') + ->inject('getProjectDB') + ->inject('queueForBuilds') + ->inject('platform') + ->callback($this->action(...)); + } + + public function action( + GitHub $github, + Request $request, + Response $response, + Database $dbForPlatform, + Authorization $authorization, + callable $getProjectDB, + Build $queueForBuilds, + array $platform + ) { + $this->preprocessEvent($request); + + $event = $request->getHeader('x-github-event', ''); + Span::add('vcs.github.event.name', $event); + + $payload = $request->getRawPayload(); + $signature = $request->getHeader('x-hub-signature-256', ''); + $secretKey = System::getEnv('_APP_VCS_GITHUB_WEBHOOK_SECRET', ''); + + $valid = empty($signature) ? true : $github->validateWebhookEvent($payload, $signature, $secretKey); + Span::add('vcs.github.event.signature.valid', $valid); + + if (!$valid) { + throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, "Invalid webhook payload signature. Please make sure the webhook secret has same value in your GitHub app and in the _APP_VCS_GITHUB_WEBHOOK_SECRET environment variable"); + } + + $githubAppId = System::getEnv('_APP_VCS_GITHUB_APP_ID'); + $privateKey = System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'); + $parsedPayload = $github->getEvent($event, $payload); + + match ($event) { + $github::EVENT_INSTALLATION => $this->handleInstallationEvent($parsedPayload, $dbForPlatform, $authorization), + $github::EVENT_PUSH => $this->handlePushEvent($parsedPayload, $githubAppId, $privateKey, $github, $dbForPlatform, $authorization, $queueForBuilds, $getProjectDB, $platform), + $github::EVENT_PULL_REQUEST => $this->handlePullRequestEvent($parsedPayload, $privateKey, $githubAppId, $github, $dbForPlatform, $authorization, $queueForBuilds, $getProjectDB, $platform), + default => null, + }; + + return $response->json($parsedPayload); + } + + protected function preprocessEvent(Request $request) + { + return; + } + + private function handleInstallationEvent( + array $parsedPayload, + Database $dbForPlatform, + Authorization $authorization, + ) { + if ($parsedPayload["action"] !== "deleted") { + return; + } + + // TODO: Use worker for this job instead (update function/site as well) + $providerInstallationId = $parsedPayload["installationId"]; + + $installations = $dbForPlatform->find('installations', [ + Query::equal('providerInstallationId', [$providerInstallationId]), + Query::limit(1000) + ]); + + foreach ($installations as $installation) { + $repositories = $authorization->skip(fn () => $dbForPlatform->find('repositories', [ + Query::equal('installationInternalId', [$installation->getSequence()]), + Query::limit(1000) + ])); + + foreach ($repositories as $repository) { + $authorization->skip(fn () => $dbForPlatform->deleteDocument('repositories', $repository->getId())); + } + + $authorization->skip(fn () => $dbForPlatform->deleteDocument('installations', $installation->getId())); + } + } + + private function handlePushEvent( + array $parsedPayload, + string $githubAppId, + string $privateKey, + GitHub $github, + Database $dbForPlatform, + Authorization $authorization, + Build $queueForBuilds, + callable $getProjectDB, + array $platform, + ) { + $providerBranchCreated = $parsedPayload["branchCreated"] ?? false; + $providerBranchDeleted = $parsedPayload["branchDeleted"] ?? false; + $providerBranch = $parsedPayload["branch"] ?? ''; + $providerBranchUrl = $parsedPayload["branchUrl"] ?? ''; + $providerRepositoryId = $parsedPayload["repositoryId"] ?? ''; + $providerRepositoryName = $parsedPayload["repositoryName"] ?? ''; + $providerInstallationId = $parsedPayload["installationId"] ?? ''; + $providerRepositoryUrl = $parsedPayload["repositoryUrl"] ?? ''; + $providerCommitHash = $parsedPayload["commitHash"] ?? ''; + $providerRepositoryOwner = $parsedPayload["owner"] ?? ''; + $providerCommitAuthorName = $parsedPayload["headCommitAuthorName"] ?? ''; + $providerCommitAuthorEmail = $parsedPayload["headCommitAuthorEmail"] ?? ''; + $providerCommitAuthorUrl = $parsedPayload["authorUrl"] ?? ''; + $providerCommitMessage = $parsedPayload["headCommitMessage"] ?? ''; + $providerCommitUrl = $parsedPayload["headCommitUrl"] ?? ''; + + Span::add('vcs.github.event.repo.id', $providerRepositoryId); + Span::add('vcs.github.event.repo.name', $providerRepositoryName); + Span::add('vcs.github.event.branch', $providerBranch); + Span::add('vcs.github.event.installation.id', $providerInstallationId); + + $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); + + // Find associated repositories + $repositories = $authorization->skip(fn () => $dbForPlatform->find('repositories', [ + Query::equal('providerRepositoryId', [$providerRepositoryId]), + Query::limit(100), + ])); + + // Create new deployment only on push (not committed by us) and not when branch is created or deleted + if ($providerCommitAuthorEmail !== APP_VCS_GITHUB_EMAIL && !$providerBranchCreated && !$providerBranchDeleted) { + $this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', false, $dbForPlatform, $authorization, $queueForBuilds, $getProjectDB, $platform); + } + } + + private function handlePullRequestEvent( + array $parsedPayload, + string $privateKey, + string $githubAppId, + GitHub $github, + Database $dbForPlatform, + Authorization $authorization, + Build $queueForBuilds, + callable $getProjectDB, + array $platform, + ) { + $action = $parsedPayload["action"] ?? ''; + + if ($action == "opened" || $action == "reopened" || $action == "synchronize") { + $providerBranch = $parsedPayload["branch"] ?? ''; + $providerBranchUrl = $parsedPayload["branchUrl"] ?? ''; + $providerRepositoryId = $parsedPayload["repositoryId"] ?? ''; + $providerRepositoryName = $parsedPayload["repositoryName"] ?? ''; + $providerInstallationId = $parsedPayload["installationId"] ?? ''; + $providerRepositoryUrl = $parsedPayload["repositoryUrl"] ?? ''; + $providerPullRequestId = $parsedPayload["pullRequestNumber"] ?? ''; + $providerCommitHash = $parsedPayload["commitHash"] ?? ''; + $providerRepositoryOwner = $parsedPayload["owner"] ?? ''; + $external = $parsedPayload["external"] ?? true; + $providerCommitUrl = $parsedPayload["headCommitUrl"] ?? ''; + $providerCommitAuthorUrl = $parsedPayload["authorUrl"] ?? ''; + + Span::add('vcs.github.event.repo.id', $providerRepositoryId); + Span::add('vcs.github.event.repo.name', $providerRepositoryName); + Span::add('vcs.github.event.branch', $providerBranch); + Span::add('vcs.github.event.installation.id', $providerInstallationId); + + // Ignore sync for non-external. We handle it in push webhook + if (!$external && $parsedPayload["action"] == "synchronize") { + return; + } + + $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); + + $commitDetails = $github->getCommit($providerRepositoryOwner, $providerRepositoryName, $providerCommitHash); + $providerCommitAuthor = $commitDetails["commitAuthor"] ?? ''; + $providerCommitMessage = $commitDetails["commitMessage"] ?? ''; + + $repositories = $authorization->skip(fn () => $dbForPlatform->find('repositories', [ + Query::equal('providerRepositoryId', [$providerRepositoryId]), + Query::orderDesc('$createdAt') + ])); + + $this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $external, $dbForPlatform, $authorization, $queueForBuilds, $getProjectDB, $platform); + } elseif ($action == "closed") { + // Allowed external contributions cleanup + + $providerRepositoryId = $parsedPayload["repositoryId"] ?? ''; + $providerPullRequestId = $parsedPayload["pullRequestNumber"] ?? ''; + $external = $parsedPayload["external"] ?? true; + + if ($external) { + $repositories = $authorization->skip(fn () => $dbForPlatform->find('repositories', [ + Query::equal('providerRepositoryId', [$providerRepositoryId]), + Query::orderDesc('$createdAt') + ])); + + foreach ($repositories as $repository) { + $providerPullRequestIds = $repository->getAttribute('providerPullRequestIds', []); + + if (\in_array($providerPullRequestId, $providerPullRequestIds)) { + $providerPullRequestIds = \array_diff($providerPullRequestIds, [$providerPullRequestId]); + $repository = $repository->setAttribute('providerPullRequestIds', $providerPullRequestIds); + $repository = $authorization->skip(fn () => $dbForPlatform->updateDocument('repositories', $repository->getId(), $repository)); + } + } + } + } + } +} diff --git a/src/Appwrite/Platform/Modules/VCS/Http/Installations/Repositories/Create.php b/src/Appwrite/Platform/Modules/VCS/Http/Installations/Repositories/Create.php index e357d1d5fa..bba062a730 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/Installations/Repositories/Create.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/Installations/Repositories/Create.php @@ -152,6 +152,8 @@ class Create extends Action $repository['pushedAt'] = $repository['pushed_at'] ?? ''; $repository['organization'] = $installation->getAttribute('organization', ''); $repository['provider'] = $installation->getAttribute('provider', ''); + $repository['providerInstallationId'] = $installation->getAttribute('providerInstallationId', ''); + $repository['authorized'] = true; $response->dynamic(new Document($repository), Response::MODEL_PROVIDER_REPOSITORY); } diff --git a/src/Appwrite/Platform/Modules/VCS/Http/Installations/Repositories/Get.php b/src/Appwrite/Platform/Modules/VCS/Http/Installations/Repositories/Get.php index 4dee84752b..9e32ca8276 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/Installations/Repositories/Get.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/Installations/Repositories/Get.php @@ -85,11 +85,23 @@ class Get extends Action $repository = $github->getRepository($owner, $repositoryName); + $authorized = false; + try { + $installationRepository = $github->getInstallationRepository($repositoryName); + if (!empty($installationRepository)) { + $authorized = true; + } + } catch (RepositoryNotFound $e) { + $authorized = false; + } + $repository['id'] = \strval($repository['id']) ?? ''; $repository['pushedAt'] = $repository['pushed_at'] ?? ''; $repository['organization'] = $installation->getAttribute('organization', ''); $repository['provider'] = $installation->getAttribute('provider', ''); $repository['defaultBranch'] = $repository['default_branch'] ?? ''; + $repository['authorized'] = $authorized; + $repository['providerInstallationId'] = $providerInstallationId; $response->dynamic(new Document($repository), Response::MODEL_PROVIDER_REPOSITORY); } diff --git a/src/Appwrite/Platform/Modules/VCS/Http/Installations/Repositories/XList.php b/src/Appwrite/Platform/Modules/VCS/Http/Installations/Repositories/XList.php index 2cb1a996a4..53713f8407 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/Installations/Repositories/XList.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/Installations/Repositories/XList.php @@ -148,6 +148,8 @@ class XList extends Action $repo['pushedAt'] = $repo['pushed_at'] ?? null; $repo['provider'] = $installation->getAttribute('provider', '') ?? ''; $repo['organization'] = $installation->getAttribute('organization', '') ?? ''; + $repo['providerInstallationId'] = $installation->getAttribute('providerInstallationId', ''); + $repo['authorized'] = true; return $repo; }, $repos); diff --git a/src/Appwrite/Platform/Modules/VCS/Services/Http.php b/src/Appwrite/Platform/Modules/VCS/Services/Http.php index 8bd2314f9e..ae766ad00e 100644 --- a/src/Appwrite/Platform/Modules/VCS/Services/Http.php +++ b/src/Appwrite/Platform/Modules/VCS/Services/Http.php @@ -2,8 +2,10 @@ namespace Appwrite\Platform\Modules\VCS\Services; +use Appwrite\Platform\Modules\VCS\Http\GitHub\Authorize\External\Update as UpdateExternalDeployment; use Appwrite\Platform\Modules\VCS\Http\GitHub\Authorize\Get as GetGitHubAuthorize; use Appwrite\Platform\Modules\VCS\Http\GitHub\Callback\Get as GetGitHubCallback; +use Appwrite\Platform\Modules\VCS\Http\GitHub\Events\Create as CreateGitHubEvent; use Appwrite\Platform\Modules\VCS\Http\Installations\Delete as DeleteInstallation; use Appwrite\Platform\Modules\VCS\Http\Installations\Get as GetInstallation; use Appwrite\Platform\Modules\VCS\Http\Installations\Repositories\Branches\XList as ListRepositoryBranches; @@ -24,6 +26,7 @@ class Http extends Service // GitHub Authorization & Callback $this->addAction(GetGitHubAuthorize::getName(), new GetGitHubAuthorize()); $this->addAction(GetGitHubCallback::getName(), new GetGitHubCallback()); + $this->addAction(UpdateExternalDeployment::getName(), new UpdateExternalDeployment()); // Installations $this->addAction(GetInstallation::getName(), new GetInstallation()); @@ -37,5 +40,8 @@ class Http extends Service $this->addAction(ListRepositoryBranches::getName(), new ListRepositoryBranches()); $this->addAction(GetRepositoryContents::getName(), new GetRepositoryContents()); $this->addAction(CreateRepositoryDetections::getName(), new CreateRepositoryDetections()); + + // Events + $this->addAction(CreateGitHubEvent::getName(), new CreateGitHubEvent()); } } diff --git a/src/Appwrite/Platform/Tasks/SDKs.php b/src/Appwrite/Platform/Tasks/SDKs.php index a480f4cb4f..949dbb3e6b 100644 --- a/src/Appwrite/Platform/Tasks/SDKs.php +++ b/src/Appwrite/Platform/Tasks/SDKs.php @@ -486,9 +486,9 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND $useAi = ($ai !== 'no'); $apiKey = $useAi ? System::getEnv('_APP_ASSISTANT_OPENAI_API_KEY', '') : ''; $aiChangelog = ''; // Track AI-generated changelog for PR description - Console::info('Checking for _APP_ASSISTANT_OPENAI_API_KEY... [' . (! empty($apiKey) ? 'FOUND' : 'NOT FOUND') . ']'); + if (! empty($apiKey) && ! $examplesOnly) { - Console::info("Using AI to determine version bump and changelog for {$language['name']} SDK..."); + Console::info("Analyzing SDK changes with AI..."); $aiResult = $this->generateVersionAndChangelog($language, $result); if ($aiResult !== null) { @@ -502,6 +502,12 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // Update the changelog file $this->updateChangelogFile($language['changelog'], $newVersion, $newChangelog); + // Also update CHANGELOG.md in the generated SDK directory + $sdkChangelogPath = $result . '/CHANGELOG.md'; + if (file_exists($sdkChangelogPath)) { + $this->updateChangelogFile($sdkChangelogPath, $newVersion, $newChangelog); + } + // Reload the language config with updated values $language['version'] = $newVersion; @@ -512,10 +518,8 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND } catch (\Throwable $exception) { Console::error($exception->getMessage()); } - - Console::success("AI determined version: {$newVersion} ({$aiResult['versionBump']} bump)"); } else { - Console::warning('AI version generation failed, using existing version'); + Console::warning('AI analysis failed, using existing version'); } } @@ -524,33 +528,45 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND $repoBranch = $language['repoBranch'] ?? 'main'; if ($git && ! empty($gitUrl)) { + Console::info("Preparing {$language['name']} SDK repository..."); + \exec('rm -rf ' . $target . ' && \ mkdir -p ' . $target . ' && \ cd ' . $target . ' && \ - git init && \ + git init --quiet && \ git config core.ignorecase false && \ git config pull.rebase false && \ + git config advice.defaultBranchName false && \ git remote add origin ' . $gitUrl . ' && \ - git fetch origin && \ + git fetch origin --quiet --no-tags --depth 1 ' . $repoBranch . ' 2>&1 | grep -v "^remote:" | grep -v "^From " | grep -v "^ \* " || true && \ (git checkout -f ' . $repoBranch . ' 2>/dev/null || git checkout -b ' . $repoBranch . ') && \ - git pull origin ' . $repoBranch . ' && \ + git pull origin ' . $repoBranch . ' --quiet --no-tags 2>&1 | grep -v "^From " | grep -v "^ \* " || true && \ (git checkout -f ' . $gitBranch . ' 2>/dev/null || git checkout -b ' . $gitBranch . ') && \ - (git fetch origin ' . $gitBranch . ' 2>/dev/null || git push -u origin ' . $gitBranch . ') && \ + (git fetch origin ' . $gitBranch . ' --quiet --no-tags --depth 1 2>/dev/null || git push -u origin ' . $gitBranch . ' --quiet 2>&1 | grep -v "^remote:" || true) && \ git reset --hard origin/' . $gitBranch . ' 2>/dev/null || true && \ - (test -d .github && cp -r .github /tmp/.github-backup-$$ || true) && \ - git rm -rf --cached . && \ - git clean -fdx -e .git -e .github && \ + (if [ -d .github ]; then cp -r .github /tmp/.github-backup-$$ 2>/dev/null; fi) && \ + git rm -rf --cached . 2>/dev/null && \ + git clean -fdx -e .git -e .github 2>/dev/null && \ cp -r ' . $result . '/. ' . $target . '/ && \ - (test -d /tmp/.github-backup-$$ && cp -rn /tmp/.github-backup-$$/.github . && rm -rf /tmp/.github-backup-$$ || true) && \ + (if [ -d /tmp/.github-backup-$$/.github ]; then cp -rn /tmp/.github-backup-$$/.github . 2>/dev/null && rm -rf /tmp/.github-backup-$$; fi) && \ git add -A && \ - git commit -m "' . $message . '" && \ - git push -u origin ' . $gitBranch . ' - '); + git commit -m "' . $message . '" --quiet && \ + git push -u origin ' . $gitBranch . ' --quiet 2>&1 | grep -E "^(To | |[0-9a-f]+\\.\\.[0-9a-f]+)" || true + ', $gitOutput, $gitReturnCode); + + if ($gitReturnCode !== 0) { + Console::warning("Git operations completed with warnings (exit code: {$gitReturnCode})"); + } Console::success("Pushed {$language['name']} SDK to {$gitUrl}"); if ($git) { $prTitle = "feat: {$language['name']} SDK update for version {$language['version']}"; - $prBody = "This PR contains updates to the {$language['name']} SDK for version {$language['version']} . "; + + // Build PR body with AI changelog if available + $prBody = "This PR contains updates to the {$language['name']} SDK for version {$language['version']}."; + if (!empty($aiChangelog) && $aiChangelog !== '* No user-facing SDK changes.') { + $prBody .= "\n\n## Changes\n\n{$aiChangelog}"; + } $repoName = $language['gitUserName'] . '/' . $language['gitRepoName']; Console::info("Creating pull request for {$language['name']} SDK..."); @@ -765,36 +781,47 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ); $prompt = <<setSchema($schema) @@ -805,11 +832,13 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ->setExcludePaths([ '.github/workflows/**', '.github/ISSUE_TEMPLATE/**', + '.git/**', ]) ->setMaxDiffLines(500) ->setUserId('sdk-analyst'); Console::info("Running DiffCheck for {$language['name']} SDK..."); + $result = (new DiffCheck())->run( runner: $adapter, base: DiffCheckRepository::remote($gitUrl, $repoBranch), @@ -819,7 +848,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ); if (!$result['hasChanges']) { - Console::warning("No changes detected for {$language['name']} SDK"); + Console::info("✓ No changes detected - SDK is up to date"); return null; } @@ -831,15 +860,11 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND return null; } - Console::log('AI raw response:'); - Console::log($responseContent); - Console::log('--- End of AI response ---'); - $parsed = json_decode($responseContent, true); if (json_last_error() !== JSON_ERROR_NONE) { Console::warning('Failed to parse AI response as JSON: ' . json_last_error_msg()); - Console::log('Raw response that failed to parse:'); + Console::log('Raw response:'); Console::log($responseContent); return null; @@ -850,7 +875,14 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND return null; } - Console::info("AI analysis complete - Version bump: {$parsed['versionBump']}, New version: {$parsed['version']}"); + Console::success("✓ Analysis complete"); + Console::log(" Version: {$language['version']} → {$parsed['version']} ({$parsed['versionBump']} bump)"); + Console::log(" Changelog:"); + foreach (explode("\n", $parsed['changelog']) as $line) { + if (trim($line)) { + Console::log(" {$line}"); + } + } return [ 'version' => $parsed['version'], @@ -864,6 +896,16 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND } } + /** + * Get the SDK config file path + * + * @return string Path to the SDK config file + */ + protected function getSdkConfigPath(): string + { + return __DIR__ . '/../../../../app/config/sdks.php'; + } + /** * Update SDK version in the config file * @@ -874,7 +916,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND */ private function updateSdkVersion(string $platform, string $sdkKey, string $newVersion): bool { - $configPath = __DIR__ . '/../../../../app/config/sdks.php'; + $configPath = $this->getSdkConfigPath(); if (! file_exists($configPath)) { Console::error("Config file not found: {$configPath}"); @@ -884,13 +926,13 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND $content = file_get_contents($configPath); - // Find and replace the version for this specific SDK - // Pattern matches the version line in the SDK array - $pattern = '/(\[\s*[\'"]key[\'"]\s*=>\s*[\'"]' . preg_quote($sdkKey, '/') . '[\'"]\s*,[\s\S]*?[\'"]version[\'"]\s*=>\s*[\'"])([^\'"]+)([\'"])/m'; + // First, try to find inline version in SDK array (pattern 1) + // Pattern matches: ['key' => 'nodejs', ... 'version' => '22.1.2'] + $inlinePattern = '/(\[\s*[\'"]key[\'"]\s*=>\s*[\'"]' . preg_quote($sdkKey, '/') . '[\'"]\s*,[\s\S]*?[\'"]version[\'"]\s*=>\s*[\'"])([^\'"]+)([\'"])/m'; - if (preg_match($pattern, $content, $matches)) { + if (preg_match($inlinePattern, $content, $matches)) { $oldVersion = $matches[2]; - $newContent = preg_replace($pattern, '${1}' . $newVersion . '${3}', $content); + $newContent = preg_replace($inlinePattern, '${1}' . $newVersion . '${3}', $content); if (file_put_contents($configPath, $newContent) !== false) { Console::success("Updated {$sdkKey} version from {$oldVersion} to {$newVersion} in config"); @@ -901,11 +943,31 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND return false; } - } else { - Console::warning("Could not find version entry for {$sdkKey} in config"); - - return false; } + + // Second, try to find version in array format (pattern 2) + // Pattern matches: 'nodejs' => '22.1.2', or "nodejs" => "22.1.2", + // Also handles extra whitespace: 'nodejs' => '22.1.2', + $arrayPattern = '/([\'"]' . preg_quote($sdkKey, '/') . '[\'"]\s*=>\s*[\'"])([^\'"]+)([\'"],)/m'; + + if (preg_match($arrayPattern, $content, $matches)) { + $oldVersion = $matches[2]; + $newContent = preg_replace($arrayPattern, '${1}' . $newVersion . '${3}', $content); + + if (file_put_contents($configPath, $newContent) !== false) { + Console::success("Updated {$sdkKey} version from {$oldVersion} to {$newVersion} in config"); + + return true; + } else { + Console::error('Failed to write config file'); + + return false; + } + } + + Console::warning("Could not find version entry for {$sdkKey} in config"); + + return false; } /** diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index 0932aea335..18ab087966 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -3,15 +3,15 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; +use Appwrite\Bus\Events\ExecutionCompleted; use Appwrite\Event\Event; -use Appwrite\Event\Execution as ExecutionEvent; use Appwrite\Event\Func; use Appwrite\Event\Realtime; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Utopia\Response\Model\Execution; use Executor\Executor; +use Utopia\Bus\Bus; use Utopia\Config\Config; use Utopia\Console; use Utopia\Database\Database; @@ -47,8 +47,7 @@ class Functions extends Action ->inject('queueForFunctions') ->inject('queueForRealtime') ->inject('queueForEvents') - ->inject('queueForStatsUsage') - ->inject('queueForExecutions') + ->inject('bus') ->inject('log') ->inject('executor') ->inject('isResourceBlocked') @@ -63,8 +62,7 @@ class Functions extends Action Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, - StatsUsage $queueForStatsUsage, - ExecutionEvent $queueForExecutions, + Bus $bus, Log $log, Executor $executor, callable $isResourceBlocked @@ -158,9 +156,8 @@ class Functions extends Action queueForWebhooks: $queueForWebhooks, queueForFunctions: $queueForFunctions, queueForRealtime: $queueForRealtime, - queueForStatsUsage: $queueForStatsUsage, queueForEvents: $queueForEvents, - queueForExecutions: $queueForExecutions, + bus: $bus, project: $project, function: $function, executor: $executor, @@ -203,9 +200,8 @@ class Functions extends Action queueForWebhooks: $queueForWebhooks, queueForFunctions: $queueForFunctions, queueForRealtime: $queueForRealtime, - queueForStatsUsage: $queueForStatsUsage, queueForEvents: $queueForEvents, - queueForExecutions: $queueForExecutions, + bus: $bus, project: $project, function: $function, executor: $executor, @@ -230,9 +226,8 @@ class Functions extends Action queueForWebhooks: $queueForWebhooks, queueForFunctions: $queueForFunctions, queueForRealtime: $queueForRealtime, - queueForStatsUsage: $queueForStatsUsage, queueForEvents: $queueForEvents, - queueForExecutions: $queueForExecutions, + bus: $bus, project: $project, function: $function, executor: $executor, @@ -266,7 +261,7 @@ class Functions extends Action private function fail( string $message, Document $project, - ExecutionEvent $queueForExecutions, + Bus $bus, Document $function, string $trigger, string $path, @@ -309,10 +304,10 @@ class Functions extends Action 'duration' => 0.0, ]); - $queueForExecutions - ->setExecution($execution) - ->setProject($project) - ->trigger(); + $bus->dispatch(new ExecutionCompleted( + execution: $execution->getArrayCopy(), + project: $project->getArrayCopy(), + )); } /** @@ -320,7 +315,6 @@ class Functions extends Action * @param Database $dbForProject * @param Func $queueForFunctions * @param Realtime $queueForRealtime - * @param StatsUsage $queueForStatsUsage * @param Event $queueForEvents * @param Document $project * @param Document $function @@ -343,9 +337,8 @@ class Functions extends Action Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, - StatsUsage $queueForStatsUsage, Event $queueForEvents, - ExecutionEvent $queueForExecutions, + Bus $bus, Document $project, Document $function, Executor $executor, @@ -373,19 +366,19 @@ class Functions extends Action if ($deployment->getAttribute('resourceId') !== $functionId) { $errorMessage = 'The execution could not be completed because a corresponding deployment was not found. A function deployment needs to be created before it can be executed. Please create a deployment for your function and try again.'; - $this->fail($errorMessage, $project, $queueForExecutions, $function, $trigger, $path, $method, $user, $jwt, $event); + $this->fail($errorMessage, $project, $bus, $function, $trigger, $path, $method, $user, $jwt, $event); return; } if ($deployment->isEmpty()) { $errorMessage = 'The execution could not be completed because a corresponding deployment was not found. A function deployment needs to be created before it can be executed. Please create a deployment for your function and try again.'; - $this->fail($errorMessage, $project, $queueForExecutions, $function, $trigger, $path, $method, $user, $jwt, $event); + $this->fail($errorMessage, $project, $bus, $function, $trigger, $path, $method, $user, $jwt, $event); return; } if ($deployment->getAttribute('status') !== 'ready') { $errorMessage = 'The execution could not be completed because the build is not ready. Please wait for the build to complete and try again.'; - $this->fail($errorMessage, $project, $queueForExecutions, $function, $trigger, $path, $method, $user, $jwt, $event); + $this->fail($errorMessage, $project, $bus, $function, $trigger, $path, $method, $user, $jwt, $event); return; } @@ -592,26 +585,12 @@ class Functions extends Action $error = $th->getMessage(); $errorCode = $th->getCode(); } finally { - /** Persist final execution status */ - $queueForExecutions - ->setExecution($execution) - ->setProject($project) - ->trigger(); - - /** Trigger usage queue */ - $queueForStatsUsage - ->setProject($project) - ->addMetric(METRIC_EXECUTIONS, 1) - ->addMetric(str_replace(['{resourceType}'], [RESOURCE_TYPE_FUNCTIONS], METRIC_RESOURCE_TYPE_EXECUTIONS), 1) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getSequence()], METRIC_RESOURCE_TYPE_ID_EXECUTIONS), 1) - ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project - ->addMetric(str_replace(['{resourceType}'], [RESOURCE_TYPE_FUNCTIONS], METRIC_RESOURCE_TYPE_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getSequence()], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) - ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) - ->addMetric(str_replace(['{resourceType}'], [RESOURCE_TYPE_FUNCTIONS], METRIC_RESOURCE_TYPE_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getSequence()], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) - ->trigger() - ; + /** Persist final execution status and record usage */ + $bus->dispatch(new ExecutionCompleted( + execution: $execution->getArrayCopy(), + project: $project->getArrayCopy(), + spec: $spec, + )); } $executionModel = new Execution(); diff --git a/src/Appwrite/Utopia/Response/Model/ProviderRepository.php b/src/Appwrite/Utopia/Response/Model/ProviderRepository.php index ddad0b059c..4dd9789872 100644 --- a/src/Appwrite/Utopia/Response/Model/ProviderRepository.php +++ b/src/Appwrite/Utopia/Response/Model/ProviderRepository.php @@ -47,6 +47,18 @@ class ProviderRepository extends Model 'default' => '', 'example' => 'main', ]) + ->addRule('providerInstallationId', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) installation ID.', + 'default' => '', + 'example' => '108104697', + ]) + ->addRule('authorized', [ + 'type' => self::TYPE_BOOLEAN, + 'description' => 'Is VCS (Version Control System) repository authorized for the installation?', + 'default' => false, + 'example' => true, + ]) ->addRule('pushedAt', [ 'type' => self::TYPE_DATETIME, 'description' => 'Last commit date in ISO 8601 format.', diff --git a/src/Utopia/Bus/Bus.php b/src/Utopia/Bus/Bus.php new file mode 100644 index 0000000000..bef39f0481 --- /dev/null +++ b/src/Utopia/Bus/Bus.php @@ -0,0 +1,51 @@ +, Listener[]> */ + private array $listeners = []; + + /** @var ?\Closure(string): mixed */ + private ?\Closure $resolver = null; + + public function setResolver(callable $resolver): self + { + $this->resolver = $resolver(...); + return $this; + } + + public function subscribe(Listener $listener): self + { + foreach ($listener::getEvents() as $event) { + $this->listeners[$event][] = $listener; + } + return $this; + } + + public function dispatch(Event $event): void + { + if ($this->resolver === null) { + throw new \LogicException('Bus resolver must be set via setResolver() before dispatching events'); + } + + $resolver = $this->resolver; + $listeners = $this->listeners[$event::class] ?? []; + + foreach ($listeners as $listener) { + $deps = array_map($resolver, $listener->getInjections()); + Span::init('listener.' . $listener::getName()); + Span::add('bus.event', $event::class); + try { + ($listener->getCallback())($event, ...$deps); + } catch (\Throwable $e) { + Span::error($e); + } finally { + Span::current()?->finish(); + } + } + } +} diff --git a/src/Utopia/Bus/Event.php b/src/Utopia/Bus/Event.php new file mode 100644 index 0000000000..1423f1198d --- /dev/null +++ b/src/Utopia/Bus/Event.php @@ -0,0 +1,7 @@ + */ + protected array $injections = []; + protected ?\Closure $callback = null; + + abstract public static function getName(): string; + + /** + * @return array> + */ + abstract public static function getEvents(): array; + + protected function desc(string $desc): self + { + $this->desc = $desc; + return $this; + } + + protected function inject(string $injection): self + { + $this->injections[] = $injection; + return $this; + } + + protected function callback(callable $callback): self + { + $this->callback = $callback(...); + return $this; + } + + /** @return array */ + public function getInjections(): array + { + return $this->injections; + } + + public function getCallback(): callable + { + if ($this->callback === null) { + throw new \LogicException(static::class . ' must set a callback via $this->callback()'); + } + + return $this->callback; + } +} diff --git a/tests/e2e/Services/Realtime/RealtimeBase.php b/tests/e2e/Services/Realtime/RealtimeBase.php index ee7946d9f7..95f3665e4c 100644 --- a/tests/e2e/Services/Realtime/RealtimeBase.php +++ b/tests/e2e/Services/Realtime/RealtimeBase.php @@ -11,7 +11,8 @@ trait RealtimeBase array $channels = [], array $headers = [], ?string $projectId = null, - ?array $queries = null + ?array $queries = null, + int $timeout = 2 ): WebSocketClient { if (is_null($projectId)) { $projectId = $this->getProject()['$id']; @@ -63,7 +64,7 @@ trait RealtimeBase "ws://appwrite.test/v1/realtime?" . $queryString, [ "headers" => $headers, - "timeout" => 45, + "timeout" => $timeout, ] ); } @@ -74,9 +75,10 @@ trait RealtimeBase * * @param array $queryParams Custom query parameters (e.g., ['channels' => ['project'], 'project' => [...]]) * @param array $headers HTTP headers + * @param int $timeout Timeout in seconds (default: 2) * @return WebSocketClient */ - private function getWebsocketWithCustomQuery(array $queryParams, array $headers = []): WebSocketClient + private function getWebsocketWithCustomQuery(array $queryParams, array $headers = [], int $timeout = 2): WebSocketClient { $queryString = http_build_query($queryParams); @@ -84,7 +86,7 @@ trait RealtimeBase "ws://appwrite.test/v1/realtime?" . $queryString, [ "headers" => $headers, - "timeout" => 45, + "timeout" => $timeout, ] ); } diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTest.php index 8104fa7bd0..30cc70e981 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTest.php @@ -2535,29 +2535,35 @@ class RealtimeCustomClientQueryTest extends Scope $projectId = 'console'; // Subscribe without queries - should receive all events - $clientNoQuery = $this->getWebsocket(['tests'], [ - 'origin' => 'http://localhost', - ], $projectId); + $clientNoQuery = $this->getWebsocket( + channels: ['tests'], + headers: ['origin' => 'http://localhost'], + projectId: $projectId, + timeout: 5 + ); $response = json_decode($clientNoQuery->receive(), true); $this->assertEquals('connected', $response['type']); // Subscribe with matching query - should receive events - $clientWithMatchingQuery = $this->getWebsocket(['tests'], [ - 'origin' => 'http://localhost', - ], $projectId, [ - Query::equal('response', ['WS:/v1/realtime:passed'])->toString(), - ]); + $clientWithMatchingQuery = $this->getWebsocket( + channels: ['tests'], + headers: ['origin' => 'http://localhost'], + projectId: $projectId, + queries: [Query::equal('response', ['WS:/v1/realtime:passed'])->toString()], + timeout: 5 + ); $response = json_decode($clientWithMatchingQuery->receive(), true); $this->assertEquals('connected', $response['type']); // Subscribe with non-matching query - should NOT receive events - $clientWithNonMatchingQuery = $this->getWebsocket(['tests'], [ - 'origin' => 'http://localhost', - ], $projectId, [ - Query::equal('response', ['failed'])->toString(), - ]); + $clientWithNonMatchingQuery = $this->getWebsocket( + channels: ['tests'], + headers: ['origin' => 'http://localhost'], + projectId: $projectId, + queries: [Query::equal('response', ['failed'])->toString()] + ); $response = json_decode($clientWithNonMatchingQuery->receive(), true); $this->assertEquals('connected', $response['type']); diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index 2c903dc365..2c949ca07e 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -123,6 +123,8 @@ class RealtimeCustomClientTest extends Scope $this->assertNotEmpty($response['data']); $this->assertNotEmpty($response['data']['user']); $this->assertCount(16, $response['data']['channels']); + $this->assertIsList($response['data']['channels']); + $this->assertTrue(array_is_list($response['data']['channels'])); $this->assertContains('account', $response['data']['channels']); $this->assertContains('account.' . $userId, $response['data']['channels']); $this->assertContains('files', $response['data']['channels']); @@ -818,8 +820,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - // documents creation on the /databases -> 6 channels - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains('documents', $response['data']['channels']); $this->assertContains('databases.' . $databaseId . '.collections.' . $actorsId . '.documents.' . $documentId, $response['data']['channels']); $this->assertContains('databases.' . $databaseId . '.collections.' . $actorsId . '.documents', $response['data']['channels']); @@ -864,7 +865,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains('documents', $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$documentId}", $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents", $response['data']['channels']); @@ -920,7 +921,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains('documents', $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$documentId}", $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents", $response['data']['channels']); @@ -976,7 +977,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response['data']['payload']['$id']}.create", $response['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.create", $response['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.create", $response['data']['events']); @@ -1008,7 +1009,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response['data']['payload']['$id']}.create", $response['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.create", $response['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.create", $response['data']['events']); @@ -1057,7 +1058,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response['data']['payload']['$id']}.update", $response['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.update", $response['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.update", $response['data']['events']); @@ -1085,7 +1086,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response['data']['payload']['$id']}.update", $response['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.update", $response['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.update", $response['data']['events']); @@ -1113,7 +1114,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response['data']['payload']['$id']}.update", $response['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.update", $response['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.update", $response['data']['events']); @@ -1150,7 +1151,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response['data']['payload']['$id']}.delete", $response['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.delete", $response['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.delete", $response['data']['events']); @@ -1179,7 +1180,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response['data']['payload']['$id']}.delete", $response['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.delete", $response['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.delete", $response['data']['events']); @@ -1208,7 +1209,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response['data']['payload']['$id']}.delete", $response['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.delete", $response['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.delete", $response['data']['events']); @@ -1255,7 +1256,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response['data']['payload']['$id']}.upsert", $response['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.upsert", $response['data']['events']); @@ -1434,7 +1435,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response1['type']); $this->assertNotEmpty($response1['data']); $this->assertArrayHasKey('timestamp', $response1['data']); - $this->assertCount(6, $response1['data']['channels']); + $this->assertCount(8, $response1['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response1['data']['payload']['$id']}.create", $response1['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.create", $response1['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.create", $response1['data']['events']); @@ -1465,7 +1466,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response2['type']); $this->assertNotEmpty($response2['data']); $this->assertArrayHasKey('timestamp', $response2['data']); - $this->assertCount(6, $response2['data']['channels']); + $this->assertCount(8, $response2['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response2['data']['payload']['$id']}.create", $response2['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.create", $response2['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.create", $response2['data']['events']); @@ -1515,7 +1516,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response1['type']); $this->assertNotEmpty($response1['data']); $this->assertArrayHasKey('timestamp', $response1['data']); - $this->assertCount(6, $response1['data']['channels']); + $this->assertCount(8, $response1['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response1['data']['payload']['$id']}.update", $response1['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.update", $response1['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.update", $response1['data']['events']); @@ -1569,7 +1570,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response2['type']); $this->assertNotEmpty($response2['data']); $this->assertArrayHasKey('timestamp', $response2['data']); - $this->assertCount(6, $response2['data']['channels']); + $this->assertCount(8, $response2['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response2['data']['payload']['$id']}.update", $response2['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.update", $response2['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.update", $response2['data']['events']); @@ -1622,7 +1623,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response1['type']); $this->assertNotEmpty($response1['data']); $this->assertArrayHasKey('timestamp', $response1['data']); - $this->assertCount(6, $response1['data']['channels']); + $this->assertCount(8, $response1['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response1['data']['payload']['$id']}.update", $response1['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.update", $response1['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.update", $response1['data']['events']); @@ -1649,7 +1650,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response2['type']); $this->assertNotEmpty($response2['data']); $this->assertArrayHasKey('timestamp', $response2['data']); - $this->assertCount(6, $response2['data']['channels']); + $this->assertCount(8, $response2['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response2['data']['payload']['$id']}.update", $response2['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.update", $response2['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.update", $response2['data']['events']); @@ -1688,7 +1689,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response1['type']); $this->assertNotEmpty($response1['data']); $this->assertArrayHasKey('timestamp', $response1['data']); - $this->assertCount(6, $response1['data']['channels']); + $this->assertCount(8, $response1['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response1['data']['payload']['$id']}.delete", $response1['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.delete", $response1['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.delete", $response1['data']['events']); @@ -1719,7 +1720,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response2['type']); $this->assertNotEmpty($response2['data']); $this->assertArrayHasKey('timestamp', $response2['data']); - $this->assertCount(6, $response2['data']['channels']); + $this->assertCount(8, $response2['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response2['data']['payload']['$id']}.delete", $response2['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.delete", $response2['data']['events']); $this->assertContains("databases.{$databaseId}.collections.*.documents.*.delete", $response2['data']['events']); @@ -1772,7 +1773,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response['data']['payload']['$id']}.upsert", $response['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.upsert", $response['data']['events']); @@ -1810,7 +1811,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$response['data']['payload']['$id']}.upsert", $response['data']['events']); $this->assertContains("databases.*.collections.*.documents.*.upsert", $response['data']['events']); @@ -1952,7 +1953,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains('documents', $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$documentId}", $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents", $response['data']['channels']); @@ -1991,7 +1992,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains('documents', $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$documentId}", $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents", $response['data']['channels']); @@ -2041,7 +2042,7 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(6, $response['data']['channels']); + $this->assertCount(8, $response['data']['channels']); $this->assertContains('documents', $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents.{$documentId}", $response['data']['channels']); $this->assertContains("databases.{$databaseId}.collections.{$actorsId}.documents", $response['data']['channels']); @@ -2224,10 +2225,14 @@ class RealtimeCustomClientTest extends Scope $session = $user['session'] ?? ''; $projectId = $this->getProject()['$id']; - $client = $this->getWebsocket(['executions'], [ - 'origin' => 'http://localhost', - 'cookie' => 'a_session_' . $projectId . '=' . $session - ]); + $client = $this->getWebsocket( + channels: ['executions'], + headers: [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session + ], + timeout: 10 + ); $response = json_decode($client->receive(), true); @@ -2566,17 +2571,66 @@ class RealtimeCustomClientTest extends Scope $session = $user['session'] ?? ''; $projectId = $this->getProject()['$id']; + /** + * Create a shared TablesDB database using the /tablesdb API. + * This database will then be accessed via both /databases and /tablesdb routes. + */ $database = $this->client->call(Client::METHOD_POST, '/tablesdb', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $projectId, 'x-appwrite-key' => $this->getProject()['apiKey'], - ], $this->getHeaders()), [ + ]), [ 'databaseId' => ID::unique(), - 'name' => 'TablesDB Realtime DB', + 'name' => 'TablesDB Cross API Realtime DB', ]); $databaseId = $database['body']['$id']; + $this->assertEquals(201, $database['headers']['status-code']); + + /** + * Legacy collection in the shared database (/databases API). + */ + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Legacy Actors', + 'permissions' => [ + Permission::create(Role::user($user['$id'])), + ], + 'documentSecurity' => true, + ]); + + $collectionId = $collection['body']['$id']; + + $attribute = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]), [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEquals(202, $attribute['headers']['status-code']); + + $this->assertEventually(function () use ($databaseId, $collectionId) { + $attribute = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/name', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ])); + + $this->assertEquals('available', $attribute['body']['status']); + }, 30000, 250); + + /** + * TablesDB table in the same database (/tablesdb API). + */ $table = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $projectId, @@ -2617,20 +2671,92 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals('available', $column['body']['status']); }, 120000, 500); - $client = $this->getWebsocket(['documents', 'collections'], [ + /** + * Two different clients subscribing via legacy (documents/collections) + * and new (rows/tables) channels. + */ + $clientLegacy = $this->getWebsocket(['documents', 'collections'], [ 'origin' => 'http://localhost', 'cookie' => 'a_session_' . $projectId . '=' . $session, ]); - $response = json_decode($client->receive(), true); + $response = json_decode($clientLegacy->receive(), true); $this->assertArrayHasKey('type', $response); $this->assertArrayHasKey('data', $response); $this->assertEquals('connected', $response['type']); $this->assertNotEmpty($response['data']); - $this->assertCount(2, $response['data']['channels']); $this->assertContains('documents', $response['data']['channels']); + $clientTables = $this->getWebsocket(['rows', 'tables'], [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ]); + + $response = json_decode($clientTables->receive(), true); + + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('connected', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertContains('rows', $response['data']['channels']); + + /** + * 1) Operation via legacy /databases API (document create). + * Both clients should receive an event that includes both document- + * style and row-style channels on the shared database. + */ + $documentId = ID::unique(); + + $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()), [ + 'documentId' => $documentId, + 'data' => [ + 'name' => 'Legacy Chris Evans', + ], + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $document['headers']['status-code']); + + $legacyEventForLegacyClient = json_decode($clientLegacy->receive(), true); + $legacyEventForTablesClient = json_decode($clientTables->receive(), true); + + foreach ([$legacyEventForLegacyClient, $legacyEventForTablesClient] as $event) { + $this->assertArrayHasKey('type', $event); + $this->assertArrayHasKey('data', $event); + $this->assertEquals('event', $event['type']); + $this->assertNotEmpty($event['data']); + $this->assertArrayHasKey('timestamp', $event['data']); + + $channels = $event['data']['channels']; + + // Legacy-style channels + $this->assertContains('documents', $channels); + $this->assertContains("databases.{$databaseId}.collections.{$collectionId}.documents", $channels); + $this->assertContains("databases.{$databaseId}.collections.{$collectionId}.documents.{$documentId}", $channels); + + // New rows-style channels mirrored for legacy API + $this->assertContains('rows', $channels); + $this->assertContains("databases.{$databaseId}.tables.{$collectionId}.rows", $channels); + $this->assertContains("databases.{$databaseId}.tables.{$collectionId}.rows.{$documentId}", $channels); + + // TablesDB-prefixed channels should also be present for a tablesdb database + $this->assertContains("tablesdb.{$databaseId}.tables.{$collectionId}.rows", $channels); + $this->assertContains("tablesdb.{$databaseId}.tables.{$collectionId}.rows.{$documentId}", $channels); + } + + /** + * 2) Operation via /tablesdb API (row create). + * Both clients should again receive an event that now also includes + * the tablesdb-prefixed channels alongside the databases-prefixed ones. + */ $rowId = ID::unique(); $row = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows', array_merge([ @@ -2651,31 +2777,296 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals(201, $row['headers']['status-code']); + $tablesEventForLegacyClient = json_decode($clientLegacy->receive(), true); + $tablesEventForTablesClient = json_decode($clientTables->receive(), true); + + foreach ([$tablesEventForLegacyClient, $tablesEventForTablesClient] as $event) { + $this->assertArrayHasKey('type', $event); + $this->assertArrayHasKey('data', $event); + $this->assertEquals('event', $event['type']); + $this->assertNotEmpty($event['data']); + $this->assertArrayHasKey('timestamp', $event['data']); + + $channels = $event['data']['channels']; + + // Core tablesdb row channels + $this->assertContains('rows', $channels); + $this->assertContains("tablesdb.{$databaseId}.tables.{$tableId}.rows", $channels); + $this->assertContains("tablesdb.{$databaseId}.tables.{$tableId}.rows.{$rowId}", $channels); + + // Collections/legacy-style compatibility channels + $this->assertContains('documents', $channels); + $this->assertContains("databases.{$databaseId}.tables.{$tableId}.rows", $channels); + $this->assertContains("databases.{$databaseId}.tables.{$tableId}.rows.{$rowId}", $channels); + $this->assertContains("databases.{$databaseId}.collections.{$tableId}.documents", $channels); + $this->assertContains("databases.{$databaseId}.collections.{$tableId}.documents.{$rowId}", $channels); + + $this->assertContains("databases.{$databaseId}.tables.{$tableId}.rows.{$rowId}.create", $event['data']['events']); + $this->assertNotEmpty($event['data']['payload']); + $this->assertEquals('Chris Evans', $event['data']['payload']['name']); + } + + /** + * 3) Legacy database accessed via /tablesdb routes. + * A database created via /databases but operated on via /tablesdb + * should also expose both legacy and tablesdb-prefixed channels. + */ + $legacyDatabase = $this->client->call(Client::METHOD_POST, '/databases', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'Legacy DB via TablesDB Route', + ]); + + $this->assertEquals(201, $legacyDatabase['headers']['status-code']); + + $legacyDatabaseId = $legacyDatabase['body']['$id']; + + $legacyTable = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $legacyDatabaseId . '/tables', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders()), [ + 'tableId' => ID::unique(), + 'name' => 'Legacy Actors', + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $legacyTableId = $legacyTable['body']['$id']; + + $legacyColumn = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $legacyDatabaseId . '/tables/' . $legacyTableId . '/columns/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders()), [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEquals(202, $legacyColumn['headers']['status-code']); + + $this->assertEventually(function () use ($legacyDatabaseId, $legacyTableId) { + $column = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $legacyDatabaseId . '/tables/' . $legacyTableId . '/columns/name', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders())); + + $this->assertEquals(200, $column['headers']['status-code']); + $this->assertEquals('available', $column['body']['status']); + }, 120000, 500); + + $legacyRowId = ID::unique(); + + $legacyRow = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $legacyDatabaseId . '/tables/' . $legacyTableId . '/rows', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders()), [ + 'rowId' => $legacyRowId, + 'data' => [ + 'name' => 'Legacy Tables Route', + ], + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $legacyRow['headers']['status-code']); + + $legacyTablesEventForLegacyClient = json_decode($clientLegacy->receive(), true); + $legacyTablesEventForTablesClient = json_decode($clientTables->receive(), true); + + foreach ([$legacyTablesEventForLegacyClient, $legacyTablesEventForTablesClient] as $event) { + $this->assertArrayHasKey('type', $event); + $this->assertArrayHasKey('data', $event); + $this->assertEquals('event', $event['type']); + $this->assertNotEmpty($event['data']); + $this->assertArrayHasKey('timestamp', $event['data']); + + $channels = $event['data']['channels']; + $events = $event['data']['events']; + $this->assertIsList($channels); + $this->assertIsList($events); + + // Core tablesdb row channels for legacy db accessed via tablesdb + $this->assertContains('rows', $channels); + $this->assertContains("tablesdb.{$legacyDatabaseId}.tables.{$legacyTableId}.rows", $channels); + $this->assertContains("tablesdb.{$legacyDatabaseId}.tables.{$legacyTableId}.rows.{$legacyRowId}", $channels); + + // Legacy compatibility channels must also exist + $this->assertContains('documents', $channels); + $this->assertContains("databases.{$legacyDatabaseId}.tables.{$legacyTableId}.rows", $channels); + $this->assertContains("databases.{$legacyDatabaseId}.tables.{$legacyTableId}.rows.{$legacyRowId}", $channels); + $this->assertContains("databases.{$legacyDatabaseId}.collections.{$legacyTableId}.documents", $channels); + $this->assertContains("databases.{$legacyDatabaseId}.collections.{$legacyTableId}.documents.{$legacyRowId}", $channels); + } + + $clientLegacy->close(); + $clientTables->close(); + } + + public function testChannelTablesDBRowUpdate() + { + $user = $this->getUser(); + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + + /** + * Create a tablesdb database + table + column + row. + */ + $database = $this->client->call(Client::METHOD_POST, '/tablesdb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders()), [ + 'databaseId' => ID::unique(), + 'name' => 'Row Update DB', + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + $table = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders()), [ + 'tableId' => ID::unique(), + 'name' => 'Assembly', + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $table['headers']['status-code']); + $tableId = $table['body']['$id']; + + $column = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders()), [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEquals(202, $column['headers']['status-code']); + + $this->assertEventually(function () use ($databaseId, $tableId) { + $column = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/name', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders())); + + $this->assertEquals(200, $column['headers']['status-code']); + $this->assertEquals('available', $column['body']['status']); + }, 120000, 500); + + // Seed a row so we can listen to its update + $rowId = ID::unique(); + + $row = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders()), [ + 'rowId' => $rowId, + 'data' => [ + 'name' => 'Initial Name', + ], + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $row['headers']['status-code']); + + /** + * Subscribe to a specific row channel using both legacy and tablesdb-style prefixes. + * This mimics a client subscribing to a concrete "resource" channel and + * expecting a single event list for updates. + */ + $client = $this->getWebsocket([ + "databases.{$databaseId}.tables.{$tableId}.rows.{$rowId}", + "tablesdb.{$databaseId}.tables.{$tableId}.rows.{$rowId}", + ], [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ]); + $response = json_decode($client->receive(), true); $this->assertArrayHasKey('type', $response); $this->assertArrayHasKey('data', $response); - $this->assertEquals('event', $response['type']); + $this->assertEquals('connected', $response['type']); $this->assertNotEmpty($response['data']); - $this->assertArrayHasKey('timestamp', $response['data']); + $this->assertIsList($response['data']['channels']); - // Core channels for tablesdb row events - $this->assertContains('rows', $response['data']['channels']); + /** + * Trigger a row update via the dedicated /tablesdb row update endpoint. + * Event label: databases.[databaseId].tables.[tableId].rows.[rowId].update + * Our Event + Realtime logic should enrich this to include: + * - databases.{dbId}.tables.{tableId}.rows.{rowId}.update + * - tablesdb.{dbId}.tables.{tableId}.rows.{rowId}.update + */ + $update = $this->client->call(Client::METHOD_PATCH, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows/' . $rowId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders()), [ + 'data' => [ + 'name' => 'Updated Name', + ], + ]); - $this->assertContains("tablesdb.{$databaseId}.tables.{$tableId}.rows", $response['data']['channels']); - $this->assertContains("tablesdb.{$databaseId}.tables.{$tableId}.rows.{$rowId}", $response['data']['channels']); + $this->assertEquals(200, $update['headers']['status-code']); - // Collections-style compatibility channels - $this->assertContains('documents', $response['data']['channels']); - $this->assertContains("databases.{$databaseId}.tables.{$tableId}.rows", $response['data']['channels']); - $this->assertContains("databases.{$databaseId}.tables.{$tableId}.rows.{$rowId}", $response['data']['channels']); - $this->assertContains("databases.{$databaseId}.collections.{$tableId}.documents", $response['data']['channels']); - $this->assertContains("databases.{$databaseId}.collections.{$tableId}.documents.{$rowId}", $response['data']['channels']); + $event = json_decode($client->receive(), true); - // Primary event should still be present - $this->assertContains("databases.{$databaseId}.tables.{$tableId}.rows.{$rowId}.create", $response['data']['events']); - $this->assertNotEmpty($response['data']['payload']); - $this->assertEquals('Chris Evans', $response['data']['payload']['name']); + $this->assertArrayHasKey('type', $event); + $this->assertArrayHasKey('data', $event); + $this->assertEquals('event', $event['type']); + $this->assertNotEmpty($event['data']); + $this->assertArrayHasKey('timestamp', $event['data']); + + $channels = $event['data']['channels']; + $events = $event['data']['events']; + + // Ensure channels and events are list-type arrays + $this->assertIsList($channels); + $this->assertIsList($events); + + // Legacy + tablesdb row channels must be present + $this->assertContains('rows', $channels); + $this->assertContains("databases.{$databaseId}.tables.{$tableId}.rows", $channels); + $this->assertContains("databases.{$databaseId}.tables.{$tableId}.rows.{$rowId}", $channels); + $this->assertContains("tablesdb.{$databaseId}.tables.{$tableId}.rows", $channels); + $this->assertContains("tablesdb.{$databaseId}.tables.{$tableId}.rows.{$rowId}", $channels); + + // Both databases.* and tablesdb.* update events should be emitted + $this->assertContains("databases.{$databaseId}.tables.{$tableId}.rows.{$rowId}.update", $events); + $this->assertContains("tablesdb.{$databaseId}.tables.{$tableId}.rows.{$rowId}.update", $events); + + $this->assertNotEmpty($event['data']['payload']); + $this->assertEquals('Updated Name', $event['data']['payload']['name']); $client->close(); }