mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Keep template-into-repo push in the Builds worker; hand off to jobs after
Reverts the jobs-worker appwrite.template.push message. The template flow enqueues to the Builds worker unchanged on both backends; the worker does its existing clone + merge + commit + push, and when _APP_BUILDS_BACKEND=orchestrator it then submits the build job for the pushed commit (presigned tarball source, buildPath onto the mounted volume) instead of running the executor build. The jobs worker's onLog promotes 'processing' (set by the Builds worker) to 'building' as well as 'waiting'. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0c075d843e
commit
e9a9a4162d
4 files changed
+50
-147
No files matched your search
@@ -4,9 +4,7 @@ namespace Appwrite\Platform\Modules\Compute;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Message\Jobs as JobsMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Event\Publisher\Jobs as JobsPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\Filter\BranchDomain as BranchDomainFilter;
|
||||
use Appwrite\Platform\Action;
|
||||
@@ -71,7 +69,7 @@ class Base extends Action
|
||||
return $allowedSpecifications[0];
|
||||
}
|
||||
|
||||
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null, ?JobsPublisher $publisherForJobs = null): Document
|
||||
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null): Document
|
||||
{
|
||||
$deploymentId = ID::unique();
|
||||
$entrypoint = $function->getAttribute('entrypoint', '');
|
||||
@@ -122,13 +120,16 @@ class Base extends Action
|
||||
|
||||
$repositoryUrl = "https://github.com/$owner/$repositoryName";
|
||||
|
||||
// Build a VCS function deployment on the jobs-service when the caller
|
||||
// opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. Sites stay on
|
||||
// the executor. When on jobs the deployment is pre-declared 'waiting'
|
||||
// with its buildPath so build.sh writes output onto the mounted volume.
|
||||
// Build a plain (non-template) VCS function deployment on the jobs-service
|
||||
// when the caller opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator.
|
||||
// Sites stay on the executor; template-into-repo pushes go through the
|
||||
// Builds worker (which does the git write, then hands the build to the
|
||||
// jobs-service itself when on orchestrator). When on jobs the deployment
|
||||
// is pre-declared 'waiting' with its buildPath so build.sh writes output
|
||||
// onto the mounted volume.
|
||||
$useJobs = $jobs !== null
|
||||
&& $function->getCollection() === 'functions'
|
||||
&& ($template->isEmpty() || $publisherForJobs !== null)
|
||||
&& $template->isEmpty()
|
||||
&& System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
|
||||
$buildFields = $useJobs
|
||||
? ['status' => 'waiting', 'buildPath' => Job::buildPath($project->getId(), $deploymentId)]
|
||||
@@ -168,22 +169,7 @@ class Base extends Action
|
||||
...$buildFields,
|
||||
]));
|
||||
|
||||
if ($useJobs && ! $template->isEmpty()) {
|
||||
// The template must be merged into the repo and pushed as a commit
|
||||
// before the build — a git *write* the jobs-service artifact system
|
||||
// has no primitive for. Hand off to the jobs worker, which pushes
|
||||
// the template and then submits the build for the resulting commit.
|
||||
$publisherForJobs->enqueue(new JobsMessage(
|
||||
project: $project,
|
||||
id: 'template-push-' . $deployment->getId(),
|
||||
event: 'appwrite.template.push',
|
||||
data: [
|
||||
'meta' => ['projectId' => $project->getId(), 'deploymentId' => $deployment->getId()],
|
||||
'template' => $template->getArrayCopy(),
|
||||
'platform' => $platform,
|
||||
],
|
||||
));
|
||||
} elseif ($useJobs) {
|
||||
if ($useJobs) {
|
||||
$ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch');
|
||||
$source = [
|
||||
'url' => $github->getRepositoryPresignedUrl($owner, $repositoryName, $ref),
|
||||
|
||||
@@ -6,7 +6,6 @@ use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Event\Publisher\Jobs as JobsPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\Platform\Action;
|
||||
use Appwrite\Platform\Modules\Compute\Base;
|
||||
@@ -84,7 +83,6 @@ class Create extends Base
|
||||
->inject('queueForEvents')
|
||||
->inject('project')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('publisherForJobs')
|
||||
->inject('jobs')
|
||||
->inject('gitHub')
|
||||
->inject('authorization')
|
||||
@@ -107,7 +105,6 @@ class Create extends Base
|
||||
Event $queueForEvents,
|
||||
Document $project,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
JobsPublisher $publisherForJobs,
|
||||
Jobs $jobs,
|
||||
GitHub $github,
|
||||
Authorization $authorization,
|
||||
@@ -132,10 +129,10 @@ class Create extends Base
|
||||
]);
|
||||
|
||||
if (!empty($function->getAttribute('providerRepositoryId'))) {
|
||||
// VCS-connected function: the template is merged into the user's repo
|
||||
// and pushed as a commit, then that commit is built. The executor does
|
||||
// the push inside the build job; the jobs backend does it in the jobs
|
||||
// worker (which then submits the build for the resulting commit).
|
||||
// VCS-connected function: the Builds worker merges the template into
|
||||
// the user's repo, pushes it as a commit, then builds that commit —
|
||||
// on the executor itself, or by submitting a job when
|
||||
// _APP_BUILDS_BACKEND=orchestrator.
|
||||
$installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId'));
|
||||
|
||||
$deployment = $this->redeployVcsFunction(
|
||||
@@ -150,9 +147,7 @@ class Create extends Base
|
||||
activate: $activate,
|
||||
platform: $platform,
|
||||
referenceType: $type,
|
||||
reference: $reference,
|
||||
jobs: $jobs,
|
||||
publisherForJobs: $publisherForJobs
|
||||
reference: $reference
|
||||
);
|
||||
|
||||
$queueForEvents
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
namespace Appwrite\Platform\Modules\Functions\Workers;
|
||||
|
||||
use Ahc\Jwt\JWT;
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Func as FunctionMessage;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
@@ -20,6 +21,7 @@ use Exception;
|
||||
use Executor\Exception as ExecutorException;
|
||||
use Executor\Exception\Timeout as ExecutorTimeout;
|
||||
use Executor\Executor;
|
||||
use OpenRuntimes\Orchestrator\Jobs as JobsClient;
|
||||
use Swoole\Coroutine as Co;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Config\Config;
|
||||
@@ -98,6 +100,7 @@ class Builds extends Action
|
||||
->inject('deviceForFiles')
|
||||
->inject('log')
|
||||
->inject('executor')
|
||||
->inject('jobs')
|
||||
->inject('plan')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
@@ -124,6 +127,7 @@ class Builds extends Action
|
||||
Device $deviceForFiles,
|
||||
Log $log,
|
||||
Executor $executor,
|
||||
JobsClient $jobs,
|
||||
array $plan
|
||||
): void {
|
||||
$payload = $message->getPayload();
|
||||
@@ -168,6 +172,7 @@ class Builds extends Action
|
||||
$getIsResourceBlocked,
|
||||
$log,
|
||||
$executor,
|
||||
$jobs,
|
||||
$plan,
|
||||
$platform,
|
||||
(int) ($payload['timeout'] ?? System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900))
|
||||
@@ -204,6 +209,7 @@ class Builds extends Action
|
||||
callable $getIsResourceBlocked,
|
||||
Log $log,
|
||||
Executor $executor,
|
||||
JobsClient $jobs,
|
||||
array $plan,
|
||||
array $platform,
|
||||
int $timeout
|
||||
@@ -518,6 +524,29 @@ class Builds extends Action
|
||||
->trigger();
|
||||
}
|
||||
|
||||
// On the jobs backend the only build reaching this worker is the
|
||||
// template-into-repo push above (a git *write* the jobs-service
|
||||
// artifact system has no primitive for). With the commit pushed,
|
||||
// hand the build to the jobs-service like any other VCS commit:
|
||||
// source from the presigned tarball, output onto the mounted
|
||||
// builds volume, progress via callbacks to the jobs worker.
|
||||
if ($resource->getCollection() === 'functions'
|
||||
&& System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
|
||||
'buildPath' => Job::buildPath($project->getId(), $deploymentId),
|
||||
]));
|
||||
|
||||
$ref = $deployment->getAttribute('providerCommitHash') ?: $branchName;
|
||||
$jobs->create(...Job::build($project, $resource, $deployment, $platform, [
|
||||
'url' => $github->getRepositoryPresignedUrl($cloneOwner, $cloneRepository, $ref),
|
||||
'subdir' => $resource->getAttribute('providerRootDirectory', ''),
|
||||
]));
|
||||
|
||||
Console::execute('rm -rf ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$tmpPath = '/tmp/builds/' . $deploymentId;
|
||||
$tmpPathFile = $tmpPath . '/code.tar.gz';
|
||||
$localDevice = new Local();
|
||||
|
||||
@@ -13,9 +13,7 @@ use Appwrite\Event\Webhook;
|
||||
use Appwrite\Usage\Build as BuildUsage;
|
||||
use Appwrite\Usage\Context as UsageContext;
|
||||
use Appwrite\Utopia\Response\Model\Deployment;
|
||||
use OpenRuntimes\Orchestrator\Jobs as JobsClient;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Console;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
@@ -66,7 +64,6 @@ class Jobs extends Action
|
||||
->inject('publisherForUsage')
|
||||
->inject('usage')
|
||||
->inject('deviceForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('gitHub')
|
||||
->inject('cache')
|
||||
->inject('locks')
|
||||
@@ -85,7 +82,6 @@ class Jobs extends Action
|
||||
UsagePublisher $publisherForUsage,
|
||||
UsageContext $usage,
|
||||
Device $deviceForBuilds,
|
||||
JobsClient $jobs,
|
||||
GitHub $github,
|
||||
Cache $cache,
|
||||
callable $locks,
|
||||
@@ -97,7 +93,7 @@ class Jobs extends Action
|
||||
return;
|
||||
}
|
||||
|
||||
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $jobs, $github, $cache, $deploymentId): void {
|
||||
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $github, $cache, $deploymentId): void {
|
||||
if ($event->id !== '') {
|
||||
$key = 'jobs-event-' . $event->id;
|
||||
if ($cache->load($key, self::DEDUPE_TTL) !== false) {
|
||||
@@ -112,13 +108,10 @@ class Jobs extends Action
|
||||
}
|
||||
|
||||
// The build writes its output onto the mounted volume before build.sh
|
||||
// exits, so the exit callback is a truthful completion signal. A
|
||||
// template push only finalizes when it fails (success just submits
|
||||
// the build job, whose own exit finalizes later).
|
||||
$finalizing = \in_array($event->event, ['orchestrator.job.exit', 'appwrite.template.push'], true);
|
||||
// exits, so the exit callback is a truthful completion signal.
|
||||
$finalizing = $event->event === 'orchestrator.job.exit';
|
||||
|
||||
$deployment = match ($event->event) {
|
||||
'appwrite.template.push' => $this->onTemplatePush($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $jobs, $github, $usage, $publisherForUsage),
|
||||
'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data),
|
||||
'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $deployment, $event->data),
|
||||
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds, $github),
|
||||
@@ -157,7 +150,9 @@ class Jobs extends Action
|
||||
|
||||
// First build output means the build is running: promote the queued
|
||||
// deployment to 'building' and stamp its start (mirrors the executor).
|
||||
if ($deployment->getAttribute('status') === 'waiting') {
|
||||
// 'processing' occurs when the Builds worker prepared the source (the
|
||||
// template-into-repo push) before handing the build to the jobs-service.
|
||||
if (\in_array($deployment->getAttribute('status'), ['waiting', 'processing'], true)) {
|
||||
$update['status'] = 'building';
|
||||
$update['buildStartedAt'] = DateTime::now();
|
||||
}
|
||||
@@ -165,108 +160,6 @@ class Jobs extends Action
|
||||
return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($update));
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge a template into a VCS-connected function's repository, push it as a
|
||||
* commit, and submit the build job for that commit. This is the one VCS
|
||||
* flow needing a git *write*, which the jobs-service artifact system has no
|
||||
* primitive for — so it runs here (mirroring the executor Builds worker)
|
||||
* before the build is handed to the jobs-service. A failed push finalizes
|
||||
* the deployment as failed; the build's own exit callback finalizes success.
|
||||
*/
|
||||
private function onTemplatePush(
|
||||
Database $dbForProject,
|
||||
Database $dbForPlatform,
|
||||
Document $project,
|
||||
Document $deployment,
|
||||
array $data,
|
||||
JobsClient $jobs,
|
||||
GitHub $github,
|
||||
UsageContext $usage,
|
||||
UsagePublisher $publisherForUsage,
|
||||
): Document {
|
||||
$function = $dbForProject->getDocument('functions', $deployment->getAttribute('resourceId'));
|
||||
|
||||
try {
|
||||
$installation = $dbForPlatform->getDocument('installations', $deployment->getAttribute('installationId', ''));
|
||||
$providerInstallationId = $installation->getAttribute('providerInstallationId', '');
|
||||
if ($function->isEmpty() || $providerInstallationId === '') {
|
||||
throw new \Exception('Function or installation not found.');
|
||||
}
|
||||
$github->initializeVariables($providerInstallationId, System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''), System::getEnv('_APP_VCS_GITHUB_APP_ID', ''));
|
||||
|
||||
$commitHash = $this->pushTemplate($github, $function, $deployment, new Document($data['template'] ?? []));
|
||||
|
||||
$owner = $deployment->getAttribute('providerRepositoryOwner', '');
|
||||
$repository = $deployment->getAttribute('providerRepositoryName', '');
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
|
||||
'providerCommitHash' => $commitHash,
|
||||
'providerCommitAuthorUrl' => APP_VCS_GITHUB_URL,
|
||||
'providerCommitAuthor' => APP_VCS_GITHUB_USERNAME,
|
||||
'providerCommitMessage' => "Create '" . $function->getAttribute('name', '') . "' function",
|
||||
'providerCommitUrl' => "https://github.com/{$owner}/{$repository}/commit/{$commitHash}",
|
||||
]));
|
||||
|
||||
$source = [
|
||||
'url' => $github->getRepositoryPresignedUrl($owner, $repository, $commitHash),
|
||||
'subdir' => $function->getAttribute('providerRootDirectory', ''),
|
||||
];
|
||||
$jobs->create(...Job::build($project, $function, $deployment, $data['platform'] ?? [], $source));
|
||||
|
||||
return $deployment;
|
||||
} catch (\Throwable $error) {
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, $error->getMessage(), $usage, $publisherForUsage, $github);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clone the function's repository, merge the template into it and push the
|
||||
* result, returning the new commit hash. Replicates the executor Builds
|
||||
* worker's clone + rsync + commit + push.
|
||||
*/
|
||||
private function pushTemplate(GitHub $github, Document $function, Document $deployment, Document $template): string
|
||||
{
|
||||
$owner = $deployment->getAttribute('providerRepositoryOwner', '');
|
||||
$repository = $deployment->getAttribute('providerRepositoryName', '');
|
||||
$branch = $deployment->getAttribute('providerBranch') ?: $function->getAttribute('providerBranch', 'main');
|
||||
|
||||
$rootDirectory = \ltrim(\ltrim(\rtrim($function->getAttribute('providerRootDirectory', ''), '/'), '.'), '/');
|
||||
$templateRootDirectory = \ltrim(\ltrim(\rtrim($template->getAttribute('rootDirectory', ''), '/'), '.'), '/');
|
||||
|
||||
$tmpDirectory = '/tmp/builds/' . $deployment->getId();
|
||||
$repoDirectory = $tmpDirectory . '/code';
|
||||
$templateDirectory = $tmpDirectory . '/template';
|
||||
$stdout = '';
|
||||
$stderr = '';
|
||||
|
||||
try {
|
||||
if (Console::execute($github->generateCloneCommand($owner, $repository, $branch, GitHub::CLONE_TYPE_BRANCH, $repoDirectory, $rootDirectory), '', $stdout, $stderr) !== 0) {
|
||||
throw new \Exception('Unable to clone repository: ' . $stderr);
|
||||
}
|
||||
if (Console::execute($github->generateCloneCommand($template->getAttribute('ownerName', ''), $template->getAttribute('repositoryName', ''), $template->getAttribute('referenceValue', ''), $template->getAttribute('referenceType', ''), $templateDirectory, $templateRootDirectory), '', $stdout, $stderr) !== 0) {
|
||||
throw new \Exception('Unable to clone template repository: ' . $stderr);
|
||||
}
|
||||
|
||||
Console::execute('mkdir -p ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr);
|
||||
Console::execute('mkdir -p ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr);
|
||||
|
||||
// Merge the template into the repo, then commit + push the branch.
|
||||
Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr);
|
||||
|
||||
$message = \escapeshellarg("Create '" . $function->getAttribute('name', '') . "' function");
|
||||
if (Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($repoDirectory) . ' && git checkout -b ' . \escapeshellarg($branch) . ' && git add . && git commit -m ' . $message . ' && git push origin ' . \escapeshellarg($branch), '', $stdout, $stderr) !== 0) {
|
||||
throw new \Exception('Unable to push to repository: ' . $stderr);
|
||||
}
|
||||
|
||||
if (Console::execute('cd ' . \escapeshellarg($repoDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr) !== 0) {
|
||||
throw new \Exception('Unable to resolve commit hash: ' . $stderr);
|
||||
}
|
||||
|
||||
return \trim($stdout);
|
||||
} finally {
|
||||
Console::execute('rm -rf ' . \escapeshellarg($tmpDirectory), '', $stdout, $stderr);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a reported artifact size. Remote-source builds (templates / VCS)
|
||||
* fetch the source in the sidecar, so Appwrite can't size it the way the
|
||||
|
||||
Reference in new issue
Block a user