Extract a Deployments service to centralize build dispatch across functions and sites

Consolidates the scattered deployment-creation and build-dispatch logic across
HTTP actions (manual upload, duplicate, template, VCS webhook, VCS reconnect)
into a single Appwrite\Service\Deployments abstraction, with Executor and
Jobs implementations selected per-request by _APP_BUILDS_BACKEND. Sites are
now forced onto the executor implementation regardless of that env var, since
the jobs-service payload builder is function-shaped.

- upload() for chunked-upload bookkeeping (no build), createFromUpload()/
  createFromRef()/createFromUrl() for the three real build-source shapes,
  delete() for best-effort cancel — replaces duplicated permissions/
  resourceId-triplet construction and deactivate-others logic at every call
  site.
- Appwrite\Compute\Job merged into Deployments\Jobs; its output/cache
  handling is now an overridable storage() extension point (late-static-bound)
  so a fork can swap the shared-volume strategy for upload/download artifacts
  without touching payload construction.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
loks0nandClaude Sonnet 5 committed 2026-07-14 14:05:13 +01:00
1 parent 8c55096753
commit 11550cf07f
18 files changed
+567 -438

No files matched your search

+15
View File
@@ -8,6 +8,7 @@ use Appwrite\Databases\TransactionState;
use Appwrite\Event\Context\Audit as AuditContext;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Func as FunctionMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Event\Publisher\Func as FunctionPublisher;
use Appwrite\Event\Realtime;
use Appwrite\Event\Webhook;
@@ -20,10 +21,14 @@ use Appwrite\Network\Cors;
use Appwrite\Network\Platform;
use Appwrite\Network\Validator\Origin;
use Appwrite\Network\Validator\Redirect;
use Appwrite\Service\Deployments\Executor as DeploymentsExecutor;
use Appwrite\Service\Deployments\Jobs as DeploymentsJobs;
use Appwrite\Usage\Context as UsageContext;
use Appwrite\Utopia\Database\Documents\User;
use Appwrite\Utopia\Request;
use Appwrite\Utopia\Response;
use Executor\Executor;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Agents\Adapters\Appwrite as AppwriteAdapter;
use Utopia\Agents\Agent;
use Utopia\Audit\Adapter\Database as AdapterDatabase;
@@ -191,6 +196,16 @@ return function (Container $context): void {
$publisher,
new Queue(System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME), 'utopia-queue', Event::FUNCTIONS_QUEUE_TTL)
), ['publisher']);
$context->set('deployments', function (BuildPublisher $publisherForBuilds, Jobs $jobs, Database $dbForProject, Document $project, Executor $executor, array $platform, Request $request) {
// The jobs-service orchestrator backend only understands functions
// (its payload builder reads function-shaped attributes like
// `runtime`); sites always build on the executor.
$isSite = \str_starts_with($request->getURI(), '/v1/sites');
return !$isSite && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'
? new DeploymentsJobs($jobs, $dbForProject, $project, $platform)
: new DeploymentsExecutor($publisherForBuilds, $dbForProject, $project, $executor, $platform);
}, ['publisherForBuilds', 'jobs', 'dbForProject', 'project', 'executor', 'platform', 'request']);
$context->set('eventProcessor', fn () => new EventProcessor(), []);
$context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory(
$pools,
+27 -32
View File
@@ -2,7 +2,6 @@
namespace Appwrite\Platform\Modules\Compute;
use Appwrite\Compute\Job;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Extend\Exception;
@@ -10,7 +9,7 @@ use Appwrite\Filter\BranchDomain as BranchDomainFilter;
use Appwrite\Platform\Action;
use Appwrite\Platform\Modules\Compute\Validator\Specification as SpecificationValidator;
use Appwrite\Platform\Permission as AppwritePermission;
use OpenRuntimes\Orchestrator\Jobs;
use Appwrite\Service\Deployments;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
@@ -69,7 +68,7 @@ class Base extends Action
return $allowedSpecifications[0];
}
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null): Document
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Deployments $deployments = null): Document
{
$deploymentId = ID::unique();
$entrypoint = $function->getAttribute('entrypoint', '');
@@ -120,28 +119,8 @@ class Base extends Action
$repositoryUrl = "https://github.com/$owner/$repositoryName";
// Build a plain (non-template) VCS function deployment on the jobs-service
// when the caller opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator.
// Sites stay on the executor; template-into-repo pushes go through the
// Builds worker (which does the git write, then hands the build to the
// jobs-service itself when on orchestrator). When on jobs the deployment
// is pre-declared 'waiting' with its buildPath so build.sh writes output
// onto the mounted volume.
$useJobs = $jobs !== null
&& $function->getCollection() === 'functions'
&& $template->isEmpty()
&& System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
$buildFields = $useJobs
? ['status' => 'waiting', 'buildPath' => Job::buildPath($project->getId(), $deploymentId)]
: [];
$deployment = $dbForProject->createDocument('deployments', new Document([
$deployment = new Document([
'$id' => $deploymentId,
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'resourceId' => $function->getId(),
'resourceInternalId' => $function->getSequence(),
'resourceType' => 'functions',
@@ -166,17 +145,33 @@ class Base extends Action
'providerBranch' => $providerBranch,
'providerRootDirectory' => $function->getAttribute('providerRootDirectory', ''),
'activate' => $activate,
...$buildFields,
]));
]);
if ($useJobs) {
// Build a plain (non-template) VCS function deployment through
// $deployments (executor or jobs-service, decided by
// _APP_BUILDS_BACKEND) when the caller opts in. Sites stay on the
// executor; template-into-repo pushes go through the Builds worker
// (which does the git write, then hands the build to the
// jobs-service itself when on orchestrator).
if ($deployments !== null && $function->getCollection() === 'functions' && $template->isEmpty()) {
$ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch');
$source = [
'url' => $github->getRepositoryPresignedUrl($owner, $repositoryName, $ref),
'subdir' => $function->getAttribute('providerRootDirectory', ''),
];
$jobs->create(...Job::build($project, $function, $deployment, $platform, $source));
$deployment = $deployments->createFromUrl(
$function,
$deployment,
$github->getRepositoryPresignedUrl($owner, $repositoryName, $ref),
$function->getAttribute('providerRootDirectory', ''),
);
} else {
$deployment = $dbForProject->createDocument('deployments', new Document([
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
...$deployment->getArrayCopy(),
'status' => 'waiting',
]));
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $function,
@@ -2,24 +2,18 @@
namespace Appwrite\Platform\Modules\Functions\Http\Deployments;
use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Extend\Exception;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\ContentType;
use Appwrite\SDK\Method;
use Appwrite\SDK\MethodType;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Service\Deployments;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
use Utopia\Http\Adapter\Swoole\Request;
@@ -92,11 +86,9 @@ class Create extends Action
->inject('project')
->inject('deviceForFunctions')
->inject('deviceForLocal')
->inject('publisherForBuilds')
->inject('jobs')
->inject('deployments')
->inject('plan')
->inject('authorization')
->inject('platform')
->inject('locks')
->callback($this->action(...));
}
@@ -114,11 +106,9 @@ class Create extends Action
Document $project,
Device $deviceForFunctions,
Device $deviceForLocal,
BuildPublisher $publisherForBuilds,
Jobs $jobs,
Deployments $deployments,
array $plan,
Authorization $authorization,
array $platform,
callable $locks
) {
$activate = \strval($activate) === 'true' || \strval($activate) === '1';
@@ -227,7 +217,7 @@ class Create extends Action
$type = $request->getHeaderLine('x-sdk-language') === 'cli' ? 'cli' : 'manual';
try {
$locks($lockKey, 600, function () use ($activate, &$chunks, $commands, $contentRange, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, &$metadata, $path, $type, &$completed, $response): void {
$locks($lockKey, 600, function () use ($activate, &$chunks, $commands, $contentRange, $dbForProject, $deploymentId, $deployments, $deviceForFunctions, $entrypoint, $fileSize, &$function, &$metadata, $path, $type, &$completed, $response): void {
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
if (!$deployment->isEmpty()) {
@@ -249,16 +239,8 @@ class Create extends Action
$deviceForFunctions->prepareUpload($path, $metadata['content_type'] ?? '', $chunks, $metadata);
if (!empty($contentRange)) {
$deployment = $dbForProject->createDocument('deployments', new Document([
$deployment = $deployments->upload($function, $deployment->setAttributes([
'$id' => $deploymentId,
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'resourceInternalId' => $function->getSequence(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'entrypoint' => $entrypoint,
'buildCommands' => $commands,
'startCommand' => $function->getAttribute('startCommand', ''),
@@ -269,7 +251,7 @@ class Create extends Action
'sourceChunksUploaded' => 0,
'activate' => $activate,
'sourceMetadata' => $metadata,
'type' => $type
'type' => $type,
]));
}
}
@@ -291,7 +273,7 @@ class Create extends Action
}
try {
$locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $mergeUploadMetadata, $platform, $project, $publisherForBuilds, $jobs, $queueForEvents, $response, $type): void {
$locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $path, &$metadata, $mergeUploadMetadata, $deployments, $queueForEvents, $response, $type): void {
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$uploaded = 0;
@@ -315,86 +297,24 @@ class Create extends Action
if ($chunksUploaded === $chunks && $uploaded < $chunks) {
$deviceForFunctions->finalizeUpload($path, $chunks, $metadata);
if ($activate) {
// Remove deploy for all other deployments.
$activeDeployments = $dbForProject->find('deployments', [
Query::equal('activate', [true]),
Query::equal('resourceId', [$functionId]),
Query::equal('resourceType', ['functions'])
]);
foreach ($activeDeployments as $activeDeployment) {
$dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document([
'activate' => false,
]));
}
}
$fileSize = $deviceForFunctions->getFileSize($path);
// Build backend for manual-upload function deployments:
// 'orchestrator' (open-runtimes jobs-service, submitted below
// in the request flow) or 'executor' (default; enqueued to the
// Builds worker). Selected by _APP_BUILDS_BACKEND.
$useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
// Fields marking the deployment as queued. The build worker
// promotes 'waiting' → 'building' on the first callback. The
// jobs path also pre-declares buildPath so build.sh writes the
// output straight onto the mounted builds volume.
$buildFields = ['status' => 'waiting'];
if ($useJobs) {
$buildFields['buildPath'] = Job::buildPath($project->getId(), $deploymentId);
}
if ($deployment->isEmpty()) {
$deployment = $dbForProject->createDocument('deployments', new Document([
'$id' => $deploymentId,
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'resourceInternalId' => $function->getSequence(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'entrypoint' => $entrypoint,
'buildCommands' => $commands,
'startCommand' => $function->getAttribute('startCommand', ''),
'sourcePath' => $path,
'sourceSize' => $fileSize,
'totalSize' => $fileSize,
'sourceChunksTotal' => $chunks,
'sourceChunksUploaded' => $chunksUploaded,
'activate' => $activate,
'sourceMetadata' => $metadata,
'type' => $type,
...$buildFields,
]));
} else {
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
'sourceSize' => $fileSize,
'sourceChunksUploaded' => $chunksUploaded,
'sourceMetadata' => $metadata,
...$buildFields,
]));
}
if ($useJobs) {
$jobs->create(...Job::build($project, $function, $deployment, $platform));
} else {
// Default: build on the executor via the Builds worker.
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $function,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
platform: $platform,
));
}
$deployment = $deployments->createFromUpload($function, $deployment->setAttributes([
'$id' => $deploymentId,
'entrypoint' => $entrypoint,
'buildCommands' => $commands,
'startCommand' => $function->getAttribute('startCommand', ''),
'sourcePath' => $path,
'sourceSize' => $fileSize,
'totalSize' => $fileSize,
'sourceChunksTotal' => $chunks,
'sourceChunksUploaded' => $chunksUploaded,
'activate' => $activate,
'sourceMetadata' => $metadata,
'type' => $type,
]));
} else {
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
$deployment = $deployments->upload($function, $deployment->setAttributes([
'sourceChunksUploaded' => $chunksUploaded,
'sourceMetadata' => $metadata,
]));
@@ -2,24 +2,19 @@
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Duplicate;
use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Extend\Exception;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Service\Deployments;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\UID;
use Utopia\Platform\Action;
use Utopia\Platform\Scope\HTTP;
use Utopia\Storage\Device;
use Utopia\System\System;
class Create extends Action
{
@@ -66,11 +61,8 @@ class Create extends Action
->inject('response')
->inject('dbForProject')
->inject('queueForEvents')
->inject('publisherForBuilds')
->inject('jobs')
->inject('deployments')
->inject('deviceForFunctions')
->inject('project')
->inject('platform')
->callback($this->action(...));
}
@@ -81,11 +73,8 @@ class Create extends Action
Response $response,
Database $dbForProject,
Event $queueForEvents,
BuildPublisher $publisherForBuilds,
Jobs $jobs,
Deployments $deployments,
Device $deviceForFunctions,
Document $project,
array $platform
) {
$function = $dbForProject->getDocument('functions', $functionId);
@@ -105,17 +94,14 @@ class Create extends Action
$deploymentId = ID::unique();
// Build backend for the rebuild: 'orchestrator' (jobs-service, submitted
// here) or 'executor' (default; Builds worker). The jobs path pre-declares
// buildPath so build.sh writes output straight onto the mounted volume.
$useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
$destination = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$deviceForFunctions->transfer($path, $destination, $deviceForFunctions);
// Cloning the source deployment's attributes onto the new one, with
// its own $id and no $sequence, tells the service to create it fresh
// rather than update the deployment being duplicated.
$deployment->removeAttribute('$sequence');
$deployment = $dbForProject->createDocument('deployments', $deployment->setAttributes([
$deployment->setAttributes([
'$id' => $deploymentId,
'sourcePath' => $destination,
'totalSize' => $deployment->getAttribute('sourceSize', 0),
@@ -126,22 +112,11 @@ class Create extends Action
'buildEndedAt' => null,
'buildDuration' => null,
'buildSize' => null,
'status' => 'waiting',
'buildPath' => $useJobs ? Job::buildPath($project->getId(), $deploymentId) : '',
'buildPath' => '',
'buildLogs' => '',
]));
]);
if ($useJobs) {
$jobs->create(...Job::build($project, $function, $deployment, $platform));
} else {
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $function,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
platform: $platform,
));
}
$deployment = $deployments->createFromUpload($function, $deployment);
$queueForEvents
->setParam('functionId', $function->getId())
@@ -2,15 +2,13 @@
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Status;
use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Extend\Exception;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Service\Deployments;
use Appwrite\Utopia\Response;
use Executor\Executor;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
@@ -60,10 +58,8 @@ class Update extends Action
->param('deploymentId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Deployment ID.', false, ['dbForProject'])
->inject('response')
->inject('dbForProject')
->inject('project')
->inject('queueForEvents')
->inject('executor')
->inject('jobs')
->inject('deployments')
->callback($this->action(...));
}
@@ -72,10 +68,8 @@ class Update extends Action
string $deploymentId,
Response $response,
Database $dbForProject,
Document $project,
Event $queueForEvents,
Executor $executor,
Jobs $jobs,
Deployments $deployments,
) {
$function = $dbForProject->getDocument('functions', $functionId);
@@ -115,15 +109,9 @@ class Update extends Action
}
}
// Best-effort cleanup on both backends — the deployment is already
// marked 'canceled', and only one backend actually holds the build.
// Best-effort cleanup — the deployment is already marked 'canceled'.
try {
$executor->deleteRuntime($project->getId(), $deploymentId . "-build");
} catch (\Throwable) {
}
try {
$jobs->delete(Job::id($project->getId(), $deploymentId));
$deployments->delete($deploymentId);
} catch (\Throwable) {
}
@@ -2,9 +2,7 @@
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Template;
use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Extend\Exception;
use Appwrite\Platform\Action;
@@ -12,19 +10,16 @@ use Appwrite\Platform\Modules\Compute\Base;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Service\Deployments;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
use Utopia\Http\Adapter\Swoole\Request;
use Utopia\Platform\Enum;
use Utopia\Platform\Scope\HTTP;
use Utopia\System\System;
use Utopia\Validator\Boolean;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
@@ -83,7 +78,7 @@ class Create extends Base
->inject('queueForEvents')
->inject('project')
->inject('publisherForBuilds')
->inject('jobs')
->inject('deployments')
->inject('gitHub')
->inject('authorization')
->inject('platform')
@@ -105,7 +100,7 @@ class Create extends Base
Event $queueForEvents,
Document $project,
BuildPublisher $publisherForBuilds,
Jobs $jobs,
Deployments $deployments,
GitHub $github,
Authorization $authorization,
array $platform
@@ -145,6 +140,7 @@ class Create extends Base
template: $template,
github: $github,
activate: $activate,
deployments: $deployments,
platform: $platform,
referenceType: $type,
reference: $reference
@@ -161,70 +157,48 @@ class Create extends Base
return;
}
// Backend for the template build: 'orchestrator' (jobs-service, which
// pulls the public GitHub tarball via artifacts) or 'executor' (default;
// the Builds worker clones the repo). The jobs path pre-declares buildPath.
$useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
$deploymentId = ID::unique();
$deployment = $dbForProject->createDocument('deployments', new Document([
'$id' => $deploymentId,
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'resourceId' => $function->getId(),
'resourceInternalId' => $function->getSequence(),
'resourceType' => 'functions',
'entrypoint' => $function->getAttribute('entrypoint', ''),
'buildCommands' => $function->getAttribute('commands', ''),
'startCommand' => $function->getAttribute('startCommand', ''),
'providerRepositoryName' => $repository,
'providerRepositoryOwner' => $owner,
'providerRepositoryUrl' => $repositoryUrl,
'providerBranchUrl' => $branchUrl,
'providerBranch' => $type == GitHub::CLONE_TYPE_BRANCH ? $reference : '',
'type' => 'vcs',
'activate' => $activate,
'status' => 'waiting',
'buildPath' => $useJobs ? Job::buildPath($project->getId(), $deploymentId) : '',
]));
// Templates can pin a version range (e.g. "0.3.*"); codeload only
// takes a concrete ref, so resolve the range to the highest matching
// tag (mirrors the executor's `git ls-remote --tags | tail -1`).
$ref = $reference;
if ($type === GitHub::CLONE_TYPE_TAG && \str_contains($reference, '*')) {
try {
$tags = $github->listTags($owner, $repository, $reference);
$ref = \end($tags) ?: $reference;
} catch (\Throwable) {
// Fall back to the raw reference; the build surfaces a bad ref.
}
}
// Public template: pull the source straight from GitHub's public repo
// (codeload tarball on the jobs backend, a plain git clone on the
// executor); unarchive/checkout strips down to the rootDirectory.
$deployment = $deployments->createFromRef(
$function,
new Document([
'$id' => $deploymentId,
'entrypoint' => $function->getAttribute('entrypoint', ''),
'buildCommands' => $function->getAttribute('commands', ''),
'startCommand' => $function->getAttribute('startCommand', ''),
'providerRepositoryName' => $repository,
'providerRepositoryOwner' => $owner,
'providerRepositoryUrl' => $repositoryUrl,
'providerBranchUrl' => $branchUrl,
'providerBranch' => $type == GitHub::CLONE_TYPE_BRANCH ? $reference : '',
'type' => 'vcs',
'activate' => $activate,
]),
$owner,
$repository,
$type,
$ref,
$rootDirectory,
);
$this->updateEmptyManualRule($project, $function, $deployment, $dbForPlatform, $authorization);
if ($useJobs) {
// Templates can pin a version range (e.g. "0.3.*"); codeload only
// takes a concrete ref, so resolve the range to the highest matching
// tag (mirrors the executor's `git ls-remote --tags | tail -1`).
$ref = $reference;
if ($type === GitHub::CLONE_TYPE_TAG && \str_contains($reference, '*')) {
try {
$tags = $github->listTags($owner, $repository, $reference);
$ref = \end($tags) ?: $reference;
} catch (\Throwable) {
// Fall back to the raw reference; the build surfaces a bad ref.
}
}
// Public template: pull the source straight from GitHub's codeload
// tarball; unarchive strips the "{repo}-{ref}/" wrapper + the rootDirectory.
$source = [
'url' => "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$ref}",
'subdir' => $rootDirectory,
];
$jobs->create(...Job::build($project, $function, $deployment, $platform, $source));
} else {
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $function,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
template: $template,
platform: $platform,
));
}
$queueForEvents
->setParam('functionId', $function->getId())
->setParam('deploymentId', $deployment->getId());
@@ -10,8 +10,8 @@ use Appwrite\Platform\Modules\Compute\Base;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Service\Deployments;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Validator\UID;
@@ -74,7 +74,7 @@ class Create extends Base
->inject('project')
->inject('queueForEvents')
->inject('publisherForBuilds')
->inject('jobs')
->inject('deployments')
->inject('gitHub')
->inject('platform')
->callback($this->action(...));
@@ -92,7 +92,7 @@ class Create extends Base
Document $project,
Event $queueForEvents,
BuildPublisher $publisherForBuilds,
Jobs $jobs,
Deployments $deployments,
GitHub $github,
array $platform,
) {
@@ -123,7 +123,7 @@ class Create extends Base
platform: $platform,
reference: $reference,
referenceType: $type,
jobs: $jobs
deployments: $deployments
);
$queueForEvents
@@ -16,11 +16,11 @@ use Appwrite\Platform\Modules\Compute\Validator\Specification;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Service\Deployments;
use Appwrite\Task\Validator\Cron;
use Appwrite\Utopia\Database\Validator\CustomId;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model\Rule;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Abuse\Abuse;
use Utopia\Config\Config;
use Utopia\Database\Database;
@@ -124,7 +124,7 @@ class Create extends Base
->inject('project')
->inject('queueForEvents')
->inject('publisherForBuilds')
->inject('jobs')
->inject('deployments')
->inject('queueForRealtime')
->inject('queueForWebhooks')
->inject('publisherForFunctions')
@@ -169,7 +169,7 @@ class Create extends Base
Document $project,
Event $queueForEvents,
BuildPublisher $publisherForBuilds,
Jobs $jobs,
Deployments $deployments,
Realtime $queueForRealtime,
Webhook $queueForWebhooks,
FunctionPublisher $publisherForFunctions,
@@ -346,7 +346,7 @@ class Create extends Base
installation: $installation,
dbForProject: $dbForProject,
publisherForBuilds: $publisherForBuilds,
jobs: $jobs,
deployments: $deployments,
template: $template,
github: $github,
activate: true,
@@ -11,10 +11,10 @@ use Appwrite\Platform\Modules\Compute\Validator\Specification;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Service\Deployments;
use Appwrite\Task\Validator\Cron;
use Appwrite\Utopia\Response;
use Executor\Executor;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
@@ -112,7 +112,7 @@ class Update extends Base
->inject('project')
->inject('queueForEvents')
->inject('publisherForBuilds')
->inject('jobs')
->inject('deployments')
->inject('dbForPlatform')
->inject('gitHub')
->inject('executor')
@@ -150,7 +150,7 @@ class Update extends Base
Document $project,
Event $queueForEvents,
BuildPublisher $publisherForBuilds,
Jobs $jobs,
Deployments $deployments,
Database $dbForPlatform,
GitHub $github,
Executor $executor,
@@ -312,7 +312,7 @@ class Update extends Base
// Redeploy logic
if (!$isConnected && !empty($providerRepositoryId)) {
$this->redeployVcsFunction($request, $function, $project, $installation, $dbForProject, $publisherForBuilds, new Document(), $github, true, $platform, jobs: $jobs);
$this->redeployVcsFunction($request, $function, $project, $installation, $dbForProject, $publisherForBuilds, new Document(), $github, true, $platform, deployments: $deployments);
}
// Inform scheduler if function is still active
@@ -3,7 +3,6 @@
namespace Appwrite\Platform\Modules\Functions\Workers;
use Ahc\Jwt\JWT;
use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Func as FunctionMessage;
use Appwrite\Event\Publisher\Func as FunctionPublisher;
@@ -13,6 +12,7 @@ use Appwrite\Event\Realtime;
use Appwrite\Event\Webhook;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Filter\BranchDomain as BranchDomainFilter;
use Appwrite\Service\Deployments\Jobs as DeploymentsJobs;
use Appwrite\Usage\Build as BuildUsage;
use Appwrite\Usage\Context;
use Appwrite\Utopia\Response\Model\Deployment;
@@ -533,11 +533,11 @@ class Builds extends Action
if ($resource->getCollection() === 'functions'
&& System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') {
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
'buildPath' => Job::buildPath($project->getId(), $deploymentId),
'buildPath' => DeploymentsJobs::buildPath($project->getId(), $deploymentId),
]));
$ref = $deployment->getAttribute('providerCommitHash') ?: $branchName;
$jobs->create(...Job::build($project, $resource, $deployment, $platform, [
$jobs->create(...DeploymentsJobs::payload($project, $resource, $deployment, $platform, [
'url' => $github->getRepositoryPresignedUrl($cloneOwner, $cloneRepository, $ref),
'subdir' => $resource->getAttribute('providerRootDirectory', ''),
]));
@@ -2,7 +2,6 @@
namespace Appwrite\Platform\Modules\Functions\Workers;
use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Func as FunctionMessage;
use Appwrite\Event\Message\Jobs as JobsMessage;
@@ -10,6 +9,7 @@ use Appwrite\Event\Publisher\Func as FunctionPublisher;
use Appwrite\Event\Publisher\Usage as UsagePublisher;
use Appwrite\Event\Realtime;
use Appwrite\Event\Webhook;
use Appwrite\Service\Deployments\Jobs as DeploymentsJobs;
use Appwrite\Usage\Build as BuildUsage;
use Appwrite\Usage\Context as UsageContext;
use Appwrite\Utopia\Response\Model\Deployment;
@@ -209,7 +209,7 @@ class Jobs extends Action
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $github);
}
$path = Job::buildPath($project->getId(), $deployment->getId());
$path = DeploymentsJobs::buildPath($project->getId(), $deployment->getId());
$size = $deviceForBuilds->exists($path) ? $deviceForBuilds->getFileSize($path) : 0;
$limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000');
@@ -3,21 +3,17 @@
namespace Appwrite\Platform\Modules\Sites\Http\Deployments;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Extend\Exception;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\ContentType;
use Appwrite\SDK\Method;
use Appwrite\SDK\MethodType;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Service\Deployments;
use Appwrite\Utopia\Response;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
use Utopia\Http\Adapter\Swoole\Request;
@@ -88,7 +84,7 @@ class Create extends Action
->inject('queueForEvents')
->inject('deviceForSites')
->inject('deviceForLocal')
->inject('publisherForBuilds')
->inject('deployments')
->inject('plan')
->inject('authorization')
->inject('platform')
@@ -111,7 +107,7 @@ class Create extends Action
Event $queueForEvents,
Device $deviceForSites,
Device $deviceForLocal,
BuildPublisher $publisherForBuilds,
Deployments $deployments,
array $plan,
Authorization $authorization,
array $platform,
@@ -231,7 +227,7 @@ class Create extends Action
}
try {
$locks($lockKey, 600, function () use ($activate, $authorization, &$chunks, $commands, $contentRange, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $outputDirectory, $path, $platform, $project, &$site, $type, &$completed, $response): void {
$locks($lockKey, 600, function () use ($activate, $authorization, &$chunks, $commands, $contentRange, $dbForPlatform, $dbForProject, $deploymentId, $deployments, $deviceForSites, $fileSize, &$metadata, $outputDirectory, $path, $platform, $project, &$site, $type, &$completed, $response): void {
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
if (!$deployment->isEmpty()) {
@@ -253,16 +249,8 @@ class Create extends Action
$deviceForSites->prepareUpload($path, $metadata['content_type'] ?? '', $chunks, $metadata);
if (!empty($contentRange)) {
$deployment = $dbForProject->createDocument('deployments', new Document([
$deployment = $deployments->upload($site, $deployment->setAttributes([
'$id' => $deploymentId,
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'resourceInternalId' => $site->getSequence(),
'resourceId' => $site->getId(),
'resourceType' => 'sites',
'buildCommands' => \implode(' && ', $commands),
'startCommand' => $site->getAttribute('startCommand', ''),
'buildOutput' => $outputDirectory,
@@ -325,7 +313,7 @@ class Create extends Action
}
try {
$locks($lockKey, 600, function () use ($activate, $authorization, $commands, &$chunks, $chunksUploaded, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $mergeUploadMetadata, $outputDirectory, $path, $platform, $project, $publisherForBuilds, $queueForEvents, $response, &$site, $siteId, $type): void {
$locks($lockKey, 600, function () use ($activate, $authorization, $commands, &$chunks, $chunksUploaded, $dbForPlatform, $dbForProject, $deploymentId, $deployments, $deviceForSites, $fileSize, &$metadata, $mergeUploadMetadata, $outputDirectory, $path, $platform, $project, $queueForEvents, $response, &$site, $type): void {
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$uploaded = 0;
@@ -349,47 +337,28 @@ class Create extends Action
if ($chunksUploaded === $chunks && $uploaded < $chunks) {
$deviceForSites->finalizeUpload($path, $chunks, $metadata);
if ($activate) {
// Remove deploy for all other deployments.
$activeDeployments = $dbForProject->find('deployments', [
Query::equal('activate', [true]),
Query::equal('resourceId', [$siteId]),
Query::equal('resourceType', ['sites'])
]);
foreach ($activeDeployments as $activeDeployment) {
$dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document(['activate' => false]));
}
}
$fileSize = $deviceForSites->getFileSize($path);
$isNewDeployment = $deployment->isEmpty();
if ($deployment->isEmpty()) {
$deployment = $dbForProject->createDocument('deployments', new Document([
'$id' => $deploymentId,
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'resourceInternalId' => $site->getSequence(),
'resourceId' => $site->getId(),
'resourceType' => 'sites',
'buildCommands' => \implode(' && ', $commands),
'startCommand' => $site->getAttribute('startCommand', ''),
'buildOutput' => $outputDirectory,
'adapter' => $site->getAttribute('adapter', ''),
'fallbackFile' => $site->getAttribute('fallbackFile', ''),
'sourcePath' => $path,
'sourceSize' => $fileSize,
'totalSize' => $fileSize,
'sourceChunksTotal' => $chunks,
'sourceChunksUploaded' => $chunksUploaded,
'activate' => $activate,
'sourceMetadata' => $metadata,
'type' => $type,
]));
// Start the build
$deployment = $deployments->createFromUpload($site, $deployment->setAttributes([
'$id' => $deploymentId,
'buildCommands' => \implode(' && ', $commands),
'startCommand' => $site->getAttribute('startCommand', ''),
'buildOutput' => $outputDirectory,
'adapter' => $site->getAttribute('adapter', ''),
'fallbackFile' => $site->getAttribute('fallbackFile', ''),
'sourcePath' => $path,
'sourceSize' => $fileSize,
'totalSize' => $fileSize,
'sourceChunksTotal' => $chunks,
'sourceChunksUploaded' => $chunksUploaded,
'activate' => $activate,
'sourceMetadata' => $metadata,
'type' => $type,
]));
if ($isNewDeployment) {
$sitesDomain = $platform['sitesDomain'];
$domain = ID::unique() . "." . $sitesDomain;
@@ -405,8 +374,8 @@ class Create extends Action
'domain' => $domain,
'type' => 'deployment',
'trigger' => 'deployment',
'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(),
'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(),
'deploymentId' => $deployment->getId(),
'deploymentInternalId' => $deployment->getSequence(),
'deploymentResourceType' => 'site',
'deploymentResourceId' => $site->getId(),
'deploymentResourceInternalId' => $site->getSequence(),
@@ -417,22 +386,7 @@ class Create extends Action
'region' => $project->getAttribute('region')
]))
);
} else {
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
'sourceSize' => $fileSize,
'sourceChunksUploaded' => $chunksUploaded,
'sourceMetadata' => $metadata,
]));
}
// Start the build
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $site,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
platform: $platform,
));
} else {
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
'sourceChunksUploaded' => $chunksUploaded,
@@ -9,8 +9,8 @@ use Appwrite\Platform\Modules\VCS\Http\GitHub\Deployment;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Service\Deployments;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
@@ -62,7 +62,7 @@ class Update extends Action
->inject('authorization')
->inject('getProjectDB')
->inject('publisherForBuilds')
->inject('jobs')
->inject('deployments')
->inject('platform')
->callback($this->action(...));
}
@@ -78,7 +78,7 @@ class Update extends Action
Authorization $authorization,
callable $getProjectDB,
BuildPublisher $publisherForBuilds,
Jobs $jobs,
Deployments $deployments,
array $platform
) {
$installation = $dbForPlatform->getDocument('installations', $installationId);
@@ -140,7 +140,7 @@ class Update extends Action
...array_filter(array_column($prFiles, 'previous_filename'))
];
$this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, true, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs);
$this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, true, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments);
$response->noContent();
}
@@ -2,14 +2,12 @@
namespace Appwrite\Platform\Modules\VCS\Http\GitHub;
use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Extend\Exception;
use Appwrite\Filter\BranchDomain as BranchDomainFilter;
use Appwrite\Service\Deployments;
use Appwrite\Vcs\Comment;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Config\Config;
use Utopia\Console;
use Utopia\Database\Database;
@@ -52,7 +50,7 @@ trait Deployment
BuildPublisher $publisherForBuilds,
callable $getProjectDB,
array $platform,
?Jobs $jobs = null,
?Deployments $deployments = null,
) {
$errors = [];
foreach ($repositories as $repository) {
@@ -363,25 +361,8 @@ trait Deployment
$commands[] = $resource->getAttribute('commands', '');
}
// Build function deployments on the jobs-service when the caller
// opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. Sites stay
// on the executor. On jobs the deployment is pre-declared 'waiting'
// with its buildPath so build.sh writes onto the mounted volume.
$useJobs = $jobs !== null
&& $resourceCollection === 'functions'
&& System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
$buildFields = $useJobs
? ['status' => 'waiting', 'buildPath' => Job::buildPath($projectId, $deploymentId)]
: [];
$deployment = $authorization->skip(fn () => $dbForProject->createDocument('deployments', new Document([
$deployment = new Document([
'$id' => $deploymentId,
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
...$buildFields,
'resourceId' => $resourceId,
'resourceInternalId' => $resourceInternalId,
'resourceType' => $resourceCollection,
@@ -409,7 +390,37 @@ trait Deployment
'providerCommentId' => \strval($latestCommentId),
'providerBranch' => $providerBranch,
'activate' => $activate,
])));
]);
// Build function deployments through $deployments (executor
// or jobs-service, decided by _APP_BUILDS_BACKEND) when the
// caller opts in. Sites always stay on the executor.
if ($deployments !== null && $resourceCollection === 'functions') {
$deployment = $authorization->skip(fn () => $deployments->createFromUrl(
$resource,
$deployment,
$github->getRepositoryPresignedUrl($providerRepositoryOwner, $providerRepositoryName, $providerCommitHash),
$resource->getAttribute('providerRootDirectory', ''),
));
} else {
$deployment = $authorization->skip(fn () => $dbForProject->createDocument('deployments', new Document([
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
...$deployment->getArrayCopy(),
'status' => 'waiting',
])));
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $resource,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
platform: $platform,
));
}
if ($resource->getCollection() === 'sites') {
$projectId = $project->getId();
@@ -572,27 +583,6 @@ trait Deployment
$github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name);
}
if ($useJobs) {
$source = [
'url' => $github->getRepositoryPresignedUrl($providerRepositoryOwner, $providerRepositoryName, $providerCommitHash),
'subdir' => $resource->getAttribute('providerRootDirectory', ''),
];
$jobs->create(...Job::build($project, $resource, $deployment, $platform, $source));
} else {
$queueName = $this->getBuildQueueName($project, $dbForPlatform, $authorization);
$publisherForBuilds->enqueue(
new BuildMessage(
project: $project,
resource: $resource,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
platform: $platform,
),
new \Utopia\Queue\Queue($queueName)
);
}
Span::add("{$logBase}.build.triggered", 'true');
//TODO: Add event?
} catch (Exception $e) {
@@ -618,9 +608,4 @@ trait Deployment
{
}
protected function getBuildQueueName(Document $project, Database $dbForPlatform, Authorization $authorization): string
{
return System::getEnv('_APP_BUILDS_QUEUE_NAME', Event::BUILDS_QUEUE_NAME);
}
}
@@ -6,9 +6,9 @@ use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Extend\Exception;
use Appwrite\Platform\Action;
use Appwrite\Platform\Modules\VCS\Http\GitHub\Deployment;
use Appwrite\Service\Deployments;
use Appwrite\Utopia\Request;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
@@ -44,7 +44,7 @@ class Create extends Action
->inject('authorization')
->inject('getProjectDB')
->inject('publisherForBuilds')
->inject('jobs')
->inject('deployments')
->inject('platform')
->callback($this->action(...));
}
@@ -57,7 +57,7 @@ class Create extends Action
Authorization $authorization,
callable $getProjectDB,
BuildPublisher $publisherForBuilds,
Jobs $jobs,
Deployments $deployments,
array $platform
) {
$this->preprocessEvent($request);
@@ -82,8 +82,8 @@ class Create extends Action
match ($event) {
$github::EVENT_INSTALLATION => $this->handleInstallationEvent($parsedPayload, $dbForPlatform, $authorization, $getProjectDB),
$github::EVENT_PUSH => $this->handlePushEvent($parsedPayload, $githubAppId, $privateKey, $github, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs),
$github::EVENT_PULL_REQUEST => $this->handlePullRequestEvent($parsedPayload, $privateKey, $githubAppId, $github, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs),
$github::EVENT_PUSH => $this->handlePushEvent($parsedPayload, $githubAppId, $privateKey, $github, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments),
$github::EVENT_PULL_REQUEST => $this->handlePullRequestEvent($parsedPayload, $privateKey, $githubAppId, $github, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments),
default => null,
};
@@ -190,7 +190,7 @@ class Create extends Action
BuildPublisher $publisherForBuilds,
callable $getProjectDB,
array $platform,
?Jobs $jobs = null,
?Deployments $deployments = null,
) {
$providerBranchDeleted = $parsedPayload["branchDeleted"] ?? false;
$providerBranch = $parsedPayload["branch"] ?? '';
@@ -223,7 +223,7 @@ class Create extends Action
// Create new deployment only on push (not committed by us) and not when branch is deleted
if ($providerCommitAuthorEmail !== APP_VCS_GITHUB_EMAIL && !$providerBranchDeleted) {
$providerAffectedFiles = $parsedPayload['affectedFiles'] ?? [];
$this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', $providerAffectedFiles, false, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs);
$this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', $providerAffectedFiles, false, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments);
}
}
@@ -237,7 +237,7 @@ class Create extends Action
BuildPublisher $publisherForBuilds,
callable $getProjectDB,
array $platform,
?Jobs $jobs = null,
?Deployments $deployments = null,
) {
$action = $parsedPayload["action"] ?? '';
@@ -288,7 +288,7 @@ class Create extends Action
Query::orderDesc('$createdAt')
]));
$this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, $external, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs);
$this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, $external, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments);
} elseif ($action == "closed") {
// Allowed external contributions cleanup
+143
View File
@@ -0,0 +1,143 @@
<?php
namespace Appwrite\Service;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
/**
* Owns a deployment's lifecycle: upload bookkeeping, creating it and
* dispatching the build, and canceling one in flight. The backend
* (open-runtimes jobs-service vs the executor via the Builds worker) is
* fixed for the request's lifetime by which implementation is wired up in
* app/init/resources/request.php.
*/
abstract readonly class Deployments
{
public function __construct(
protected Database $dbForProject,
protected Document $project,
) {
}
/**
* Saves chunked-upload progress onto the deployment — source path/size,
* chunk counters, metadata. Never triggers a build; call create() once
* the upload is complete. Pass a single `Document` carrying every field
* the deployment should end up with: either a fresh, not-yet-persisted
* one (a plain `new Document([...])`), or the existing one fetched from
* the database with more attributes set via `setAttributes()`. A new
* document (one with no $sequence, i.e. not yet persisted — a document
* only gets one from the database, never from `setAttributes()`) gets
* the standard $permissions and resourceId/resourceInternalId/
* resourceType merged in automatically.
*/
public function upload(Document $resource, Document $deployment): Document
{
if ($deployment->getSequence() === null) {
return $this->dbForProject->createDocument('deployments', new Document([
'$permissions' => self::permissions(),
...self::resourceFields($resource),
...$deployment->getArrayCopy(),
]));
}
return $this->dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
}
/**
* Finalizes the deployment document (see upload() for what `$deployment`
* should carry) and dispatches it for building from its own uploaded
* source. Deactivates any other active deployment for $resource, marks
* it queued (and writes whatever else the backend needs, e.g.
* buildPath). Returns the persisted, updated deployment.
*/
abstract public function createFromUpload(Document $resource, Document $deployment): Document;
/**
* Same as createFromUpload(), but builds from a public git reference
* (a template's repository) instead of the deployment's own uploaded
* source. $reference is already resolved to a concrete commit/branch/tag
* — resolving a version range (e.g. "0.3.*") is the caller's job, since
* only it holds the GitHub client that can do so.
*/
abstract public function createFromRef(
Document $resource,
Document $deployment,
string $owner,
string $repository,
string $type,
string $reference,
string $rootDirectory = '',
): Document;
/**
* Same as createFromUpload(), but builds from a remote tarball at $url
* (a VCS presigned URL) instead of the deployment's own uploaded source.
*/
abstract public function createFromUrl(
Document $resource,
Document $deployment,
string $url,
string $rootDirectory = '',
): Document;
/**
* Best-effort cancel of an in-flight build. The deployment is already
* marked canceled by the caller; this only needs to stop the backend from
* still writing to it.
*/
abstract public function delete(string $deploymentId): void;
/**
* Deactivates any other active deployment for $resource before this one
* goes live. Shared by both backends, called right before they mark the
* deployment queued.
*/
protected function deactivateOthers(Document $resource, Document $deployment): void
{
if (!$deployment->getAttribute('activate', false)) {
return;
}
$others = $this->dbForProject->find('deployments', [
Query::equal('activate', [true]),
Query::equal('resourceId', [$resource->getId()]),
Query::equal('resourceType', [$resource->getCollection()]),
Query::notEqual('$id', $deployment->getId()),
]);
foreach ($others as $other) {
$this->dbForProject->updateDocument('deployments', $other->getId(), new Document([
'activate' => false,
]));
}
}
/**
* @return array<string>
*/
private static function permissions(): array
{
return [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
];
}
/**
* @return array<string, mixed>
*/
private static function resourceFields(Document $resource): array
{
return [
'resourceInternalId' => $resource->getSequence(),
'resourceId' => $resource->getId(),
'resourceType' => $resource->getCollection(),
];
}
}
@@ -0,0 +1,86 @@
<?php
namespace Appwrite\Service\Deployments;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Service\Deployments;
use Executor\Executor as ExecutorClient;
use Utopia\Database\Database;
use Utopia\Database\Document;
readonly class Executor extends Deployments
{
public function __construct(
private BuildPublisher $publisherForBuilds,
Database $dbForProject,
Document $project,
private ExecutorClient $executor,
private array $platform,
) {
parent::__construct($dbForProject, $project);
}
public function createFromUpload(Document $resource, Document $deployment): Document
{
return $this->dispatch($resource, $deployment, null);
}
public function createFromRef(
Document $resource,
Document $deployment,
string $owner,
string $repository,
string $type,
string $reference,
string $rootDirectory = '',
): Document {
// No presigned URL to fetch here — the Builds worker clones the
// public repo itself via these raw git coordinates.
return $this->dispatch($resource, $deployment, new Document([
'ownerName' => $owner,
'repositoryName' => $repository,
'referenceType' => $type,
'referenceValue' => $reference,
'rootDirectory' => $rootDirectory,
]));
}
public function createFromUrl(
Document $resource,
Document $deployment,
string $url,
string $rootDirectory = '',
): Document {
// The Builds worker clones the repo via the deployment's own
// installationId/providerRepositoryId instead of $url.
return $this->dispatch($resource, $deployment, null);
}
private function dispatch(Document $resource, Document $deployment, ?Document $template): Document
{
$deployment = $this->upload($resource, $deployment);
$this->deactivateOthers($resource, $deployment);
// The Builds worker promotes 'waiting' → 'building' on pickup.
$deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
'status' => 'waiting',
]));
$this->publisherForBuilds->enqueue(new BuildMessage(
project: $this->project,
resource: $resource,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
template: $template,
platform: $this->platform,
));
return $deployment;
}
public function delete(string $deploymentId): void
{
$this->executor->deleteRuntime($this->project->getId(), $deploymentId . '-build');
}
}
@@ -1,45 +1,108 @@
<?php
namespace Appwrite\Compute;
namespace Appwrite\Service\Deployments;
use Ahc\Jwt\JWT;
use Appwrite\Deployment\Token;
use Appwrite\Service\Deployments;
use OpenRuntimes\Orchestrator\Enum\CallbackEvent;
use OpenRuntimes\Orchestrator\Jobs as JobsClient;
use OpenRuntimes\Orchestrator\Model\Artifact\DownloadArtifact;
use OpenRuntimes\Orchestrator\Model\Artifact\StatArtifact;
use OpenRuntimes\Orchestrator\Model\Artifact\UnarchiveArtifact;
use OpenRuntimes\Orchestrator\Model\Callback;
use OpenRuntimes\Orchestrator\Model\Volume;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\System\System;
/**
* Builds an open-runtimes jobs-service job payload for a function deployment.
* Builds and submits an open-runtimes jobs-service job for a deployment.
*
* Source crosses the boundary via the artifacts system (presigned GET download
* + unarchive, run by the sidecar) — a GET has no request-body cap, so large
* sources are fine. The build output and package-manager cache instead go on a
* mounted volume: the builds storage volume is attached to the build worker at
* its Appwrite path, so build.sh writes code.tar.gz + the cache squashfs
* straight onto the volume Appwrite already reads. That keeps the multi-hundred-MB
* output off the (capped) HTTP upload path and out of the Appwrite process.
* sources are fine. The build output and package-manager cache, by default,
* go on a mounted volume: the builds storage volume is attached to the build
* worker at its Appwrite path, so build.sh writes code.tar.gz + the cache
* squashfs straight onto the volume Appwrite already reads. That keeps the
* multi-hundred-MB output off the (capped) HTTP upload path and out of the
* Appwrite process. Deployments that need a different strategy (e.g. S3
* upload/download artifacts instead of a shared volume) override storage()
* — everything else about the payload stays the same.
*
* Covers function deployments whose source is a tarball: manual upload,
* duplicate/rebuild, and templates (public GitHub tarball via `$template`).
* duplicate/rebuild, and templates (public GitHub tarball resolved from a
* git reference).
*/
final class Job
readonly class Jobs extends Deployments
{
public function __construct(
private JobsClient $jobs,
Database $dbForProject,
Document $project,
private array $platform,
) {
parent::__construct($dbForProject, $project);
}
public function createFromUpload(Document $resource, Document $deployment): Document
{
return $this->submit($resource, $deployment, null);
}
public function createFromRef(
Document $resource,
Document $deployment,
string $owner,
string $repository,
string $type,
string $reference,
string $rootDirectory = '',
): Document {
// The jobs-service has no GitHub client of its own — it only fetches
// tarballs — so $reference must already be a concrete commit/branch/
// tag; codeload only understands one ref per tarball, not a range.
$url = "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$reference}";
return $this->submit($resource, $deployment, ['url' => $url, 'subdir' => $rootDirectory]);
}
public function createFromUrl(
Document $resource,
Document $deployment,
string $url,
string $rootDirectory = '',
): Document {
return $this->submit($resource, $deployment, ['url' => $url, 'subdir' => $rootDirectory]);
}
private function submit(Document $resource, Document $deployment, ?array $source): Document
{
$deployment = $this->upload($resource, $deployment);
$this->deactivateOthers($resource, $deployment);
// Pre-declare buildPath so build.sh writes output straight onto the
// mounted builds volume, at the path this deployment expects.
$deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
'status' => 'waiting',
'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()),
]));
$this->jobs->create(...static::payload($this->project, $resource, $deployment, $this->platform, $source));
return $deployment;
}
public function delete(string $deploymentId): void
{
$this->jobs->delete(static::id($this->project->getId(), $deploymentId));
}
/**
* @return array<string, mixed> Named arguments for OpenRuntimes\Orchestrator\Jobs::create().
*/
/**
* @param ?array{url: string, subdir?: string} $source Remote tarball source
* (template codeload URL, or a VCS presigned URL). When null, the source is
* the deployment's own uploaded tarball, fetched from Appwrite via a
* presigned token (manual upload / duplicate).
*/
public static function build(
public static function payload(
Document $project,
Document $function,
Document $deployment,
@@ -97,20 +160,18 @@ final class Job
];
}
// Output + cache live on the mounted builds volume (see class doc): the
// build writes code.tar.gz to the deployment's build dir and the cache
// squashfs to its per-function keyed path — both directly on the volume
// Appwrite reads, so nothing large crosses the HTTP boundary.
$cacheKey = self::cacheKey($projectId, $function->getId(), $runtime['image'] ?? '');
// Where output + cache land is a swappable strategy (see storage()) —
// the default mounts the shared builds volume; nothing else here cares
// which strategy is active.
$output = static::storage($project, $function, $deployment);
$command = $deployment->getAttribute('buildCommands', '');
$env = self::variables($project, $function, $deployment, $runtime, $cpus, $memory, $endpoint, $timeout) + [
'OPEN_RUNTIMES_BUILD_INPUT_DIR' => '/mnt/code/source',
'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => \dirname(self::buildPath($projectId, $deploymentId)),
'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => self::cachePath($projectId, $cacheKey),
];
] + $output['environment'];
return [
'id' => self::id($projectId, $deploymentId),
'id' => static::id($projectId, $deploymentId),
'image' => $runtime['image'],
'command' => '/usr/local/server/helpers/build.sh ' . \escapeshellarg($command),
'cpu' => $cpus,
@@ -125,13 +186,8 @@ final class Job
],
// The orchestrator expects environment as a string->string map.
'environment' => \array_map('strval', $env),
'artifacts' => $sourceArtifacts,
// Attach the builds storage volume (Docker volume / K8s PVC named by
// _APP_BUILDS_VOLUME) to the worker at its Appwrite path so build.sh
// writes output + cache straight onto it.
'volumes' => [
new Volume(source: System::getEnv('_APP_BUILDS_VOLUME', 'appwrite-builds'), path: APP_STORAGE_BUILDS),
],
'artifacts' => [...$sourceArtifacts, ...$output['artifacts']],
'volumes' => $output['volumes'],
'callback' => new Callback(
url: "{$endpoint}/v1/jobs/event?" . \http_build_query(['project' => $projectId]),
// Artifact callbacks only carry the source-size stat, which exists
@@ -176,6 +232,44 @@ final class Job
return APP_STORAGE_BUILDS . "/app-{$projectId}/cache/{$cacheKey}.sqfs";
}
/**
* Where build.sh's output (code.tar.gz) and package-manager cache
* (a squashfs) land, and what the job needs to get them there. The
* default mounts the shared builds volume at buildPath()/cachePath();
* build.sh only cares that OPEN_RUNTIMES_BUILD_OUTPUT_DIR/_CACHE_ARTIFACT
* point somewhere on its local filesystem, volume-backed or not — so a
* strategy without a shared volume (e.g. S3) instead points them at a
* local tmp path and moves things in/out via 'artifacts':
* - cache pull, before the build: a plain DownloadArtifact (no
* `depends`, so it runs before the command) into the local cache path.
* - cache push and output upload, after the build: an UploadArtifact
* with `depends: 'job'` — 'job' is the orchestrator's sentinel id for
* "after the build command finishes", not an id of another artifact.
*
* @return array{volumes: array<Volume>, artifacts: array<mixed>, environment: array<string, string>}
*/
protected static function storage(Document $project, Document $resource, Document $deployment): array
{
$projectId = $project->getId();
$deploymentId = $deployment->getId();
$runtime = self::runtime($resource, $resource->getAttribute('version', 'v2'));
$cacheKey = static::cacheKey($projectId, $resource->getId(), $runtime['image'] ?? '');
return [
// Docker volume / K8s PVC named by _APP_BUILDS_VOLUME, attached
// to the worker at its Appwrite path so build.sh writes output +
// cache straight onto it.
'volumes' => [
new Volume(source: System::getEnv('_APP_BUILDS_VOLUME', 'appwrite-builds'), path: APP_STORAGE_BUILDS),
],
'artifacts' => [],
'environment' => [
'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => \dirname(static::buildPath($projectId, $deploymentId)),
'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => static::cachePath($projectId, $cacheKey),
],
];
}
private static function runtime(Document $function, string $version): array
{
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);