mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge pull request #13028 from appwrite/chore/remove-executor-backend-from-builds
chore: remove the executor build backend
This commit is contained in:
30 files changed
+426
-1482
No files matched your search
@@ -117,7 +117,6 @@ _APP_SITES_TIMEOUT=30
|
||||
_APP_EXECUTOR_HOST=http://exc1/v1
|
||||
_APP_EXECUTOR_SECRET=your-secret-key
|
||||
_APP_EXECUTOR_IMAGES=openruntimes/node:v5-22,openruntimes/static:v5-1
|
||||
_APP_BUILDS_BACKEND=orchestrator
|
||||
_APP_BUILDS_VOLUME=appwrite-builds
|
||||
_APP_JOBS_HOST=http://orchestrator-jobs:8080
|
||||
_APP_JOBS_SECRET=your-secret-key
|
||||
|
||||
@@ -988,18 +988,9 @@ return [
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_BUILDS_BACKEND',
|
||||
'description' => 'Backend that builds manual-upload function deployments: "executor" (default; the open-runtimes executor, via the Builds worker) or "orchestrator" (the open-runtimes jobs-service, submitted in the request flow). Other build flows always use the executor.',
|
||||
'introduction' => '1.9.0',
|
||||
'default' => 'executor',
|
||||
'required' => false,
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_BUILDS_VOLUME',
|
||||
'description' => 'The Docker volume (or Kubernetes PersistentVolumeClaim) holding build storage, attached to jobs-service build workers so they write output directly onto it. Must match the storage the "builds" device is backed by. Only used when _APP_BUILDS_BACKEND is "orchestrator".',
|
||||
'description' => 'The Docker volume (or Kubernetes PersistentVolumeClaim) holding build storage, attached to jobs-service build workers so they write output directly onto it. Must match the storage the "builds" device is backed by.',
|
||||
'introduction' => '1.9.0',
|
||||
'default' => 'appwrite-builds',
|
||||
'required' => false,
|
||||
|
||||
@@ -77,9 +77,8 @@ $container->set('jobs', function () {
|
||||
->withBearerAuth(System::getEnv('_APP_JOBS_SECRET', ''))
|
||||
->withTimeout(30);
|
||||
|
||||
// No host on executor-only installs: keep the injection resolvable and
|
||||
// fail at call time instead (the client is only used when
|
||||
// _APP_BUILDS_BACKEND=orchestrator, which requires _APP_JOBS_HOST).
|
||||
// Keep the injection resolvable without _APP_JOBS_HOST and fail at call
|
||||
// time instead, so installs that never build stay bootable.
|
||||
$host = System::getEnv('_APP_JOBS_HOST', '');
|
||||
if ($host !== '') {
|
||||
$client = $client->withBaseUri($host);
|
||||
|
||||
@@ -5,13 +5,10 @@ use Ahc\Jwt\JWTException;
|
||||
use Appwrite\Auth\Key;
|
||||
use Appwrite\Database\Factory as DatabaseFactory;
|
||||
use Appwrite\Databases\TransactionState;
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Backend\Executor as ExecutorBackend;
|
||||
use Appwrite\Deployment\Backend\Orchestrator;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Context\Audit as AuditContext;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Func as FunctionMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
use Appwrite\Event\Realtime;
|
||||
use Appwrite\Event\Webhook;
|
||||
@@ -28,7 +25,6 @@ use Appwrite\Usage\Context as UsageContext;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Appwrite\Utopia\Request;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Executor\Executor;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Agents\Adapters\Appwrite as AppwriteAdapter;
|
||||
use Utopia\Agents\Agent;
|
||||
@@ -197,13 +193,11 @@ return function (Container $context): void {
|
||||
$publisher,
|
||||
new Queue(System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME), 'utopia-queue', Event::FUNCTIONS_QUEUE_TTL)
|
||||
), ['publisher']);
|
||||
// Builds a Backend bound to a given project — webhook handlers resolve
|
||||
// Builds a Deployments bound to a given project — webhook handlers resolve
|
||||
// their tenant projects mid-request, after this container is initialized.
|
||||
$context->set('deploymentsFactory', function (BuildPublisher $publisherForBuilds, Jobs $jobs, Executor $executor, array $platform) {
|
||||
return fn (Database $dbForProject, Document $project): Backend => System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'
|
||||
? new Orchestrator($jobs, $dbForProject, $project, $platform)
|
||||
: new ExecutorBackend($publisherForBuilds, $dbForProject, $project, $executor, $platform);
|
||||
}, ['publisherForBuilds', 'jobs', 'executor', 'platform']);
|
||||
$context->set('deploymentsFactory', function (Jobs $jobs, array $platform) {
|
||||
return fn (Database $dbForProject, Document $project): Deployments => new Deployments($jobs, $dbForProject, $project, $platform);
|
||||
}, ['jobs', 'platform']);
|
||||
$context->set('deployments', fn (callable $deploymentsFactory, Database $dbForProject, Document $project) => $deploymentsFactory($dbForProject, $project), ['deploymentsFactory', 'dbForProject', 'project']);
|
||||
$context->set('eventProcessor', fn () => new EventProcessor(), []);
|
||||
$context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<?php
|
||||
|
||||
use Appwrite\Database\Factory as DatabaseFactory;
|
||||
use Appwrite\Deployment\Backend\Orchestrator;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
use Appwrite\Event\Publisher\Notification as NotificationPublisher;
|
||||
@@ -189,9 +189,9 @@ return function (Container $container): void {
|
||||
}, ['project', 'telemetry']);
|
||||
|
||||
// Only the Builds worker uses this, handing template-into-repo pushes to
|
||||
// the jobs-service when _APP_BUILDS_BACKEND=orchestrator — no backend switch.
|
||||
// the jobs-service.
|
||||
$container->set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform) {
|
||||
return new Orchestrator($jobs, $dbForProject, $project, $platform);
|
||||
return new Deployments($jobs, $dbForProject, $project, $platform);
|
||||
}, ['jobs', 'dbForProject', 'project', 'platform']);
|
||||
|
||||
$container->set('logError', function (Registry $register, Document $project) {
|
||||
|
||||
@@ -171,7 +171,6 @@ services:
|
||||
- _APP_DOMAIN_SITES
|
||||
- _APP_EXECUTOR_SECRET
|
||||
- _APP_EXECUTOR_HOST
|
||||
- _APP_BUILDS_BACKEND
|
||||
- _APP_BUILDS_VOLUME
|
||||
- _APP_JOBS_HOST
|
||||
- _APP_JOBS_SECRET
|
||||
@@ -538,7 +537,6 @@ services:
|
||||
- _APP_STORAGE_WASABI_BUCKET
|
||||
- _APP_DATABASE_SHARED_TABLES
|
||||
- _APP_DOMAIN_SITES
|
||||
- _APP_BUILDS_BACKEND
|
||||
- _APP_BUILDS_VOLUME
|
||||
- _APP_JOBS_HOST
|
||||
- _APP_JOBS_SECRET
|
||||
|
||||
@@ -1,164 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Deployment;
|
||||
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\Permission;
|
||||
use Utopia\Database\Helpers\Role;
|
||||
use Utopia\Database\Query;
|
||||
|
||||
/**
|
||||
* Owns a deployment's lifecycle: upload bookkeeping, creating it and
|
||||
* dispatching the build, and canceling one in flight. The backend
|
||||
* (open-runtimes jobs-service vs the executor via the Builds worker) is
|
||||
* fixed for the request's lifetime by which implementation is wired up in
|
||||
* app/init/resources/request.php.
|
||||
*/
|
||||
abstract readonly class Backend
|
||||
{
|
||||
public function __construct(
|
||||
protected Database $dbForProject,
|
||||
protected Document $project,
|
||||
) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves chunked-upload progress onto the deployment — source path/size,
|
||||
* chunk counters, metadata. Never triggers a build; call createFromUpload()
|
||||
* once the upload is complete. Pass a single `Document` carrying every field
|
||||
* the deployment should end up with: either a fresh, not-yet-persisted
|
||||
* one (a plain `new Document([...])`), or the existing one fetched from
|
||||
* the database with more attributes set via `setAttributes()`. A new
|
||||
* document (one with no $sequence, i.e. not yet persisted — a document
|
||||
* only gets one from the database, never from `setAttributes()`) gets
|
||||
* the standard $permissions and resourceId/resourceInternalId/
|
||||
* resourceType merged in automatically.
|
||||
*/
|
||||
public function upload(Document $resource, Document $deployment): Document
|
||||
{
|
||||
if ($deployment->getSequence() === null) {
|
||||
return $this->dbForProject->createDocument('deployments', new Document([
|
||||
'$permissions' => self::permissions(),
|
||||
...self::resourceFields($resource),
|
||||
...$deployment->getArrayCopy(),
|
||||
]));
|
||||
}
|
||||
|
||||
return $this->dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
|
||||
}
|
||||
|
||||
/**
|
||||
* Finalizes the deployment document (see upload() for what `$deployment`
|
||||
* should carry) and dispatches it for building from its own uploaded
|
||||
* source. Deactivates any other active deployment for $resource, marks
|
||||
* it queued (and writes whatever else the backend needs, e.g.
|
||||
* buildPath). Returns the persisted, updated deployment.
|
||||
*/
|
||||
abstract public function createFromUpload(Document $resource, Document $deployment): Document;
|
||||
|
||||
/**
|
||||
* Same as createFromUpload(), but builds from a public git reference
|
||||
* (a template's repository) instead of the deployment's own uploaded
|
||||
* source. $reference is already resolved to a concrete commit/branch/tag
|
||||
* — resolving a version range (e.g. "0.3.*") is the caller's job, since
|
||||
* only it holds the GitHub client that can do so.
|
||||
*/
|
||||
abstract public function createFromRef(
|
||||
Document $resource,
|
||||
Document $deployment,
|
||||
string $owner,
|
||||
string $repository,
|
||||
string $type,
|
||||
string $reference,
|
||||
string $rootDirectory = '',
|
||||
): Document;
|
||||
|
||||
/**
|
||||
* Same as createFromUpload(), but builds from a remote tarball at $url
|
||||
* (a VCS presigned URL) instead of the deployment's own uploaded source.
|
||||
*/
|
||||
abstract public function createFromUrl(
|
||||
Document $resource,
|
||||
Document $deployment,
|
||||
string $url,
|
||||
string $rootDirectory = '',
|
||||
): Document;
|
||||
|
||||
/**
|
||||
* Best-effort cancel of an in-flight build. The deployment is already
|
||||
* marked canceled by the caller; this only needs to stop the backend from
|
||||
* still writing to it.
|
||||
*/
|
||||
abstract public function cancel(string $deploymentId): void;
|
||||
|
||||
/**
|
||||
* Deactivates any other active deployment for $resource before this one
|
||||
* goes live. Shared by both backends, called right before they mark the
|
||||
* deployment queued.
|
||||
*/
|
||||
protected function deactivateOthers(Document $resource, Document $deployment): void
|
||||
{
|
||||
if (!$deployment->getAttribute('activate', false)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$others = $this->dbForProject->find('deployments', [
|
||||
Query::equal('activate', [true]),
|
||||
Query::equal('resourceId', [$resource->getId()]),
|
||||
Query::equal('resourceType', [$resource->getCollection()]),
|
||||
Query::notEqual('$id', $deployment->getId()),
|
||||
]);
|
||||
|
||||
foreach ($others as $other) {
|
||||
$this->dbForProject->updateDocument('deployments', $other->getId(), new Document([
|
||||
'activate' => false,
|
||||
]));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The build command for a deployment: its buildCommands, wrapped for
|
||||
* sites with the framework's env and bundle commands.
|
||||
*/
|
||||
public static function command(Document $resource, Document $deployment): string
|
||||
{
|
||||
$command = $deployment->getAttribute('buildCommands', '');
|
||||
if ($resource->getCollection() !== 'sites') {
|
||||
return $command;
|
||||
}
|
||||
|
||||
$framework = Config::getParam('frameworks', [])[$resource->getAttribute('framework', '')] ?? [];
|
||||
|
||||
return \implode(' && ', \array_filter([
|
||||
$framework['envCommand'] ?? '',
|
||||
$command,
|
||||
$framework['bundleCommand'] ?? '',
|
||||
], fn ($command) => !empty($command)));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string>
|
||||
*/
|
||||
private static function permissions(): array
|
||||
{
|
||||
return [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private static function resourceFields(Document $resource): array
|
||||
{
|
||||
return [
|
||||
'resourceInternalId' => $resource->getSequence(),
|
||||
'resourceId' => $resource->getId(),
|
||||
'resourceType' => $resource->getCollection(),
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Deployment\Backend;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Executor\Executor as ExecutorClient;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
|
||||
readonly class Executor extends Backend
|
||||
{
|
||||
public function __construct(
|
||||
private BuildPublisher $publisherForBuilds,
|
||||
Database $dbForProject,
|
||||
Document $project,
|
||||
private ExecutorClient $executor,
|
||||
private array $platform,
|
||||
) {
|
||||
parent::__construct($dbForProject, $project);
|
||||
}
|
||||
|
||||
|
||||
public function createFromUpload(Document $resource, Document $deployment): Document
|
||||
{
|
||||
return $this->dispatch($resource, $deployment, null);
|
||||
}
|
||||
|
||||
public function createFromRef(
|
||||
Document $resource,
|
||||
Document $deployment,
|
||||
string $owner,
|
||||
string $repository,
|
||||
string $type,
|
||||
string $reference,
|
||||
string $rootDirectory = '',
|
||||
): Document {
|
||||
// No presigned URL to fetch here — the Builds worker clones the
|
||||
// public repo itself via these raw git coordinates.
|
||||
return $this->dispatch($resource, $deployment, new Document([
|
||||
'ownerName' => $owner,
|
||||
'repositoryName' => $repository,
|
||||
'referenceType' => $type,
|
||||
'referenceValue' => $reference,
|
||||
'rootDirectory' => $rootDirectory,
|
||||
]));
|
||||
}
|
||||
|
||||
public function createFromUrl(
|
||||
Document $resource,
|
||||
Document $deployment,
|
||||
string $url,
|
||||
string $rootDirectory = '',
|
||||
): Document {
|
||||
// The Builds worker clones the repo via the deployment's own
|
||||
// installationId/providerRepositoryId instead of $url.
|
||||
return $this->dispatch($resource, $deployment, null);
|
||||
}
|
||||
|
||||
private function dispatch(Document $resource, Document $deployment, ?Document $template): Document
|
||||
{
|
||||
$deployment = $this->upload($resource, $deployment);
|
||||
$this->deactivateOthers($resource, $deployment);
|
||||
|
||||
// The Builds worker promotes 'waiting' → 'building' on pickup.
|
||||
$deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
|
||||
'status' => 'waiting',
|
||||
]));
|
||||
|
||||
$this->publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $this->project,
|
||||
resource: $resource,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
template: $template,
|
||||
platform: $this->platform,
|
||||
));
|
||||
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
public function cancel(string $deploymentId): void
|
||||
{
|
||||
$this->executor->deleteRuntime($this->project->getId(), $deploymentId . '-build');
|
||||
}
|
||||
}
|
||||
+181
-16
@@ -1,10 +1,8 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Deployment\Backend;
|
||||
namespace Appwrite\Deployment;
|
||||
|
||||
use Ahc\Jwt\JWT;
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Token;
|
||||
use OpenRuntimes\Orchestrator\Enum\CallbackEvent;
|
||||
use OpenRuntimes\Orchestrator\Enum\ReadFormat;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
@@ -18,10 +16,15 @@ use Utopia\Config\Config;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\Permission;
|
||||
use Utopia\Database\Helpers\Role;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\System\System;
|
||||
|
||||
/**
|
||||
* Builds and submits an open-runtimes jobs-service job for a deployment.
|
||||
* Owns a deployment's lifecycle: upload bookkeeping, creating it and
|
||||
* dispatching the build as an open-runtimes jobs-service job, and canceling one
|
||||
* in flight.
|
||||
*
|
||||
* Source crosses the boundary via the artifacts system (presigned GET download
|
||||
* + unarchive, run by the sidecar) — a GET has no request-body cap, so large
|
||||
@@ -35,27 +38,65 @@ use Utopia\System\System;
|
||||
* — everything else about the payload stays the same.
|
||||
*
|
||||
* Covers function and site deployments whose source is a tarball: manual
|
||||
* upload, duplicate/rebuild, and templates (public GitHub tarball resolved
|
||||
* from a git reference). Site builds also emit a JSON build manifest, read
|
||||
* back post-job as an artifact callback for adapter detection.
|
||||
* upload, duplicate/rebuild, VCS commits, and templates (public GitHub tarball
|
||||
* resolved from a git reference). Site builds also emit a JSON build manifest,
|
||||
* read back post-job as an artifact callback for adapter detection.
|
||||
*/
|
||||
readonly class Orchestrator extends Backend
|
||||
readonly class Deployments
|
||||
{
|
||||
public function __construct(
|
||||
private Jobs $jobs,
|
||||
Database $dbForProject,
|
||||
Document $project,
|
||||
protected Database $dbForProject,
|
||||
protected Document $project,
|
||||
private array $platform,
|
||||
) {
|
||||
parent::__construct($dbForProject, $project);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves chunked-upload progress onto the deployment — source path/size,
|
||||
* chunk counters, metadata. Never triggers a build; call createFromUpload()
|
||||
* once the upload is complete. Pass a single `Document` carrying every field
|
||||
* the deployment should end up with: either a fresh, not-yet-persisted
|
||||
* one (a plain `new Document([...])`), or the existing one fetched from
|
||||
* the database with more attributes set via `setAttributes()`. A new
|
||||
* document (one with no $sequence, i.e. not yet persisted — a document
|
||||
* only gets one from the database, never from `setAttributes()`) gets
|
||||
* the standard $permissions and resourceId/resourceInternalId/
|
||||
* resourceType merged in automatically.
|
||||
*/
|
||||
public function upload(Document $resource, Document $deployment): Document
|
||||
{
|
||||
if ($deployment->getSequence() === null) {
|
||||
return $this->dbForProject->createDocument('deployments', new Document([
|
||||
'$permissions' => self::permissions(),
|
||||
...self::resourceFields($resource),
|
||||
...$deployment->getArrayCopy(),
|
||||
]));
|
||||
}
|
||||
|
||||
return $this->dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
|
||||
}
|
||||
|
||||
/**
|
||||
* Finalizes the deployment document (see upload() for what `$deployment`
|
||||
* should carry) and dispatches it for building from its own uploaded
|
||||
* source. Marks it queued, writes its buildPath and deactivates any other
|
||||
* active deployment for $resource. A deployment canceled before it is
|
||||
* queued is left canceled and never dispatched. Returns the persisted,
|
||||
* updated deployment.
|
||||
*/
|
||||
public function createFromUpload(Document $resource, Document $deployment): Document
|
||||
{
|
||||
return $this->submit($resource, $deployment, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as createFromUpload(), but builds from a public git reference
|
||||
* (a template's repository) instead of the deployment's own uploaded
|
||||
* source. $reference is already resolved to a concrete commit/branch/tag
|
||||
* — resolving a version range (e.g. "0.3.*") is the caller's job, since
|
||||
* only it holds the GitHub client that can do so.
|
||||
*/
|
||||
public function createFromRef(
|
||||
Document $resource,
|
||||
Document $deployment,
|
||||
@@ -73,6 +114,10 @@ readonly class Orchestrator extends Backend
|
||||
return $this->submit($resource, $deployment, ['url' => $url, 'subdir' => $rootDirectory]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as createFromUpload(), but builds from a remote tarball at $url
|
||||
* (a VCS presigned URL) instead of the deployment's own uploaded source.
|
||||
*/
|
||||
public function createFromUrl(
|
||||
Document $resource,
|
||||
Document $deployment,
|
||||
@@ -84,22 +129,57 @@ readonly class Orchestrator extends Backend
|
||||
|
||||
private function submit(Document $resource, Document $deployment, ?array $source): Document
|
||||
{
|
||||
// The caller may have been holding this deployment for a while (the
|
||||
// Builds worker pushes a template commit first), so its status is stale
|
||||
// by now — dropping it keeps upload() from writing a cancel away, and
|
||||
// the queued transition below is guarded instead.
|
||||
$deployment->removeAttribute('status');
|
||||
$deployment = $this->upload($resource, $deployment);
|
||||
$this->deactivateOthers($resource, $deployment);
|
||||
|
||||
$deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
|
||||
$queued = $this->dbForProject->updateDocuments('deployments', new Document([
|
||||
'status' => 'waiting',
|
||||
'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()),
|
||||
]));
|
||||
]), [
|
||||
Query::equal('$id', [$deployment->getId()]),
|
||||
Query::notEqual('status', 'canceled'),
|
||||
]);
|
||||
|
||||
$deployment = $this->dbForProject->getDocument('deployments', $deployment->getId());
|
||||
|
||||
// Canceled before it could be queued: leave it canceled, submit nothing,
|
||||
// and leave the currently active deployment alone.
|
||||
if ($queued === 0) {
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
// Claiming activation takes it away from the other pending deployments,
|
||||
// so a cancel landing while we do it would leave nothing able to go
|
||||
// live. Hand the claim back to exactly the deployments it was taken
|
||||
// from, and stop before submitting a job for a canceled deployment.
|
||||
$deactivated = $this->deactivateOthers($resource, $deployment);
|
||||
if ($deactivated !== [] && $this->status($deployment->getId()) === 'canceled') {
|
||||
foreach ($deactivated as $other) {
|
||||
$this->dbForProject->updateDocument('deployments', $other, new Document([
|
||||
'activate' => true,
|
||||
]));
|
||||
}
|
||||
|
||||
return $this->dbForProject->getDocument('deployments', $deployment->getId());
|
||||
}
|
||||
|
||||
try {
|
||||
$this->jobs->create(...static::payload($this->project, $resource, $deployment, $this->platform, $source));
|
||||
} catch (\Throwable $error) {
|
||||
$deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
|
||||
// Guarded like the transition above: a cancel that landed while the
|
||||
// job was being submitted must not be reported as a failure.
|
||||
$this->dbForProject->updateDocuments('deployments', new Document([
|
||||
'status' => 'failed',
|
||||
'buildLogs' => "\nAn internal error occurred while building. Please try again, and contact support if the problem persists.\n",
|
||||
'buildEndedAt' => DateTime::now(),
|
||||
]));
|
||||
]), [
|
||||
Query::equal('$id', [$deployment->getId()]),
|
||||
Query::notEqual('status', 'canceled'),
|
||||
]);
|
||||
|
||||
throw $error;
|
||||
}
|
||||
@@ -107,11 +187,72 @@ readonly class Orchestrator extends Backend
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort cancel of an in-flight build. The deployment is already
|
||||
* marked canceled by the caller; this only needs to stop the backend from
|
||||
* still writing to it.
|
||||
*/
|
||||
public function cancel(string $deploymentId): void
|
||||
{
|
||||
$this->jobs->delete(static::id($this->project->getId(), $deploymentId));
|
||||
}
|
||||
|
||||
/**
|
||||
* Deactivates any other active deployment for $resource before this one
|
||||
* goes live, called once the deployment is queued for building.
|
||||
*
|
||||
* @return array<string> The ids it deactivated, so the caller can hand the
|
||||
* claim back if this deployment never gets to build.
|
||||
*/
|
||||
protected function deactivateOthers(Document $resource, Document $deployment): array
|
||||
{
|
||||
if (!$deployment->getAttribute('activate', false)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$others = $this->dbForProject->find('deployments', [
|
||||
Query::equal('activate', [true]),
|
||||
Query::equal('resourceId', [$resource->getId()]),
|
||||
Query::equal('resourceType', [$resource->getCollection()]),
|
||||
Query::notEqual('$id', $deployment->getId()),
|
||||
]);
|
||||
|
||||
$deactivated = [];
|
||||
foreach ($others as $other) {
|
||||
$this->dbForProject->updateDocument('deployments', $other->getId(), new Document([
|
||||
'activate' => false,
|
||||
]));
|
||||
$deactivated[] = $other->getId();
|
||||
}
|
||||
|
||||
return $deactivated;
|
||||
}
|
||||
|
||||
private function status(string $deploymentId): string
|
||||
{
|
||||
return $this->dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status', '');
|
||||
}
|
||||
|
||||
/**
|
||||
* The build command for a deployment: its buildCommands, wrapped for
|
||||
* sites with the framework's env and bundle commands.
|
||||
*/
|
||||
public static function command(Document $resource, Document $deployment): string
|
||||
{
|
||||
$command = $deployment->getAttribute('buildCommands', '');
|
||||
if ($resource->getCollection() !== 'sites') {
|
||||
return $command;
|
||||
}
|
||||
|
||||
$framework = Config::getParam('frameworks', [])[$resource->getAttribute('framework', '')] ?? [];
|
||||
|
||||
return \implode(' && ', \array_filter([
|
||||
$framework['envCommand'] ?? '',
|
||||
$command,
|
||||
$framework['bundleCommand'] ?? '',
|
||||
], fn ($command) => !empty($command)));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed> Named arguments for OpenRuntimes\Orchestrator\Jobs::create().
|
||||
*/
|
||||
@@ -398,4 +539,28 @@ readonly class Orchestrator extends Backend
|
||||
'OPEN_RUNTIMES_NFT' => System::getEnv('_APP_OPEN_RUNTIMES_NFT', 'enabled'),
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string>
|
||||
*/
|
||||
private static function permissions(): array
|
||||
{
|
||||
return [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private static function resourceFields(Document $resource): array
|
||||
{
|
||||
return [
|
||||
'resourceInternalId' => $resource->getSequence(),
|
||||
'resourceId' => $resource->getId(),
|
||||
'resourceType' => $resource->getCollection(),
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Compute;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
@@ -10,6 +10,7 @@ use Appwrite\Filter\BranchDomain as BranchDomainFilter;
|
||||
use Appwrite\Platform\Action;
|
||||
use Appwrite\Platform\Modules\Compute\Validator\Specification as SpecificationValidator;
|
||||
use Appwrite\Platform\Permission as AppwritePermission;
|
||||
use Appwrite\Vcs\Factory as VcsFactory;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
@@ -68,7 +69,29 @@ class Base extends Action
|
||||
return $allowedSpecifications[0];
|
||||
}
|
||||
|
||||
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Backend $deployments, array $platform = [], string $referenceType = 'branch', string $reference = ''): Document
|
||||
/**
|
||||
* Templates can pin a version range (e.g. "0.3.*"); codeload only takes a
|
||||
* concrete ref, so resolve a range to the highest matching tag. Templates
|
||||
* are public github.com repositories regardless of the resource's own
|
||||
* provider, so this always uses the GitHub adapter.
|
||||
*/
|
||||
public static function resolveTemplateRef(VcsFactory $vcsFactory, string $owner, string $repository, string $type, string $reference): string
|
||||
{
|
||||
if ($type !== Git::CLONE_TYPE_TAG || ! \str_contains($reference, '*')) {
|
||||
return $reference;
|
||||
}
|
||||
|
||||
try {
|
||||
$tags = $vcsFactory->fromProvider('github')->listTags($owner, $repository, $reference);
|
||||
|
||||
return \end($tags) ?: $reference;
|
||||
} catch (\Throwable) {
|
||||
// Fall back to the raw reference; the build surfaces a bad ref.
|
||||
return $reference;
|
||||
}
|
||||
}
|
||||
|
||||
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Deployments $deployments, array $platform = [], string $referenceType = 'branch', string $reference = ''): Document
|
||||
{
|
||||
$deploymentId = ID::unique();
|
||||
$entrypoint = $function->getAttribute('entrypoint', '');
|
||||
@@ -144,11 +167,9 @@ class Base extends Action
|
||||
'activate' => $activate,
|
||||
]);
|
||||
|
||||
// Build a plain (non-template) VCS deployment through $deployments
|
||||
// (executor or jobs-service, decided by _APP_BUILDS_BACKEND).
|
||||
// Template-into-repo pushes go through the Builds worker (which does
|
||||
// the git write, then hands the build to the jobs-service itself
|
||||
// when on orchestrator).
|
||||
// Build a plain (non-template) VCS deployment through $deployments.
|
||||
// Template-into-repo pushes go through the Builds worker, which does the
|
||||
// git write and then hands the build to the jobs-service itself.
|
||||
if ($template->isEmpty()) {
|
||||
$ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch');
|
||||
$deployment = $deployments->createFromUrl(
|
||||
@@ -181,7 +202,7 @@ class Base extends Action
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
public function redeployVcsSite(Request $request, Document $site, Document $project, Document $installation, Database $dbForProject, Database $dbForPlatform, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Authorization $authorization, Backend $deployments, array $platform, string $referenceType = 'branch', string $reference = ''): Document
|
||||
public function redeployVcsSite(Request $request, Document $site, Document $project, Document $installation, Database $dbForProject, Database $dbForPlatform, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Authorization $authorization, Deployments $deployments, array $platform, string $referenceType = 'branch', string $reference = ''): Document
|
||||
{
|
||||
$deploymentId = ID::unique();
|
||||
$providerInstallationId = $installation->getAttribute('providerInstallationId', '');
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
@@ -106,7 +106,7 @@ class Create extends Action
|
||||
Document $project,
|
||||
Device $deviceForFunctions,
|
||||
Device $deviceForLocal,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
array $plan,
|
||||
Authorization $authorization,
|
||||
callable $locks
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Duplicate;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
@@ -78,7 +78,7 @@ class Create extends Action
|
||||
Database $dbForProject,
|
||||
Database $dbForPlatform,
|
||||
Event $queueForEvents,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
Device $deviceForFunctions,
|
||||
VcsFactory $vcsFactory,
|
||||
) {
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Status;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Backend\Orchestrator;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
@@ -72,7 +71,7 @@ class Update extends Action
|
||||
Response $response,
|
||||
Database $dbForProject,
|
||||
Event $queueForEvents,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
callable $locks,
|
||||
) {
|
||||
$function = $dbForProject->getDocument('functions', $functionId);
|
||||
@@ -98,9 +97,9 @@ class Update extends Action
|
||||
// Write under the Jobs worker's per-deployment lock: its handlers
|
||||
// read-modify-write buildLogs, and an unserialized cancel write here
|
||||
// loses the closing log line to an in-flight append.
|
||||
$cancel = function () use ($dbForProject, $deployment, $duration, $deployments) {
|
||||
$cancel = function () use ($dbForProject, $deployment, $duration) {
|
||||
try {
|
||||
return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator)));
|
||||
return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
|
||||
} catch (TransactionException) {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
|
||||
|
||||
@@ -113,7 +112,7 @@ class Update extends Action
|
||||
}
|
||||
|
||||
if ($deployment->getAttribute('status') !== 'canceled') {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator)));
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
|
||||
}
|
||||
|
||||
return $deployment;
|
||||
@@ -140,25 +139,20 @@ class Update extends Action
|
||||
}
|
||||
|
||||
/**
|
||||
* The sparse update marking a build canceled. Jobs-backed builds have no
|
||||
* cancel worker to write the closing log line the executor's Builds worker
|
||||
* adds, so it is appended here; executor deployments get it from their worker.
|
||||
* The sparse update marking a build canceled. No worker writes the closing
|
||||
* log line for a canceled build, so it is appended here.
|
||||
*
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function cancel(Document $deployment, int $duration, bool $appendLog): array
|
||||
private function cancel(Document $deployment, int $duration): array
|
||||
{
|
||||
$update = [
|
||||
$logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n";
|
||||
|
||||
return [
|
||||
'buildEndedAt' => DateTime::now(),
|
||||
'buildDuration' => $duration,
|
||||
'status' => 'canceled',
|
||||
'buildLogs' => \substr($logs, -APP_LOG_LENGTH_LIMIT),
|
||||
];
|
||||
|
||||
if ($appendLog) {
|
||||
$logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n";
|
||||
$update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT);
|
||||
}
|
||||
|
||||
return $update;
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Template;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
@@ -24,7 +24,6 @@ use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\Validator\Boolean;
|
||||
use Utopia\Validator\Text;
|
||||
use Utopia\Validator\WhiteList;
|
||||
use Utopia\VCS\Adapter\Git\GitHub;
|
||||
|
||||
class Create extends Base
|
||||
{
|
||||
@@ -102,7 +101,7 @@ class Create extends Base
|
||||
Document $project,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
VcsFactory $vcsFactory,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
Authorization $authorization,
|
||||
array $platform
|
||||
) {
|
||||
@@ -126,9 +125,7 @@ class Create extends Base
|
||||
|
||||
if (!empty($function->getAttribute('providerRepositoryId'))) {
|
||||
// VCS-connected function: the Builds worker merges the template into
|
||||
// the user's repo, pushes it as a commit, then builds that commit —
|
||||
// on the executor itself, or by submitting a job when
|
||||
// _APP_BUILDS_BACKEND=orchestrator.
|
||||
// the user's repo, pushes it as a commit, then builds that commit.
|
||||
$installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId'));
|
||||
|
||||
$deployment = $this->redeployVcsFunction(
|
||||
@@ -160,24 +157,10 @@ class Create extends Base
|
||||
|
||||
$deploymentId = ID::unique();
|
||||
|
||||
// Templates can pin a version range (e.g. "0.3.*"); codeload only
|
||||
// takes a concrete ref, so resolve the range to the highest matching
|
||||
// tag (mirrors the executor's `git ls-remote --tags | tail -1`).
|
||||
// Templates are public github.com repositories regardless of the
|
||||
// function's own provider, so this always uses the GitHub adapter.
|
||||
$ref = $reference;
|
||||
if ($type === GitHub::CLONE_TYPE_TAG && \str_contains($reference, '*')) {
|
||||
try {
|
||||
$tags = $vcsFactory->fromProvider('github')->listTags($owner, $repository, $reference);
|
||||
$ref = \end($tags) ?: $reference;
|
||||
} catch (\Throwable) {
|
||||
// Fall back to the raw reference; the build surfaces a bad ref.
|
||||
}
|
||||
}
|
||||
$ref = Base::resolveTemplateRef($vcsFactory, $owner, $repository, $type, $reference);
|
||||
|
||||
// Public template: pull the source straight from GitHub's public repo
|
||||
// (codeload tarball on the jobs backend, a plain git clone on the
|
||||
// executor); unarchive/checkout strips down to the rootDirectory.
|
||||
// as a codeload tarball; unarchive strips down to the rootDirectory.
|
||||
$deployment = $deployments->createFromRef(
|
||||
$function,
|
||||
new Document([
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Vcs;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
@@ -93,7 +93,7 @@ class Create extends Base
|
||||
Event $queueForEvents,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
VcsFactory $vcsFactory,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
array $platform,
|
||||
) {
|
||||
$function = $dbForProject->getDocument('functions', $functionId);
|
||||
|
||||
@@ -2,9 +2,8 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Functions;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Message\Func as FunctionMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
@@ -30,8 +29,6 @@ use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Exception\Duplicate as DuplicateException;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
use Utopia\Database\Helpers\Permission;
|
||||
use Utopia\Database\Helpers\Role;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Database\Validator\Roles;
|
||||
use Utopia\Http\Request;
|
||||
@@ -172,7 +169,7 @@ class Create extends Base
|
||||
Document $project,
|
||||
Event $queueForEvents,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
Realtime $queueForRealtime,
|
||||
Webhook $queueForWebhooks,
|
||||
FunctionPublisher $publisherForFunctions,
|
||||
@@ -372,33 +369,33 @@ class Create extends Base
|
||||
);
|
||||
|
||||
} elseif (!$template->isEmpty()) {
|
||||
// Deploy non-VCS from template
|
||||
$deploymentId = ID::unique();
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceId' => $function->getId(),
|
||||
'resourceInternalId' => $function->getSequence(),
|
||||
'resourceType' => 'functions',
|
||||
'entrypoint' => $function->getAttribute('entrypoint', ''),
|
||||
'buildCommands' => $function->getAttribute('commands', ''),
|
||||
'startCommand' => $function->getAttribute('startCommand', ''),
|
||||
'type' => 'manual',
|
||||
'activate' => true,
|
||||
]));
|
||||
// Deploy non-VCS from the template's public GitHub repository.
|
||||
$templateVersion = Base::resolveTemplateRef($vcsFactory, $templateOwner, $templateRepository, Git::CLONE_TYPE_TAG, $templateVersion);
|
||||
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
template: $template,
|
||||
platform: $platform,
|
||||
));
|
||||
$deployment = $deployments->createFromRef(
|
||||
$function,
|
||||
new Document([
|
||||
'$id' => ID::unique(),
|
||||
'entrypoint' => $function->getAttribute('entrypoint', ''),
|
||||
'buildCommands' => $function->getAttribute('commands', ''),
|
||||
'startCommand' => $function->getAttribute('startCommand', ''),
|
||||
'providerRepositoryName' => $templateRepository,
|
||||
'providerRepositoryOwner' => $templateOwner,
|
||||
'providerRepositoryUrl' => "https://github.com/{$templateOwner}/{$templateRepository}",
|
||||
'providerBranchUrl' => "https://github.com/{$templateOwner}/{$templateRepository}/blob/{$templateVersion}",
|
||||
// The coordinates a redeploy needs: remote-source builds
|
||||
// never store a tarball.
|
||||
'providerBranch' => $templateVersion,
|
||||
'providerRootDirectory' => $templateRootDirectory,
|
||||
'type' => 'vcs',
|
||||
'activate' => true,
|
||||
]),
|
||||
$templateOwner,
|
||||
$templateRepository,
|
||||
Git::CLONE_TYPE_TAG,
|
||||
$templateVersion,
|
||||
$templateRootDirectory,
|
||||
);
|
||||
}
|
||||
|
||||
$functionsDomain = $platform['functionsDomain'];
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Functions;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Event\Validator\FunctionEvent;
|
||||
@@ -153,7 +153,7 @@ class Update extends Base
|
||||
Document $project,
|
||||
Event $queueForEvents,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
Database $dbForPlatform,
|
||||
VcsFactory $vcsFactory,
|
||||
RepositoryWebhooks $repositoryWebhooks,
|
||||
|
||||
@@ -9,7 +9,7 @@ use Appwrite\Extend\Exception;
|
||||
*
|
||||
* Every build error the worker raises intentionally is a BuildException, so the
|
||||
* failure handler can show its message to users while masking any other
|
||||
* throwable (e.g. database or executor failures) behind a generic message.
|
||||
* throwable (e.g. database or jobs-service failures) behind a generic message.
|
||||
*/
|
||||
class BuildException extends Exception
|
||||
{
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -30,8 +30,7 @@ use Utopia\System\System;
|
||||
/**
|
||||
* Applies open-runtimes jobs-service callbacks to a deployment. The API
|
||||
* callback endpoint verifies + enqueues the CloudEvent; this worker owns the
|
||||
* deployment state transitions (log streaming and the ready/failed outcome)
|
||||
* that the executor path performs inline in the Builds worker.
|
||||
* deployment state transitions: log streaming and the ready/failed outcome.
|
||||
*
|
||||
* Callbacks arrive concurrently and out of order (one log line each), so
|
||||
* processing for a deployment is serialized under a per-deployment lock — that
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Sites\Http\Deployments;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
@@ -107,7 +107,7 @@ class Create extends Action
|
||||
Event $queueForEvents,
|
||||
Device $deviceForSites,
|
||||
Device $deviceForLocal,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
array $plan,
|
||||
Authorization $authorization,
|
||||
array $platform,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Duplicate;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
@@ -83,7 +83,7 @@ class Create extends Action
|
||||
Database $dbForProject,
|
||||
Database $dbForPlatform,
|
||||
Event $queueForEvents,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
Device $deviceForSites,
|
||||
VcsFactory $vcsFactory,
|
||||
Authorization $authorization,
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Status;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Backend\Orchestrator;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
@@ -70,7 +69,7 @@ class Update extends Action
|
||||
Response $response,
|
||||
Database $dbForProject,
|
||||
Event $queueForEvents,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
callable $locks
|
||||
) {
|
||||
$site = $dbForProject->getDocument('sites', $siteId);
|
||||
@@ -96,9 +95,9 @@ class Update extends Action
|
||||
// Write under the Jobs worker's per-deployment lock: its handlers
|
||||
// read-modify-write buildLogs, and an unserialized cancel write here
|
||||
// loses the closing log line to an in-flight append.
|
||||
$cancel = function () use ($dbForProject, $deployment, $duration, $deployments) {
|
||||
$cancel = function () use ($dbForProject, $deployment, $duration) {
|
||||
try {
|
||||
return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator)));
|
||||
return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
|
||||
} catch (TransactionException) {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
|
||||
|
||||
@@ -111,7 +110,7 @@ class Update extends Action
|
||||
}
|
||||
|
||||
if ($deployment->getAttribute('status') !== 'canceled') {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator)));
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
|
||||
}
|
||||
|
||||
return $deployment;
|
||||
@@ -138,25 +137,20 @@ class Update extends Action
|
||||
}
|
||||
|
||||
/**
|
||||
* The sparse update marking a build canceled. Jobs-backed builds have no
|
||||
* cancel worker to write the closing log line the executor's Builds worker
|
||||
* adds, so it is appended here; executor deployments get it from their worker.
|
||||
* The sparse update marking a build canceled. No worker writes the closing
|
||||
* log line for a canceled build, so it is appended here.
|
||||
*
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function cancel(Document $deployment, int $duration, bool $appendLog): array
|
||||
private function cancel(Document $deployment, int $duration): array
|
||||
{
|
||||
$update = [
|
||||
$logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n";
|
||||
|
||||
return [
|
||||
'buildEndedAt' => DateTime::now(),
|
||||
'buildDuration' => $duration,
|
||||
'status' => 'canceled',
|
||||
'buildLogs' => \substr($logs, -APP_LOG_LENGTH_LIMIT),
|
||||
];
|
||||
|
||||
if ($appendLog) {
|
||||
$logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n";
|
||||
$update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT);
|
||||
}
|
||||
|
||||
return $update;
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Template;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
@@ -27,7 +27,6 @@ use Utopia\System\System;
|
||||
use Utopia\Validator\Boolean;
|
||||
use Utopia\Validator\Text;
|
||||
use Utopia\Validator\WhiteList;
|
||||
use Utopia\VCS\Adapter\Git;
|
||||
|
||||
class Create extends Base
|
||||
{
|
||||
@@ -106,7 +105,7 @@ class Create extends Base
|
||||
BuildPublisher $publisherForBuilds,
|
||||
VcsFactory $vcsFactory,
|
||||
Authorization $authorization,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
array $platform
|
||||
) {
|
||||
$site = $dbForProject->getDocument('sites', $siteId);
|
||||
@@ -164,17 +163,7 @@ class Create extends Base
|
||||
$commands[] = $site->getAttribute('buildCommand', '');
|
||||
}
|
||||
|
||||
// Templates can pin a version range (e.g. "0.3.*"); codeload only takes
|
||||
// a concrete ref, so resolve it to the highest matching tag via GitHub.
|
||||
$ref = $reference;
|
||||
if ($type === Git::CLONE_TYPE_TAG && \str_contains($reference, '*')) {
|
||||
try {
|
||||
$tags = $vcsFactory->fromProvider('github')->listTags($owner, $repository, $reference);
|
||||
$ref = \end($tags) ?: $reference;
|
||||
} catch (\Throwable) {
|
||||
// Fall back to the raw reference; the build surfaces a bad ref.
|
||||
}
|
||||
}
|
||||
$ref = Base::resolveTemplateRef($vcsFactory, $owner, $repository, $type, $reference);
|
||||
|
||||
$deploymentId = ID::unique();
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
@@ -233,8 +222,7 @@ class Create extends Base
|
||||
$this->updateEmptyManualRule($project, $site, $deployment, $dbForPlatform, $authorization);
|
||||
|
||||
// Public template: pull the source straight from GitHub's public repo
|
||||
// (codeload tarball on the jobs backend, a plain git clone on the
|
||||
// executor); unarchive/checkout strips down to the rootDirectory.
|
||||
// as a codeload tarball; unarchive strips down to the rootDirectory.
|
||||
$deployment = $deployments->createFromRef(
|
||||
$site,
|
||||
$deployment,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Vcs;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
@@ -96,7 +96,7 @@ class Create extends Base
|
||||
BuildPublisher $publisherForBuilds,
|
||||
VcsFactory $vcsFactory,
|
||||
Authorization $authorization,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
array $platform
|
||||
) {
|
||||
$site = $dbForProject->getDocument('sites', $siteId);
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Sites\Http\Sites;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
@@ -156,7 +156,7 @@ class Update extends Base
|
||||
RepositoryWebhooks $repositoryWebhooks,
|
||||
Executor $executor,
|
||||
Authorization $authorization,
|
||||
Backend $deployments,
|
||||
Deployments $deployments,
|
||||
array $platform
|
||||
) {
|
||||
if (!empty($adapter)) {
|
||||
|
||||
@@ -389,7 +389,7 @@ trait Deployment
|
||||
'activate' => $activate,
|
||||
]);
|
||||
|
||||
// The Backend is built per repository: a webhook fans out to
|
||||
// The Deployments service is built per repository: a webhook fans out to
|
||||
// many tenant projects, each with its own database.
|
||||
$deployment = $authorization->skip(fn () => $deploymentsFactory($dbForProject, $project)
|
||||
->createFromUrl(
|
||||
|
||||
@@ -8,9 +8,9 @@ use Utopia\Config\Config;
|
||||
use Utopia\Database\Document;
|
||||
|
||||
/**
|
||||
* Emits build usage/billing metrics for a completed deployment. Shared by both
|
||||
* build backends — the executor (Builds worker) and the jobs-service (Jobs
|
||||
* worker) — so a build is counted identically however it was produced.
|
||||
* Emits build usage/billing metrics for a completed deployment. Shared by the
|
||||
* Jobs worker and the Builds worker, so a build is counted identically whether
|
||||
* it finished on the jobs-service or failed before reaching it.
|
||||
*/
|
||||
final class Build
|
||||
{
|
||||
|
||||
@@ -41,100 +41,6 @@ class Executor
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Create runtime
|
||||
*
|
||||
* Launches a runtime container for a deployment ready for execution
|
||||
*
|
||||
* @param string $deploymentId
|
||||
* @param string $projectId
|
||||
* @param string $source
|
||||
* @param string $image
|
||||
* @param bool $remove
|
||||
* @param string $entrypoint
|
||||
* @param string $destination
|
||||
* @param array $variables
|
||||
* @param string $command
|
||||
*/
|
||||
public function createRuntime(
|
||||
string $deploymentId,
|
||||
string $projectId,
|
||||
string $source,
|
||||
string $image,
|
||||
string $version,
|
||||
float $cpus,
|
||||
int $memory,
|
||||
int $timeout,
|
||||
bool $remove = false,
|
||||
string $entrypoint = '',
|
||||
string $destination = '',
|
||||
array $variables = [],
|
||||
?string $command = null,
|
||||
string $outputDirectory = '',
|
||||
string $runtimeEntrypoint = '',
|
||||
string $cacheKey = ''
|
||||
) {
|
||||
$runtimeId = "$projectId-$deploymentId-build";
|
||||
$route = "/runtimes";
|
||||
|
||||
// Remove after migration
|
||||
if ($version === 'v3' || $version === 'v4') {
|
||||
$version = 'v5';
|
||||
}
|
||||
|
||||
$params = [
|
||||
'runtimeId' => $runtimeId,
|
||||
'source' => $source,
|
||||
'destination' => $destination,
|
||||
'image' => $image,
|
||||
'entrypoint' => $entrypoint,
|
||||
'variables' => $variables,
|
||||
'remove' => $remove,
|
||||
'command' => $command,
|
||||
'cpus' => $cpus,
|
||||
'memory' => $memory,
|
||||
'version' => $version,
|
||||
'timeout' => $timeout,
|
||||
'outputDirectory' => $outputDirectory,
|
||||
'runtimeEntrypoint' => $runtimeEntrypoint,
|
||||
'cacheKey' => $cacheKey
|
||||
];
|
||||
|
||||
|
||||
$response = $this->call($this->endpoint, self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout);
|
||||
|
||||
$status = $response['headers']['status-code'];
|
||||
if ($status >= 400) {
|
||||
$message = \is_string($response['body']) ? $response['body'] : ($response['body']['message'] ?? '');
|
||||
$type = \is_array($response['body']) ? ($response['body']['type'] ?? ExecutorException::GENERAL_UNKNOWN) : ExecutorException::GENERAL_UNKNOWN;
|
||||
throw new ExecutorException($message, $status, type: $type);
|
||||
}
|
||||
|
||||
return $response['body'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Listen to realtime logs stream of a runtime
|
||||
*
|
||||
* @param string $deploymentId
|
||||
* @param string $projectId
|
||||
* @param callable $callback
|
||||
*/
|
||||
public function getLogs(
|
||||
string $deploymentId,
|
||||
string $projectId,
|
||||
string $timeout,
|
||||
callable $callback
|
||||
) {
|
||||
$runtimeId = "$projectId-$deploymentId-build";
|
||||
$route = "/runtimes/{$runtimeId}/logs";
|
||||
$params = [
|
||||
'timeout' => $timeout
|
||||
];
|
||||
|
||||
$this->call($this->endpoint, self::METHOD_GET, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout, $callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete Runtime
|
||||
*
|
||||
|
||||
@@ -4,12 +4,12 @@ declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\Deployment;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Deployments;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Document;
|
||||
|
||||
final class BackendTest extends TestCase
|
||||
final class DeploymentsTest extends TestCase
|
||||
{
|
||||
public function testSiteCommandIncludesFrameworkAndDeploymentCommands(): void
|
||||
{
|
||||
@@ -30,7 +30,7 @@ final class BackendTest extends TestCase
|
||||
|
||||
$this->assertSame(
|
||||
'cp .env.example .env && npm run build && npm run bundle',
|
||||
Backend::command($resource, $deployment)
|
||||
Deployments::command($resource, $deployment)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -39,6 +39,6 @@ final class BackendTest extends TestCase
|
||||
$resource = new Document(['$collection' => 'functions']);
|
||||
$deployment = new Document(['buildCommands' => 'npm install']);
|
||||
|
||||
$this->assertSame('npm install', Backend::command($resource, $deployment));
|
||||
$this->assertSame('npm install', Deployments::command($resource, $deployment));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user