chore: revert scheduler & hook changes

This commit is contained in:
loks0n committed 2025-09-03 12:47:33 +01:00
1 parent e4adff4edc
commit 1b586bbc51
11 files changed
+125 -153

No files matched your search

+15 -4
View File
@@ -5,7 +5,6 @@ require_once __DIR__ . '/init.php';
use Appwrite\Event\Certificate;
use Appwrite\Event\Delete;
use Appwrite\Event\Func;
use Appwrite\Event\Messaging;
use Appwrite\Event\StatsResources;
use Appwrite\Event\StatsUsage;
use Appwrite\Platform\Appwrite;
@@ -192,6 +191,21 @@ CLI::setResource('getLogsDB', function (Group $pools, Cache $cache) {
CLI::setResource('publisher', function (Group $pools) {
return new BrokerPool(publisher: $pools->get('publisher'));
}, ['pools']);
CLI::setResource('publisherDatabases', function (BrokerPool $publisher) {
return $publisher;
}, ['publisher']);
CLI::setResource('publisherFunctions', function (BrokerPool $publisher) {
return $publisher;
}, ['publisher']);
CLI::setResource('publisherMigrations', function (BrokerPool $publisher) {
return $publisher;
}, ['publisher']);
CLI::setResource('publisherStatsUsage', function (BrokerPool $publisher) {
return $publisher;
}, ['publisher']);
CLI::setResource('publisherMessaging', function (BrokerPool $publisher) {
return $publisher;
}, ['publisher']);
CLI::setResource('queueForStatsUsage', function (Publisher $publisher) {
return new StatsUsage($publisher);
}, ['publisher']);
@@ -204,9 +218,6 @@ CLI::setResource('queueForFunctions', function (Publisher $publisher) {
CLI::setResource('queueForDeletes', function (Publisher $publisher) {
return new Delete($publisher);
}, ['publisher']);
CLI::setResource('queueForMessaging', function (Publisher $publisher) {
return new Messaging($publisher);
}, ['publisher']);
CLI::setResource('queueForCertificates', function (Publisher $publisher) {
return new Certificate($publisher);
}, ['publisher']);
+16 -13
View File
@@ -29,6 +29,7 @@ use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\Authorization;
use Utopia\Queue\Publisher;
use Utopia\System\System;
use Utopia\Telemetry\Adapter as Telemetry;
use Utopia\Validator\WhiteList;
@@ -412,6 +413,9 @@ App::init()
->inject('response')
->inject('project')
->inject('user')
->inject('publisher')
->inject('publisherFunctions')
->inject('publisherWebhooks')
->inject('queueForEvents')
->inject('queueForMessaging')
->inject('queueForAudits')
@@ -419,9 +423,6 @@ App::init()
->inject('queueForDatabase')
->inject('queueForBuilds')
->inject('queueForStatsUsage')
->inject('queueForFunctions')
->inject('queueForWebhooks')
->inject('queueForRealtime')
->inject('dbForProject')
->inject('timelimit')
->inject('resourceToken')
@@ -430,7 +431,7 @@ App::init()
->inject('plan')
->inject('devKey')
->inject('telemetry')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Telemetry $telemetry) use ($usageDatabaseListener, $eventDatabaseListener) {
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Telemetry $telemetry) use ($usageDatabaseListener, $eventDatabaseListener) {
$route = $utopia->getRoute();
@@ -538,9 +539,13 @@ App::init()
$queueForDatabase->setProject($project);
$queueForBuilds->setProject($project);
$queueForMessaging->setProject($project);
$queueForFunctions->setProject($project);
$queueForWebhooks->setProject($project);
$queueForRealtime->setProject($project);
// Clone the queues, to prevent events triggered by the database listener
// from overwriting the events that are supposed to be triggered in the shutdown hook.
$queueForEventsClone = new Event($publisher);
$queueForFunctions = new Func($publisherFunctions);
$queueForWebhooks = new Webhook($publisherWebhooks);
$queueForRealtime = new Realtime();
$dbForProject
->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage))
@@ -552,12 +557,10 @@ App::init()
$project,
$document,
$response,
// Clone the queues used by database, to prevent overwriting the events that are supposed to be triggered in the shutdown hook.
// This is a hack, we should define this listener in the domain layer and allow it to inject it's own dependencies.
clone $queueForEvents,
clone $queueForFunctions,
clone $queueForWebhooks,
clone $queueForRealtime
$queueForEventsClone->from($queueForEvents),
$queueForFunctions->from($queueForEvents),
$queueForWebhooks->from($queueForEvents),
$queueForRealtime->from($queueForEvents)
));
$useCache = $route->getLabel('cache', false);
+27 -3
View File
@@ -85,6 +85,30 @@ App::setResource('localeCodes', function () {
App::setResource('publisher', function (Group $pools) {
return new BrokerPool(publisher: $pools->get('publisher'));
}, ['pools']);
App::setResource('publisherDatabases', function (Publisher $publisher) {
return $publisher;
}, ['publisher']);
App::setResource('publisherFunctions', function (Publisher $publisher) {
return $publisher;
}, ['publisher']);
App::setResource('publisherMigrations', function (Publisher $publisher) {
return $publisher;
}, ['publisher']);
App::setResource('publisherStatsUsage', function (Publisher $publisher) {
return $publisher;
}, ['publisher']);
App::setResource('publisherMails', function (Publisher $publisher) {
return $publisher;
}, ['publisher']);
App::setResource('publisherDeletes', function (Publisher $publisher) {
return $publisher;
}, ['publisher']);
App::setResource('publisherMessaging', function (Publisher $publisher) {
return $publisher;
}, ['publisher']);
App::setResource('publisherWebhooks', function (Publisher $publisher) {
return $publisher;
}, ['publisher']);
App::setResource('queueForMessaging', function (Publisher $publisher) {
return new Messaging($publisher);
}, ['publisher']);
@@ -109,9 +133,6 @@ App::setResource('queueForWebhooks', function (Publisher $publisher) {
App::setResource('queueForRealtime', function () {
return new Realtime();
}, []);
App::setResource('queueForStatsResources', function (Publisher $publisher) {
return new StatsResources($publisher);
}, ['publisher']);
App::setResource('queueForStatsUsage', function (Publisher $publisher) {
return new StatsUsage($publisher);
}, ['publisher']);
@@ -127,6 +148,9 @@ App::setResource('queueForCertificates', function (Publisher $publisher) {
App::setResource('queueForMigrations', function (Publisher $publisher) {
return new Migration($publisher);
}, ['publisher']);
App::setResource('queueForStatsResources', function (Publisher $publisher) {
return new StatsResources($publisher);
}, ['publisher']);
App::setResource('platforms', function (Request $request, Document $console, Document $project) {
$console->setAttribute('platforms', [ // Always allow current host
'$collection' => ID::custom('platforms'),
+32
View File
@@ -247,10 +247,42 @@ Server::setResource('publisher', function (Group $pools) {
return new BrokerPool(publisher: $pools->get('publisher'));
}, ['pools']);
Server::setResource('publisherDatabases', function (BrokerPool $publisher) {
return $publisher;
}, ['publisher']);
Server::setResource('publisherFunctions', function (BrokerPool $publisher) {
return $publisher;
}, ['publisher']);
Server::setResource('publisherMigrations', function (BrokerPool $publisher) {
return $publisher;
}, ['publisher']);
Server::setResource('publisherStatsUsage', function (BrokerPool $publisher) {
return $publisher;
}, ['publisher']);
Server::setResource('publisherMessaging', function (BrokerPool $publisher) {
return $publisher;
}, ['publisher']);
Server::setResource('consumer', function (Group $pools) {
return new BrokerPool(consumer: $pools->get('consumer'));
}, ['pools']);
Server::setResource('consumerDatabases', function (BrokerPool $consumer) {
return $consumer;
}, ['consumer']);
Server::setResource('consumerMigrations', function (BrokerPool $consumer) {
return $consumer;
}, ['consumer']);
Server::setResource('consumerStatsUsage', function (BrokerPool $consumer) {
return $consumer;
}, ['consumer']);
Server::setResource('queueForStatsUsage', function (Publisher $publisher) {
return new StatsUsage($publisher);
}, ['publisher']);
-28
View File
@@ -193,32 +193,4 @@ class Database extends Event
'events' => Event::generateEvents($this->getEvent(), $this->getParams())
];
}
/**
* Clone the database event instance.
*/
public function __clone(): void
{
parent::__clone();
if ($this->database !== null) {
$this->database = clone $this->database;
}
if ($this->row !== null) {
$this->row = clone $this->row;
}
if ($this->table !== null) {
$this->table = clone $this->table;
}
if ($this->document !== null) {
$this->document = clone $this->document;
}
if ($this->collection !== null) {
$this->collection = clone $this->collection;
}
}
}
-23
View File
@@ -639,29 +639,6 @@ class Event
return $events;
}
/**
* Clone the event instance.
*
* Creates a deep copy of the event with all properties,
* including nested objects and arrays.
*/
public function __clone(): void
{
if ($this->project !== null) {
$this->project = clone $this->project;
}
if ($this->user !== null) {
$this->user = clone $this->user;
}
$clonedContext = [];
foreach ($this->context as $key => $document) {
$clonedContext[$key] = clone $document;
}
$this->context = $clonedContext;
}
/**
* Returns the size of the queue.
*
-16
View File
@@ -219,20 +219,4 @@ class Func extends Event
'method' => $this->method,
];
}
/**
* Clone the function event instance.
*/
public function __clone(): void
{
parent::__clone();
if ($this->function !== null) {
$this->function = clone $this->function;
}
if ($this->execution !== null) {
$this->execution = clone $this->execution;
}
}
}
+27 -2
View File
@@ -11,6 +11,7 @@ use Utopia\Database\Exception;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Action;
use Utopia\Queue\Broker\Pool as BrokerPool;
use Utopia\System\System;
use Utopia\Telemetry\Adapter as Telemetry;
use Utopia\Telemetry\Gauge;
@@ -23,6 +24,10 @@ abstract class ScheduleBase extends Action
protected array $schedules = [];
protected BrokerPool $publisher;
protected BrokerPool $publisherMigrations;
protected BrokerPool $publisherFunctions;
protected BrokerPool $publisherMessaging;
private ?Histogram $collectSchedulesTelemetryDuration = null;
private ?Gauge $collectSchedulesTelemetryCount = null;
@@ -34,6 +39,21 @@ abstract class ScheduleBase extends Action
abstract public static function getCollectionId(): string;
abstract protected function enqueueResources(Database $dbForPlatform, callable $getProjectDB): void;
public function __construct()
{
$type = static::getSupportedResource();
$this
->desc("Execute {$type}s scheduled in Appwrite")
->inject('publisher')
->inject('publisherMigrations')
->inject('publisherFunctions')
->inject('publisherMessaging')
->inject('dbForPlatform')
->inject('getProjectDB')
->inject('telemetry')
->callback($this->action(...));
}
protected function updateProjectAccess(Document $project, Database $dbForPlatform): void
{
@@ -51,11 +71,16 @@ abstract class ScheduleBase extends Action
* 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute
* 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutine sleeps until exact time before sending request to worker.
*/
protected function schedule(Database $dbForPlatform, callable $getProjectDB, Telemetry $telemetry): void
public function action(BrokerPool $publisher, BrokerPool $publisherMigrations, BrokerPool $publisherFunctions, BrokerPool $publisherMessaging, Database $dbForPlatform, callable $getProjectDB, Telemetry $telemetry): void
{
Console::title(\ucfirst(static::getSupportedResource()) . ' scheduler V1');
Console::success(APP_NAME . ' ' . \ucfirst(static::getSupportedResource()) . ' scheduler v1 has started');
$this->publisher = $publisher;
$this->publisherMigrations = $publisherMigrations;
$this->publisherFunctions = $publisherFunctions;
$this->publisherMessaging = $publisherMessaging;
$this->scheduleTelemetryCount = $telemetry->createGauge('task.schedule.count');
$this->collectSchedulesTelemetryDuration = $telemetry->createHistogram('task.schedule.collect_schedules.duration', 's');
$this->collectSchedulesTelemetryCount = $telemetry->createGauge('task.schedule.collect_schedules.count');
@@ -87,7 +112,7 @@ abstract class ScheduleBase extends Action
}
}
protected function collectSchedules(Database $dbForPlatform, callable $getProjectDB, string &$lastSyncUpdate): void
private function collectSchedules(Database $dbForPlatform, callable $getProjectDB, string &$lastSyncUpdate): void
{
// If we haven't synced yet, load all active schedules
$initialLoad = $lastSyncUpdate === "0";
@@ -5,15 +5,12 @@ namespace Appwrite\Platform\Tasks;
use Appwrite\Event\Func;
use Swoole\Coroutine as Co;
use Utopia\Database\Database;
use Utopia\Telemetry\Adapter as Telemetry;
class ScheduleExecutions extends ScheduleBase
{
public const UPDATE_TIMER = 3; // seconds
public const ENQUEUE_TIMER = 4; // seconds
protected Func $queueForFunctions;
public static function getName(): string
{
return 'schedule-executions';
@@ -29,27 +26,12 @@ class ScheduleExecutions extends ScheduleBase
return 'executions';
}
public function __construct()
{
$this
->desc('Execute executions scheduled in Appwrite')
->inject('queueForFunctions')
->inject('dbForPlatform')
->inject('getProjectDB')
->inject('telemetry')
->callback($this->action(...));
}
public function action(Func $queueForFunctions, Database $dbForPlatform, callable $getProjectDB, Telemetry $telemetry): void
{
$this->queueForFunctions = $queueForFunctions;
$this->schedule($dbForPlatform, $getProjectDB, $telemetry);
}
protected function enqueueResources(Database $dbForPlatform, callable $getProjectDB): void
{
$intervalEnd = (new \DateTime())->modify('+' . self::ENQUEUE_TIMER . ' seconds');
$queueForFunctions = new Func($this->publisherFunctions);
foreach ($this->schedules as $schedule) {
if (!$schedule['active']) {
$dbForPlatform->deleteDocument(
@@ -75,8 +57,7 @@ class ScheduleExecutions extends ScheduleBase
$this->updateProjectAccess($schedule['project'], $dbForPlatform);
\go(function () use ($schedule, $scheduledAt, $delay, $data) {
$queueForFunctions = clone $this->queueForFunctions;
\go(function () use ($queueForFunctions, $schedule, $scheduledAt, $delay, $data) {
Co::sleep($delay);
$queueForFunctions->setType('schedule')
@@ -7,7 +7,7 @@ use Cron\CronExpression;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Telemetry\Adapter as Telemetry;
use Utopia\Pools\Group;
class ScheduleFunctions extends ScheduleBase
{
@@ -15,7 +15,6 @@ class ScheduleFunctions extends ScheduleBase
public const ENQUEUE_TIMER = 60; // seconds
private ?float $lastEnqueueUpdate = null;
protected Func $queueForFunctions;
public static function getName(): string
{
@@ -32,23 +31,6 @@ class ScheduleFunctions extends ScheduleBase
return 'functions';
}
public function __construct()
{
$this
->desc('Execute functions scheduled in Appwrite')
->inject('queueForFunctions')
->inject('dbForPlatform')
->inject('getProjectDB')
->inject('telemetry')
->callback($this->action(...));
}
public function action(Func $queueForFunctions, Database $dbForPlatform, callable $getProjectDB, Telemetry $telemetry): void
{
$this->queueForFunctions = $queueForFunctions;
$this->schedule($dbForPlatform, $getProjectDB, $telemetry);
}
protected function enqueueResources(Database $dbForPlatform, callable $getProjectDB): void
{
$timerStart = \microtime(true);
@@ -95,8 +77,6 @@ class ScheduleFunctions extends ScheduleBase
foreach ($delayedExecutions as $delay => $schedules) {
\go(function () use ($delay, $schedules, $dbForPlatform) {
$queueForFunctions = clone $this->queueForFunctions;
\sleep($delay); // in seconds
foreach ($schedules as $delayConfig) {
@@ -110,6 +90,8 @@ class ScheduleFunctions extends ScheduleBase
$this->updateProjectAccess($schedule['project'], $dbForPlatform);
$queueForFunctions = new Func($this->publisherFunctions);
$queueForFunctions
->setType('schedule')
->setFunction($schedule['resource'])
@@ -4,15 +4,12 @@ namespace Appwrite\Platform\Tasks;
use Appwrite\Event\Messaging;
use Utopia\Database\Database;
use Utopia\Telemetry\Adapter as Telemetry;
class ScheduleMessages extends ScheduleBase
{
public const UPDATE_TIMER = 3; // seconds
public const ENQUEUE_TIMER = 4; // seconds
protected Messaging $queueForMessaging;
public static function getName(): string
{
return 'schedule-messages';
@@ -28,23 +25,6 @@ class ScheduleMessages extends ScheduleBase
return 'messages';
}
public function __construct()
{
$this
->desc('Execute messages scheduled in Appwrite')
->inject('queueForMessaging')
->inject('dbForPlatform')
->inject('getProjectDB')
->inject('telemetry')
->callback($this->action(...));
}
public function action(Messaging $queueForMessaging, Database $dbForPlatform, callable $getProjectDB, Telemetry $telemetry): void
{
$this->queueForMessaging = $queueForMessaging;
$this->schedule($dbForPlatform, $getProjectDB, $telemetry);
}
protected function enqueueResources(Database $dbForPlatform, callable $getProjectDB): void
{
foreach ($this->schedules as $schedule) {
@@ -60,9 +40,10 @@ class ScheduleMessages extends ScheduleBase
}
\go(function () use ($schedule, $scheduledAt, $dbForPlatform) {
$queueForMessaging = new Messaging($this->publisherMessaging);
$this->updateProjectAccess($schedule['project'], $dbForPlatform);
$queueForMessaging = clone $this->queueForMessaging;
$queueForMessaging
->setType(MESSAGE_SEND_TYPE_EXTERNAL)
->setMessageId($schedule['resourceId'])