mirror of
https://github.com/appwrite/appwrite.git
synced 2026-05-26 13:51:13 +00:00
adds scheduling commit, still need to resolve pools error
This commit is contained in:
1 parent
396ff87fbc
commit
b8aa2faa7b
15 files changed
+364
-68
No files matched your search
@@ -20,6 +20,7 @@ use Appwrite\Platform\Tasks\CalcTierStats;
|
||||
use Appwrite\Platform\Tasks\Upgrade;
|
||||
use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
|
||||
use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
|
||||
use Appwrite\Platform\Tasks\ScheduleMessage;
|
||||
|
||||
class Tasks extends Service
|
||||
{
|
||||
@@ -36,6 +37,7 @@ class Tasks extends Service
|
||||
->addAction(Install::getName(), new Install())
|
||||
->addAction(Upgrade::getName(), new Upgrade())
|
||||
->addAction(Maintenance::getName(), new Maintenance())
|
||||
->addAction(ScheduleMessage::getName(), new ScheduleMessage())
|
||||
->addAction(Schedule::getName(), new Schedule())
|
||||
->addAction(Migrate::getName(), new Migrate())
|
||||
->addAction(SDKs::getName(), new SDKs())
|
||||
|
||||
@@ -85,7 +85,7 @@ class Maintenance extends Action
|
||||
|
||||
private function notifyDeleteUsageStats(int $usageStatsRetentionHourly, Delete $queueForDeletes): void
|
||||
{
|
||||
($queueForDeletes)
|
||||
($queueFor)
|
||||
->setType(DELETE_TYPE_USAGE)
|
||||
->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly))
|
||||
->trigger();
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Tasks;
|
||||
|
||||
use Appwrite\Event\Delete;
|
||||
use Swoole\Timer;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Pools\Group;
|
||||
use Appwrite\Event\Messaging;
|
||||
|
||||
use function Swoole\Coroutine\run;
|
||||
|
||||
class ScheduleMessage extends Action
|
||||
{
|
||||
public const MESSAGE_UPDATE_TIMER = 10; //seconds
|
||||
public const MESSAGE_ENQUEUE_TIMER = 60; //seconds
|
||||
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'schedule-message';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->desc('Execute functions scheduled in Appwrite')
|
||||
->inject('pools')
|
||||
->inject('dbForConsole')
|
||||
->inject('getProjectDB')
|
||||
->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB));
|
||||
}
|
||||
|
||||
/**
|
||||
* 1. Load all documents from 'schedules' collection to create local copy
|
||||
* 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute
|
||||
* 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutime sleeps until exact time before sending request to worker.
|
||||
*/
|
||||
public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void
|
||||
{
|
||||
Console::title('Scheduler V1');
|
||||
Console::success(APP_NAME . ' Scheduler v1 has started');
|
||||
|
||||
$schedules = []; // Local copy of 'schedules' collection
|
||||
$lastSyncUpdate = DateTime::now();
|
||||
|
||||
$limit = 10000;
|
||||
$sum = $limit;
|
||||
$total = 0;
|
||||
$loadStart = \microtime(true);
|
||||
$latestDocument = null;
|
||||
|
||||
while ($sum === $limit) {
|
||||
$paginationQueries = [Query::limit($limit)];
|
||||
if ($latestDocument !== null) {
|
||||
$paginationQueries[] = Query::cursorAfter($latestDocument);
|
||||
}
|
||||
try{
|
||||
|
||||
|
||||
$results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
|
||||
Query::lessThanEqual('schedule', DateTime::formatTz(DateTime::now())),
|
||||
Query::equal('resourceType', ['message']),
|
||||
Query::equal('active', [true]),
|
||||
]));
|
||||
} catch (\Exception $e) {
|
||||
var_dump($e->getTraceAsString());
|
||||
}
|
||||
|
||||
$sum = count($results);
|
||||
$total = $total + $sum;
|
||||
foreach($results as $schedule) {
|
||||
$schedules[$schedule->getId()] = $schedule;
|
||||
}
|
||||
|
||||
$latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null;
|
||||
}
|
||||
|
||||
$pools->reclaim();
|
||||
|
||||
Console::success("{$total} message were loaded in " . (microtime(true) - $loadStart) . " seconds");
|
||||
|
||||
Console::success("Starting timers at " . DateTime::now());
|
||||
|
||||
run(
|
||||
function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $pools) {
|
||||
/**
|
||||
* The timer synchronize $schedules copy with database collection.
|
||||
*/
|
||||
Timer::tick(self::MESSAGE_UPDATE_TIMER * 1000, function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $pools) {
|
||||
$time = DateTime::now();
|
||||
$timerStart = \microtime(true);
|
||||
|
||||
$limit = 1000;
|
||||
$sum = $limit;
|
||||
$total = 0;
|
||||
$latestDocument = null;
|
||||
|
||||
Console::log("Sync tick: Running at $time");
|
||||
|
||||
while ($sum === $limit) {
|
||||
$paginationQueries = [Query::limit($limit)];
|
||||
if ($latestDocument !== null) {
|
||||
$paginationQueries[] = Query::cursorAfter($latestDocument);
|
||||
}
|
||||
$results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
|
||||
Query::lessThanEqual('schedule', DateTime::formatTz(DateTime::now())),
|
||||
Query::equal('resourceType', ['message']),
|
||||
Query::equal('active', [true]),
|
||||
]));
|
||||
$sum = \count($results);
|
||||
$total = $total + $sum;
|
||||
foreach ($results as $schedule) {
|
||||
$schedules[$schedule->getId()] = $schedule;
|
||||
}
|
||||
|
||||
$latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null;
|
||||
}
|
||||
|
||||
$lastSyncUpdate = $time;
|
||||
$timerEnd = \microtime(true);
|
||||
|
||||
$pools->reclaim();
|
||||
|
||||
Console::log("Sync tick: {$total} schedules were updated in " . ($timerEnd - $timerStart) . " seconds");
|
||||
});
|
||||
|
||||
/**
|
||||
* The timer to prepare soon-to-execute schedules.
|
||||
*/
|
||||
$enqueueMessages = function () use (&$schedules, $pools, $dbForConsole) {
|
||||
foreach ($schedules as $scheduleId => $schedule) {
|
||||
\go(function () use ($schedules, $schedule, $pools, $dbForConsole) {
|
||||
$queue = $pools->get('queue')->pop();
|
||||
$connection = $queue->getResource();
|
||||
$queueForMessaging = new Messaging($connection);
|
||||
$queueForDeletes = new Delete($connection);
|
||||
$project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId'));
|
||||
$queueForMessaging
|
||||
->setMessageId($schedule->getAttribute('resourceId'))
|
||||
->setProject($project)
|
||||
->trigger();
|
||||
$schedule->setAttribute('active', false);
|
||||
$dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule);
|
||||
|
||||
$queueForDeletes
|
||||
->setType(DELETE_TYPE_SCHEDULES)
|
||||
->setDocument($schedule);
|
||||
|
||||
$queue->reclaim();
|
||||
unset($schedules[$schedule->getId()]);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
Timer::tick(self::MESSAGE_ENQUEUE_TIMER * 1000, fn () => $enqueueMessages());
|
||||
$enqueueMessages();
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -148,7 +148,7 @@ class Deletes extends Action
|
||||
$this->deleteCacheByDate($project, $getProjectDB, $datetime);
|
||||
break;
|
||||
case DELETE_TYPE_SCHEDULES:
|
||||
$this->deleteSchedules($dbForConsole, $getProjectDB, $datetime);
|
||||
$this->deleteSchedules($dbForConsole, $getProjectDB, $datetime, $document);
|
||||
break;
|
||||
case DELETE_TYPE_TOPIC:
|
||||
$this->deleteTopic($project, $getProjectDB, $document);
|
||||
@@ -167,13 +167,13 @@ class Deletes extends Action
|
||||
* @throws Authorization
|
||||
* @throws Throwable
|
||||
*/
|
||||
private function deleteSchedules(Database $dbForConsole, callable $getProjectDB, string $datetime): void
|
||||
private function deleteSchedules(Database $dbForConsole, callable $getProjectDB, string $datetime, ?Document $document = null): void
|
||||
{
|
||||
$this->listByGroup(
|
||||
'schedules',
|
||||
[
|
||||
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
|
||||
Query::equal('resourceType', ['function']),
|
||||
Query::equal('resourceType', [$document ?? $document->getAttribute('resourceType')]),
|
||||
Query::lessThanEqual('resourceUpdatedAt', $datetime),
|
||||
Query::equal('active', [false]),
|
||||
],
|
||||
|
||||
@@ -106,7 +106,6 @@ class Messaging extends Action
|
||||
$targets = $dbForProject->find('targets', [Query::equal('$id', $targetsId)]);
|
||||
$recipients = \array_merge($recipients, $targets);
|
||||
}
|
||||
|
||||
$primaryProvider = $dbForProject->findOne('providers', [
|
||||
Query::equal('enabled', [true]),
|
||||
Query::equal('type', [$recipients[0]->getAttribute('providerType')]),
|
||||
@@ -155,7 +154,6 @@ class Messaging extends Action
|
||||
|
||||
$providers[] = $provider;
|
||||
$identifiers = $identifiersByProviderId[$providerId];
|
||||
|
||||
$adapter = match ($provider->getAttribute('type')) {
|
||||
MESSAGE_TYPE_SMS => $this->sms($provider),
|
||||
MESSAGE_TYPE_PUSH => $this->push($provider),
|
||||
|
||||
Reference in new issue
Block a user