Merge branch '1.5.x' of https://github.com/appwrite/appwrite into feat-mfa

This commit is contained in:
Torsten Dittmann
2024-01-22 14:45:07 +01:00
41 changed files with 1609 additions and 587 deletions
+6 -6
View File
@@ -2,26 +2,26 @@
namespace Appwrite\Enum;
enum MessageStatus: string
class MessageStatus
{
/**
* Message that is not ready to be sent
*/
case Draft = 'draft';
public const DRAFT = 'draft';
/**
* Scheduled to be sent for a later time
*/
case Scheduled = 'scheduled';
public const SCHEDULED = 'scheduled';
/**
* Picked up by the worker and starting to send
*/
case Processing = 'processing';
public const PROCESSING = 'processing';
/**
* Sent without errors
*/
case Sent = 'sent';
public const SENT = 'sent';
/**
* Sent with some errors
*/
case Failed = 'failed';
public const FAILED = 'failed';
}
+4
View File
@@ -267,6 +267,10 @@ class Exception extends \Exception
public const MESSAGE_TARGET_NOT_EMAIL = 'message_target_not_email';
public const MESSAGE_TARGET_NOT_SMS = 'message_target_not_sms';
public const MESSAGE_TARGET_NOT_PUSH = 'message_target_not_push';
public const MESSAGE_MISSING_SCHEDULE = 'message_missing_schedule';
/** Schedules */
public const SCHEDULE_NOT_FOUND = 'schedule_not_found';
protected string $type = '';
+28 -26
View File
@@ -2,26 +2,27 @@
namespace Appwrite\Platform\Services;
use Utopia\Platform\Service;
use Appwrite\Platform\Tasks\CalcTierStats;
use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
use Appwrite\Platform\Tasks\DevGenerateTranslations;
use Appwrite\Platform\Tasks\Doctor;
use Appwrite\Platform\Tasks\GetMigrationStats;
use Appwrite\Platform\Tasks\Hamster;
use Appwrite\Platform\Tasks\Install;
use Appwrite\Platform\Tasks\Maintenance;
use Appwrite\Platform\Tasks\Migrate;
use Appwrite\Platform\Tasks\Schedule;
use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
use Appwrite\Platform\Tasks\SDKs;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\SSL;
use Appwrite\Platform\Tasks\Hamster;
use Appwrite\Platform\Tasks\ScheduleFunctions;
use Appwrite\Platform\Tasks\ScheduleMessages;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\Upgrade;
use Appwrite\Platform\Tasks\Usage;
use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
use Appwrite\Platform\Tasks\VolumeSync;
use Appwrite\Platform\Tasks\CalcTierStats;
use Appwrite\Platform\Tasks\Upgrade;
use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
use Appwrite\Platform\Tasks\DevGenerateTranslations;
use Appwrite\Platform\Tasks\GetMigrationStats;
use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
use Utopia\Platform\Service;
class Tasks extends Service
{
@@ -29,25 +30,26 @@ class Tasks extends Service
{
$this->type = self::TYPE_CLI;
$this
->addAction(Version::getName(), new Version())
->addAction(Usage::getName(), new Usage())
->addAction(Vars::getName(), new Vars())
->addAction(SSL::getName(), new SSL())
->addAction(Hamster::getName(), new Hamster())
->addAction(Doctor::getName(), new Doctor())
->addAction(Install::getName(), new Install())
->addAction(Upgrade::getName(), new Upgrade())
->addAction(Maintenance::getName(), new Maintenance())
->addAction(Schedule::getName(), new Schedule())
->addAction(Migrate::getName(), new Migrate())
->addAction(SDKs::getName(), new SDKs())
->addAction(VolumeSync::getName(), new VolumeSync())
->addAction(Specs::getName(), new Specs())
->addAction(CalcTierStats::getName(), new CalcTierStats())
->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
->addAction(GetMigrationStats::getName(), new GetMigrationStats())
->addAction(DevGenerateTranslations::getName(), new DevGenerateTranslations())
->addAction(Doctor::getName(), new Doctor())
->addAction(GetMigrationStats::getName(), new GetMigrationStats())
->addAction(Hamster::getName(), new Hamster())
->addAction(Install::getName(), new Install())
->addAction(Maintenance::getName(), new Maintenance())
->addAction(Migrate::getName(), new Migrate())
->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
->addAction(SDKs::getName(), new SDKs())
->addAction(SSL::getName(), new SSL())
->addAction(ScheduleFunctions::getName(), new ScheduleFunctions())
->addAction(ScheduleMessages::getName(), new ScheduleMessages())
->addAction(Specs::getName(), new Specs())
->addAction(Upgrade::getName(), new Upgrade())
->addAction(Usage::getName(), new Usage())
->addAction(Vars::getName(), new Vars())
->addAction(Version::getName(), new Version())
->addAction(VolumeSync::getName(), new VolumeSync())
;
}
+6 -6
View File
@@ -22,16 +22,16 @@ class Workers extends Service
$this->type = self::TYPE_WORKER;
$this
->addAction(Audits::getName(), new Audits())
->addAction(Webhooks::getName(), new Webhooks())
->addAction(Mails::getName(), new Mails())
->addAction(Messaging::getName(), new Messaging())
->addAction(Builds::getName(), new Builds())
->addAction(Certificates::getName(), new Certificates())
->addAction(Databases::getName(), new Databases())
->addAction(Functions::getName(), new Functions())
->addAction(Builds::getName(), new Builds())
->addAction(Deletes::getName(), new Deletes())
->addAction(Migrations::getName(), new Migrations())
->addAction(Functions::getName(), new Functions())
->addAction(Hamster::getName(), new Hamster())
->addAction(Mails::getName(), new Mails())
->addAction(Messaging::getName(), new Messaging())
->addAction(Migrations::getName(), new Migrations())
->addAction(Webhooks::getName(), new Webhooks())
;
}
+6 -6
View File
@@ -61,7 +61,7 @@ class Maintenance extends Action
private function notifyDeleteExecutionLogs(int $interval, Delete $queueForDeletes): void
{
($queueForDeletes)
$queueForDeletes
->setType(DELETE_TYPE_EXECUTIONS)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
@@ -69,7 +69,7 @@ class Maintenance extends Action
private function notifyDeleteAbuseLogs(int $interval, Delete $queueForDeletes): void
{
($queueForDeletes)
$queueForDeletes
->setType(DELETE_TYPE_ABUSE)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
@@ -77,7 +77,7 @@ class Maintenance extends Action
private function notifyDeleteAuditLogs(int $interval, Delete $queueForDeletes): void
{
($queueForDeletes)
$queueForDeletes
->setType(DELETE_TYPE_AUDIT)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
@@ -85,7 +85,7 @@ class Maintenance extends Action
private function notifyDeleteUsageStats(int $usageStatsRetentionHourly, Delete $queueForDeletes): void
{
($queueForDeletes)
$queueForDeletes
->setType(DELETE_TYPE_USAGE)
->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly))
->trigger();
@@ -93,7 +93,7 @@ class Maintenance extends Action
private function notifyDeleteConnections(Delete $queueForDeletes): void
{
($queueForDeletes)
$queueForDeletes
->setType(DELETE_TYPE_REALTIME)
->setDatetime(DateTime::addSeconds(new \DateTime(), -60))
->trigger();
@@ -101,7 +101,7 @@ class Maintenance extends Action
private function notifyDeleteExpiredSessions(Delete $queueForDeletes): void
{
($queueForDeletes)
$queueForDeletes
->setType(DELETE_TYPE_SESSIONS)
->trigger();
}
-244
View File
@@ -1,244 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Cron\CronExpression;
use Swoole\Timer;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Pools\Group;
use Appwrite\Event\Func;
use function Swoole\Coroutine\run;
class Schedule extends Action
{
public const FUNCTION_UPDATE_TIMER = 10; //seconds
public const FUNCTION_ENQUEUE_TIMER = 60; //seconds
public static function getName(): string
{
return 'schedule';
}
public function __construct()
{
$this
->desc('Execute functions scheduled in Appwrite')
->inject('pools')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB));
}
/**
* 1. Load all documents from 'schedules' collection to create local copy
* 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute
* 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutime sleeps until exact time before sending request to worker.
*/
public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void
{
Console::title('Scheduler V1');
Console::success(APP_NAME . ' Scheduler v1 has started');
/**
* Extract only nessessary attributes to lower memory used.
*
* @var Document $schedule
* @return array
*/
$getSchedule = function (Document $schedule) use ($dbForConsole, $getProjectDB): array {
$project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId'));
$function = $getProjectDB($project)->getDocument('functions', $schedule->getAttribute('resourceId'));
return [
'resourceId' => $schedule->getAttribute('resourceId'),
'schedule' => $schedule->getAttribute('schedule'),
'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'),
'project' => $project, // TODO: @Meldiron Send only ID to worker to reduce memory usage here
'function' => $function, // TODO: @Meldiron Send only ID to worker to reduce memory usage here
];
};
$schedules = []; // Local copy of 'schedules' collection
$lastSyncUpdate = DateTime::now();
$limit = 10000;
$sum = $limit;
$total = 0;
$loadStart = \microtime(true);
$latestDocument = null;
while ($sum === $limit) {
$paginationQueries = [Query::limit($limit)];
if ($latestDocument !== null) {
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
$results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', ['function']),
Query::equal('active', [true]),
]));
$sum = count($results);
$total = $total + $sum;
foreach ($results as $document) {
try {
$schedules[$document['resourceId']] = $getSchedule($document);
} catch (\Throwable $th) {
Console::error("Failed to load schedule for project {$document['projectId']} and function {$document['resourceId']}");
Console::error($th->getMessage());
}
}
$latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null;
}
$pools->reclaim();
Console::success("{$total} functions were loaded in " . (microtime(true) - $loadStart) . " seconds");
Console::success("Starting timers at " . DateTime::now());
run(
function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule, $pools) {
/**
* The timer synchronize $schedules copy with database collection.
*/
Timer::tick(self::FUNCTION_UPDATE_TIMER * 1000, function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule, $pools) {
$time = DateTime::now();
$timerStart = \microtime(true);
$limit = 1000;
$sum = $limit;
$total = 0;
$latestDocument = null;
Console::log("Sync tick: Running at $time");
while ($sum === $limit) {
$paginationQueries = [Query::limit($limit)];
if ($latestDocument !== null) {
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
$results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', ['function']),
Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate),
]));
$sum = count($results);
$total = $total + $sum;
foreach ($results as $document) {
$localDocument = $schedules[$document['resourceId']] ?? null;
$org = $localDocument !== null ? strtotime($localDocument['resourceUpdatedAt']) : null;
$new = strtotime($document['resourceUpdatedAt']);
if ($document['active'] === false) {
Console::info("Removing: {$document['resourceId']}");
unset($schedules[$document['resourceId']]);
} elseif ($new !== $org) {
Console::info("Updating: {$document['resourceId']}");
$schedules[$document['resourceId']] = $getSchedule($document);
}
}
$latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null;
}
$lastSyncUpdate = $time;
$timerEnd = \microtime(true);
$pools->reclaim();
Console::log("Sync tick: {$total} schedules were updated in " . ($timerEnd - $timerStart) . " seconds");
});
/**
* The timer to prepare soon-to-execute schedules.
*/
$lastEnqueueUpdate = null;
$enqueueFunctions = function () use (&$schedules, $lastEnqueueUpdate, $pools) {
$timerStart = \microtime(true);
$time = DateTime::now();
$enqueueDiff = $lastEnqueueUpdate === null ? 0 : $timerStart - $lastEnqueueUpdate;
$timeFrame = DateTime::addSeconds(new \DateTime(), self::FUNCTION_ENQUEUE_TIMER - $enqueueDiff);
Console::log("Enqueue tick: started at: $time (with diff $enqueueDiff)");
$total = 0;
$delayedExecutions = []; // Group executions with same delay to share one coroutine
foreach ($schedules as $key => $schedule) {
$cron = new CronExpression($schedule['schedule']);
$nextDate = $cron->getNextRunDate();
$next = DateTime::format($nextDate);
$currentTick = $next < $timeFrame;
if (!$currentTick) {
continue;
}
$total++;
$promiseStart = \time(); // in seconds
$executionStart = $nextDate->getTimestamp(); // in seconds
$delay = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued
if (!isset($delayedExecutions[$delay])) {
$delayedExecutions[$delay] = [];
}
$delayedExecutions[$delay][] = $key;
}
foreach ($delayedExecutions as $delay => $scheduleKeys) {
\go(function () use ($delay, $schedules, $scheduleKeys, $pools) {
\sleep($delay); // in seconds
$queue = $pools->get('queue')->pop();
$connection = $queue->getResource();
foreach ($scheduleKeys as $scheduleKey) {
// Ensure schedule was not deleted
if (!isset($schedules[$scheduleKey])) {
return;
}
$schedule = $schedules[$scheduleKey];
$functions = new Func($connection);
$functions
->setType('schedule')
->setFunction($schedule['function'])
->setMethod('POST')
->setPath('/')
->setProject($schedule['project'])
->trigger();
}
$queue->reclaim();
});
}
$timerEnd = \microtime(true);
$lastEnqueueUpdate = $timerStart;
Console::log("Enqueue tick: {$total} executions were enqueued in " . ($timerEnd - $timerStart) . " seconds");
};
Timer::tick(self::FUNCTION_ENQUEUE_TIMER * 1000, fn() => $enqueueFunctions());
$enqueueFunctions();
}
);
}
}
@@ -0,0 +1,190 @@
<?php
namespace Appwrite\Platform\Tasks;
use Cron\CronExpression;
use Swoole\Timer;
use Utopia\App;
use Utopia\Database\Exception;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Pools\Group;
use Appwrite\Event\Func;
use function Swoole\Coroutine\run;
abstract class ScheduleBase extends Action
{
protected const UPDATE_TIMER = 10; //seconds
protected const ENQUEUE_TIMER = 60; //seconds
protected array $schedules = [];
abstract public static function getName(): string;
abstract public static function getSupportedResource(): string;
abstract protected function enqueueResources(
Group $pools,
Database $dbForConsole
);
public function __construct()
{
$type = static::getSupportedResource();
$this
->desc("Execute {$type}s scheduled in Appwrite")
->inject('pools')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn(Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB));
}
/**
* 1. Load all documents from 'schedules' collection to create local copy
* 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute
* 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutine sleeps until exact time before sending request to worker.
*/
public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void
{
Console::title(\ucfirst(static::getSupportedResource()) . ' scheduler V1');
Console::success(APP_NAME . ' ' . \ucfirst(static::getSupportedResource()) . ' scheduler v1 has started');
/**
* Extract only necessary attributes to lower memory used.
*
* @return array
* @throws Exception
* @var Document $schedule
*/
$getSchedule = function (Document $schedule) use ($dbForConsole, $getProjectDB): array {
$project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId'));
$resource = $getProjectDB($project)->getDocument(
$schedule->getAttribute('resourceCollection'),
$schedule->getAttribute('resourceId')
);
return [
'$id' => $schedule->getId(),
'resourceId' => $schedule->getAttribute('resourceId'),
'schedule' => $schedule->getAttribute('schedule'),
'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'),
'project' => $project, // TODO: @Meldiron Send only ID to worker to reduce memory usage here
'resource' => $resource, // TODO: @Meldiron Send only ID to worker to reduce memory usage here
];
};
$lastSyncUpdate = DateTime::now();
$limit = 10_000;
$sum = $limit;
$total = 0;
$loadStart = \microtime(true);
$latestDocument = null;
while ($sum === $limit) {
$paginationQueries = [Query::limit($limit)];
if ($latestDocument) {
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
$results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', [static::getSupportedResource()]),
Query::equal('active', [true]),
]));
$sum = \count($results);
$total = $total + $sum;
foreach ($results as $document) {
try {
$this->schedules[$document['resourceId']] = $getSchedule($document);
} catch (\Throwable $th) {
Console::error("Failed to load schedule for project {$document['projectId']} {$document['resourceCollection']} {$document['resourceId']}");
Console::error($th->getMessage());
}
}
$latestDocument = \end($results);
}
$pools->reclaim();
Console::success("{$total} resources were loaded in " . (\microtime(true) - $loadStart) . " seconds");
Console::success("Starting timers at " . DateTime::now());
run(function () use ($dbForConsole, &$lastSyncUpdate, $getSchedule, $pools) {
/**
* The timer synchronize $schedules copy with database collection.
*/
Timer::tick(static::UPDATE_TIMER * 1000, function () use ($dbForConsole, &$lastSyncUpdate, $getSchedule, $pools) {
$time = DateTime::now();
$timerStart = \microtime(true);
$limit = 1000;
$sum = $limit;
$total = 0;
$latestDocument = null;
Console::log("Sync tick: Running at $time");
while ($sum === $limit) {
$paginationQueries = [Query::limit($limit)];
if ($latestDocument) {
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
$results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', [static::getSupportedResource()]),
Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate),
]));
$sum = count($results);
$total = $total + $sum;
foreach ($results as $document) {
$localDocument = $schedules[$document['resourceId']] ?? null;
// Check if resource has been updated since last sync
$org = $localDocument !== null ? \strtotime($localDocument['resourceUpdatedAt']) : null;
$new = \strtotime($document['resourceUpdatedAt']);
if (!$document['active']) {
Console::info("Removing: {$document['resourceId']}");
unset($this->schedules[$document['resourceId']]);
} elseif ($new !== $org) {
Console::info("Updating: {$document['resourceId']}");
$this->schedules[$document['resourceId']] = $getSchedule($document);
}
}
$latestDocument = \end($results);
}
$lastSyncUpdate = $time;
$timerEnd = \microtime(true);
$pools->reclaim();
Console::log("Sync tick: {$total} schedules were updated in " . ($timerEnd - $timerStart) . " seconds");
});
Timer::tick(
static::ENQUEUE_TIMER * 1000,
fn() => $this->enqueueResources($pools, $dbForConsole)
);
$this->enqueueResources($pools, $dbForConsole);
});
}
}
@@ -0,0 +1,104 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Event\Func;
use Cron\CronExpression;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Pools\Group;
class ScheduleFunctions extends ScheduleBase
{
public const UPDATE_TIMER = 10; // seconds
public const ENQUEUE_TIMER = 60; // seconds
private ?float $lastEnqueueUpdate = null;
public static function getName(): string
{
return 'schedule-functions';
}
public static function getSupportedResource(): string
{
return 'function';
}
protected function enqueueResources(Group $pools, Database $dbForConsole): void
{
$timerStart = \microtime(true);
$time = DateTime::now();
$enqueueDiff = $this->lastEnqueueUpdate === null ? 0 : $timerStart - $this->lastEnqueueUpdate;
$timeFrame = DateTime::addSeconds(new \DateTime(), static::ENQUEUE_TIMER - $enqueueDiff);
Console::log("Enqueue tick: started at: $time (with diff $enqueueDiff)");
$total = 0;
$delayedExecutions = []; // Group executions with same delay to share one coroutine
foreach ($this->schedules as $key => $schedule) {
$cron = new CronExpression($schedule['schedule']);
$nextDate = $cron->getNextRunDate();
$next = DateTime::format($nextDate);
$currentTick = $next < $timeFrame;
if (!$currentTick) {
continue;
}
$total++;
$promiseStart = \time(); // in seconds
$executionStart = $nextDate->getTimestamp(); // in seconds
$delay = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued
if (!isset($delayedExecutions[$delay])) {
$delayedExecutions[$delay] = [];
}
$delayedExecutions[$delay][] = $key;
}
foreach ($delayedExecutions as $delay => $scheduleKeys) {
\go(function () use ($delay, $scheduleKeys, $pools) {
\sleep($delay); // in seconds
$queue = $pools->get('queue')->pop();
$connection = $queue->getResource();
foreach ($scheduleKeys as $scheduleKey) {
// Ensure schedule was not deleted
if (!\array_key_exists($scheduleKey, $this->schedules)) {
return;
}
$schedule = $this->schedules[$scheduleKey];
$queueForFunctions = new Func($connection);
$queueForFunctions
->setType('schedule')
->setFunction($schedule['resource'])
->setMethod('POST')
->setPath('/')
->setProject($schedule['project'])
->trigger();
}
$queue->reclaim();
});
}
$timerEnd = \microtime(true);
// TODO: This was a bug before because it wasn't passed by reference, enabling it breaks scheduling
//$this->lastEnqueueUpdate = $timerStart;
Console::log("Enqueue tick: {$total} executions were enqueued in " . ($timerEnd - $timerStart) . " seconds");
}
}
@@ -0,0 +1,71 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Event\Delete;
use Swoole\Timer;
use Utopia\Database\Document;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Pools\Group;
use Appwrite\Event\Messaging;
use function Swoole\Coroutine\run;
class ScheduleMessages extends ScheduleBase
{
public const UPDATE_TIMER = 10; // seconds
public const ENQUEUE_TIMER = 60; // seconds
public static function getName(): string
{
return 'schedule-messages';
}
public static function getSupportedResource(): string
{
return 'message';
}
protected function enqueueResources(Group $pools, Database $dbForConsole): void
{
foreach ($this->schedules as $schedule) {
$now = DateTime::now();
$scheduledAt = DateTime::formatTz($schedule['schedule']);
if ($scheduledAt > $now) {
continue;
}
\go(function () use ($schedule, $pools, $dbForConsole) {
$queue = $pools->get('queue')->pop();
$connection = $queue->getResource();
$queueForMessaging = new Messaging($connection);
$queueForDeletes = new Delete($connection);
$queueForMessaging
->setMessageId($schedule['resourceId'])
->setProject($schedule['project'])
->trigger();
$dbForConsole->updateDocument(
'schedules',
$schedule['$id'],
new Document(['active' => false])
);
$queueForDeletes
->setType(DELETE_TYPE_SCHEDULES)
->setDocument($schedule)
->trigger();
$queue->reclaim();
unset($this->schedules[$schedule['resourceId']]);
});
}
}
}
+65 -10
View File
@@ -20,6 +20,7 @@ use Utopia\Database\Exception\Authorization;
use Utopia\Database\Exception\Conflict;
use Utopia\Database\Exception\Restricted;
use Utopia\Database\Exception\Structure;
use Utopia\Database\Exception as DatabaseException;
use Utopia\Database\Query;
use Utopia\Logger\Log;
use Utopia\Platform\Action;
@@ -153,11 +154,14 @@ class Deletes extends Action
$this->deleteCacheByDate($project, $getProjectDB, $datetime);
break;
case DELETE_TYPE_SCHEDULES:
$this->deleteSchedules($dbForConsole, $getProjectDB, $datetime);
$this->deleteSchedules($dbForConsole, $getProjectDB, $datetime, $document);
break;
case DELETE_TYPE_TOPIC:
$this->deleteTopic($project, $getProjectDB, $document);
break;
case DELETE_TYPE_TARGET:
$this->deleteTarget($project, $getProjectDB, $document);
break;
default:
throw new \Exception('No delete operation for type: ' . \strval($type));
break;
@@ -168,17 +172,21 @@ class Deletes extends Action
* @param Database $dbForConsole
* @param callable $getProjectDB
* @param string $datetime
* @param Document|null $document
* @return void
* @throws Authorization
* @throws Throwable
* @throws Conflict
* @throws Restricted
* @throws Structure
* @throws DatabaseException
*/
private function deleteSchedules(Database $dbForConsole, callable $getProjectDB, string $datetime): void
private function deleteSchedules(Database $dbForConsole, callable $getProjectDB, string $datetime, ?Document $document = null): void
{
$this->listByGroup(
'schedules',
[
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', ['function']),
Query::equal('resourceType', [$document->getAttribute('resourceType')]),
Query::lessThanEqual('resourceUpdatedAt', $datetime),
Query::equal('active', [false]),
],
@@ -192,11 +200,22 @@ class Deletes extends Action
return;
}
$function = $getProjectDB($project)->getDocument('functions', $document->getAttribute('resourceId'));
$resource = $getProjectDB($project)->getDocument(
$document->getAttribute('resourceCollection'),
$document->getAttribute('resourceId')
);
if ($function->isEmpty()) {
$delete = true;
switch ($document->getAttribute('resourceType')) {
case 'function':
$delete = $resource->isEmpty();
break;
}
if ($delete) {
$dbForConsole->deleteDocument('schedules', $document->getId());
Console::success('Deleting schedule for function ' . $document->getAttribute('resourceId'));
Console::success('Deleting schedule for ' . $document->getAttribute('resourceType') . ' ' . $document->getAttribute('resourceId'));
}
}
);
@@ -221,6 +240,35 @@ class Deletes extends Action
], $dbForProject);
}
/**
* @param Document $project
* @param callable $getProjectDB
* @param Document $target
* @throws Exception
*/
protected function deleteTarget(Document $project, callable $getProjectDB, Document $target)
{
/** @var Database */
$dbForProject = $getProjectDB($project);
// Delete subscribers and decrement topic counts
$this->deleteByGroup(
'subscribers',
[
Query::equal('targetInternalId', [$target->getInternalId()])
],
$dbForProject,
function (Document $subscriber) use ($dbForProject) {
$topicId = $subscriber->getAttribute('topicId');
$topicInternalId = $subscriber->getAttribute('topicInternalId');
$topic = $dbForProject->getDocument('topics', $topicId);
if (!$topic->isEmpty() && $topic->getInternalId() === $topicInternalId) {
$dbForProject->decreaseDocumentAttribute('topics', $topicId, 'total', min: 0);
}
}
);
}
/**
* @param Document $project
* @param callable $getProjectDB
@@ -563,9 +611,16 @@ class Deletes extends Action
], $dbForProject);
// Delete targets
$this->deleteByGroup('targets', [
Query::equal('userInternalId', [$userInternalId])
], $dbForProject);
$this->listByGroup(
'targets',
[
Query::equal('userInternalId', [$userInternalId])
],
$dbForProject,
function (Document $target) use ($getProjectDB, $project) {
$this->deleteTarget($project, $getProjectDB, $target);
}
);
}
/**
+5 -1
View File
@@ -29,7 +29,7 @@ class Mails extends Action
->inject('message')
->inject('register')
->inject('log')
->callback(fn(Message $message, Registry $register, Log $log) => $this->action($message, $register, $log));
->callback(fn (Message $message, Registry $register, Log $log) => $this->action($message, $register, $log));
}
/**
@@ -57,9 +57,13 @@ class Mails extends Action
$log->addTag('type', empty($smtp) ? 'cloud' : 'smtp');
$protocol = App::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https';
$hostname = App::getEnv('_APP_DOMAIN');
$recipient = $payload['recipient'];
$subject = $payload['subject'];
$variables = $payload['variables'];
$variables['host'] = $protocol . '://' . $hostname;
$name = $payload['name'];
$body = $payload['body'];
+93 -40
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Enum\MessageStatus;
use Appwrite\Extend\Exception;
use Utopia\App;
use Utopia\CLI\Console;
@@ -38,7 +39,7 @@ class Messaging extends Action
{
public static function getName(): string
{
return "messaging";
return 'messaging';
}
/**
@@ -69,10 +70,13 @@ class Messaging extends Action
throw new \Exception('Payload not found.');
}
if (!\is_null($payload['message']) && !\is_null($payload['recipients'])) {
if ($payload['providerType'] === MESSAGE_TYPE_SMS) {
$this->processInternalSMSMessage($log, new Document($payload['message']), $payload['recipients']);
}
if (
!\is_null($payload['message'])
&& !\is_null($payload['recipients'])
&& $payload['providerType'] === MESSAGE_TYPE_SMS
) {
// Message was triggered internally
$this->processInternalSMSMessage($log, new Document($payload['message']), $payload['recipients']);
} else {
$message = $dbForProject->getDocument('messages', $payload['messageId']);
@@ -82,85 +86,124 @@ class Messaging extends Action
private function processMessage(Database $dbForProject, Document $message): void
{
$topicsId = $message->getAttribute('topics', []);
$targetsId = $message->getAttribute('targets', []);
$usersId = $message->getAttribute('users', []);
$topicIds = $message->getAttribute('topics', []);
$targetIds = $message->getAttribute('targets', []);
$userIds = $message->getAttribute('users', []);
/**
* @var Document[] $recipients
* @var array<Document> $recipients
*/
$recipients = [];
if (\count($topicsId) > 0) {
$topics = $dbForProject->find('topics', [Query::equal('$id', $topicsId)]);
if (\count($topicIds) > 0) {
$topics = $dbForProject->find('topics', [
Query::equal('$id', $topicIds),
Query::limit(\count($topicIds)),
]);
foreach ($topics as $topic) {
$targets = \array_filter($topic->getAttribute('targets'), fn(Document $target) => $target->getAttribute('providerType') === $message->getAttribute('providerType'));
$targets = \array_filter($topic->getAttribute('targets'), fn(Document $target) =>
$target->getAttribute('providerType') === $message->getAttribute('providerType'));
$recipients = \array_merge($recipients, $targets);
}
}
if (\count($usersId) > 0) {
$users = $dbForProject->find('users', [Query::equal('$id', $usersId)]);
if (\count($userIds) > 0) {
$users = $dbForProject->find('users', [
Query::equal('$id', $userIds),
Query::limit(\count($userIds)),
]);
foreach ($users as $user) {
$targets = \array_filter($user->getAttribute('targets'), fn(Document $target) => $target->getAttribute('providerType') === $message->getAttribute('providerType'));
$targets = \array_filter($user->getAttribute('targets'), fn(Document $target) =>
$target->getAttribute('providerType') === $message->getAttribute('providerType'));
$recipients = \array_merge($recipients, $targets);
}
}
if (\count($targetsId) > 0) {
$targets = $dbForProject->find('targets', [Query::equal('$id', $targetsId)]);
if (\count($targetIds) > 0) {
$targets = $dbForProject->find('targets', [
Query::equal('$id', $targetIds),
Query::limit(\count($targetIds)),
]);
$targets = \array_filter($targets, fn(Document $target) =>
$target->getAttribute('providerType') === $message->getAttribute('providerType'));
$recipients = \array_merge($recipients, $targets);
}
$primaryProvider = $dbForProject->findOne('providers', [
if (empty($recipients)) {
$dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([
'status' => MessageStatus::FAILED,
'deliveryErrors' => ['No valid recipients found.']
]));
Console::warning('No valid recipients found.');
return;
}
$fallback = $dbForProject->findOne('providers', [
Query::equal('enabled', [true]),
Query::equal('type', [$recipients[0]->getAttribute('providerType')]),
]);
if ($fallback === false || $fallback->isEmpty()) {
$dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([
'status' => MessageStatus::FAILED,
'deliveryErrors' => ['No fallback provider found.']
]));
Console::warning('No fallback provider found.');
return;
}
/**
* @var array<string, array<string>> $identifiersByProviderId
* @var array<string, array<string>> $identifiers
*/
$identifiersByProviderId = [];
$identifiers = [];
/**
* @var Document[] $providers
*/
$providers = [
$primaryProvider->getId() => $primaryProvider
$fallback->getId() => $fallback
];
foreach ($recipients as $recipient) {
$providerId = $recipient->getAttribute('providerId');
if (!$providerId && $primaryProvider instanceof Document && !$primaryProvider->isEmpty()) {
$providerId = $primaryProvider->getId();
if (
!$providerId
&& $fallback instanceof Document
&& !$fallback->isEmpty()
&& $fallback->getAttribute('enabled')
) {
$providerId = $fallback->getId();
}
if ($providerId) {
if (!isset($identifiersByProviderId[$providerId])) {
$identifiersByProviderId[$providerId] = [];
if (!\array_key_exists($providerId, $identifiers)) {
$identifiers[$providerId] = [];
}
$identifiersByProviderId[$providerId][] = $recipient->getAttribute('identifier');
$identifiers[$providerId][] = $recipient->getAttribute('identifier');
}
}
/**
* @var array[] $results
* @var array<array> $results
*/
$results = batch(\array_map(function ($providerId) use ($identifiersByProviderId, $providers, $primaryProvider, $message, $dbForProject) {
return function () use ($providerId, $identifiersByProviderId, $providers, $primaryProvider, $message, $dbForProject) {
$results = batch(\array_map(function ($providerId) use ($identifiers, $providers, $fallback, $message, $dbForProject) {
return function () use ($providerId, $identifiers, $providers, $fallback, $message, $dbForProject) {
if (\array_key_exists($providerId, $providers)) {
$provider = $providers[$providerId];
} else {
$provider = $dbForProject->getDocument('providers', $providerId, [Query::equal('enabled', [true])]);
$provider = $dbForProject->getDocument('providers', $providerId);
if ($provider->isEmpty()) {
$provider = $primaryProvider;
if ($provider->isEmpty() || !$provider->getAttribute('enabled')) {
$provider = $fallback;
} else {
$providers[$providerId] = $provider;
}
}
$identifiers = $identifiersByProviderId[$providerId];
$identifiers = $identifiers[$providerId];
$adapter = match ($provider->getAttribute('type')) {
MESSAGE_TYPE_SMS => $this->sms($provider),
@@ -200,7 +243,10 @@ class Messaging extends Action
// Deleting push targets when token has expired.
if ($detail['error'] === 'Expired device token.') {
$target = $dbForProject->findOne('targets', [Query::equal('identifier', [$detail['recipient']])]);
$target = $dbForProject->findOne('targets', [
Query::equal('identifier', [$detail['recipient']])
]);
if ($target instanceof Document && !$target->isEmpty()) {
$dbForProject->deleteDocument('targets', $target->getId());
}
@@ -210,6 +256,7 @@ class Messaging extends Action
$deliveryErrors[] = 'Failed sending to targets ' . $batchIndex + 1 . '-' . \count($batch) . ' with error: ' . $e->getMessage();
} finally {
$batchIndex++;
return [
'deliveredTotal' => $deliveredTotal,
'deliveryErrors' => $deliveryErrors,
@@ -218,7 +265,7 @@ class Messaging extends Action
};
}, $batches));
};
}, \array_keys($identifiersByProviderId)));
}, \array_keys($identifiers)));
$results = array_merge(...$results);
@@ -233,9 +280,9 @@ class Messaging extends Action
$message->setAttribute('deliveryErrors', $deliveryErrors);
if (\count($message->getAttribute('deliveryErrors')) > 0) {
$message->setAttribute('status', 'failed');
$message->setAttribute('status', MessageStatus::FAILED);
} else {
$message->setAttribute('status', 'sent');
$message->setAttribute('status', MessageStatus::SENT);
}
$message->removeAttribute('to');
@@ -253,7 +300,7 @@ class Messaging extends Action
private function processInternalSMSMessage(Log $log, Document $message, array $recipients): void
{
if (empty(App::getEnv('_APP_SMS_PROVIDER')) || empty(App::getEnv('_APP_SMS_FROM'))) {
throw new \Exception('Skipped SMS processing. No Phone configuration has been set.');
throw new \Exception('Skipped SMS processing. Missing "_APP_SMS_PROVIDER" or "_APP_SMS_FROM" environment variables.');
}
$smsDSN = new DSN(App::getEnv('_APP_SMS_PROVIDER'));
@@ -384,14 +431,20 @@ class Messaging extends Action
$bcc = [];
if (\count($ccTargets) > 0) {
$ccTargets = $dbForProject->find('targets', [Query::equal('identifier', $ccTargets)]);
$ccTargets = $dbForProject->find('targets', [
Query::equal('$id', $ccTargets),
Query::limit(\count($ccTargets)),
]);
foreach ($ccTargets as $ccTarget) {
$cc[] = ['email' => $ccTarget['identifier']];
}
}
if (\count($bccTargets) > 0) {
$bccTargets = $dbForProject->find('targets', [Query::equal('identifier', $bccTargets)]);
$bccTargets = $dbForProject->find('targets', [
Query::equal('$id', $bccTargets),
Query::limit(\count($bccTargets)),
]);
foreach ($bccTargets as $bccTarget) {
$bcc[] = ['email' => $bccTarget['identifier']];
}
+114 -10
View File
@@ -2,9 +2,13 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Event\Mail;
use Appwrite\Template\Template;
use Exception;
use Utopia\App;
use Utopia\Database\Document;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Logger\Log;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
@@ -12,6 +16,8 @@ use Utopia\Queue\Message;
class Webhooks extends Action
{
private array $errors = [];
private const MAX_FAILED_ATTEMPTS = 10;
private const MAX_FILE_SIZE = 5242880; // 5 MB
public static function getName(): string
{
@@ -26,18 +32,23 @@ class Webhooks extends Action
$this
->desc('Webhooks worker')
->inject('message')
->inject('dbForConsole')
->inject('queueForMails')
->inject('log')
->callback(fn (Message $message, Log $log) => $this->action($message, $log));
->callback(fn (Message $message, Database $dbForConsole, Mail $queueForMails, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $log));
}
/**
* @param Message $message
* @param Database $dbForConsole
* @param Mail $queueForMails
* @param Log $log
* @return void
* @throws Exception
*/
public function action(Message $message, Log $log): void
public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Log $log): void
{
$this->errors = [];
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
@@ -53,7 +64,7 @@ class Webhooks extends Action
foreach ($project->getAttribute('webhooks', []) as $webhook) {
if (array_intersect($webhook->getAttribute('events', []), $events)) {
$this->execute($events, $webhookPayload, $webhook, $user, $project);
$this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForConsole, $queueForMails);
}
}
@@ -68,10 +79,15 @@ class Webhooks extends Action
* @param Document $webhook
* @param Document $user
* @param Document $project
* @param Database $dbForConsole
* @param Mail $queueForMails
* @return void
*/
private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project): void
private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForConsole, Mail $queueForMails): void
{
if ($webhook->getAttribute('enabled') !== true) {
return;
}
$url = \rawurldecode($webhook->getAttribute('url'));
$signatureKey = $webhook->getAttribute('signatureKey');
@@ -83,9 +99,9 @@ class Webhooks extends Action
\curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
\curl_setopt($ch, CURLOPT_POSTFIELDS, $payload);
\curl_setopt($ch, CURLOPT_HEADER, 0);
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, 0);
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
\curl_setopt($ch, CURLOPT_TIMEOUT, 15);
\curl_setopt($ch, CURLOPT_MAXFILESIZE, 5242880);
\curl_setopt($ch, CURLOPT_MAXFILESIZE, self::MAX_FILE_SIZE);
\curl_setopt($ch, CURLOPT_USERAGENT, \sprintf(
APP_USERAGENT,
App::getEnv('_APP_VERSION', 'UNKNOWN'),
@@ -117,10 +133,98 @@ class Webhooks extends Action
\curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
}
if (false === \curl_exec($ch)) {
$this->errors[] = \curl_error($ch) . ' in events ' . implode(', ', $events) . ' for webhook ' . $webhook->getAttribute('name');
}
$responseBody = \curl_exec($ch);
$curlError = \curl_error($ch);
$statusCode = curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
\curl_close($ch);
if (!empty($curlError) || $statusCode >= 400) {
$dbForConsole->increaseDocumentAttribute('webhooks', $webhook->getId(), 'attempts', 1);
$webhook = $dbForConsole->getDocument('webhooks', $webhook->getId());
$attempts = $webhook->getAttribute('attempts');
$logs = '';
$logs .= 'URL: ' . $webhook->getAttribute('url') . "\n";
$logs .= 'Method: ' . 'POST' . "\n";
if (!empty($curlError)) {
$logs .= 'CURL Error: ' . $curlError . "\n";
$logs .= 'Events: ' . implode(', ', $events) . "\n";
} else {
$logs .= 'Status code: ' . $statusCode . "\n";
$logs .= 'Body: ' . "\n" . \mb_strcut($responseBody, 0, 10000) . "\n"; // Limit to 10kb
}
$webhook->setAttribute('logs', $logs);
if ($attempts >= self::MAX_FAILED_ATTEMPTS) {
$webhook->setAttribute('enabled', false);
$this->sendEmailAlert($attempts, $statusCode, $webhook, $project, $dbForConsole, $queueForMails);
}
$dbForConsole->updateDocument('webhooks', $webhook->getId(), $webhook);
$dbForConsole->deleteCachedDocument('projects', $project->getId());
$this->errors[] = $logs;
} else {
$webhook->setAttribute('attempts', 0); // Reset attempts on success
$dbForConsole->updateDocument('webhooks', $webhook->getId(), $webhook);
$dbForConsole->deleteCachedDocument('projects', $project->getId());
}
}
/**
* @param int $attempts
* @param mixed $statusCode
* @param Document $webhook
* @param Document $project
* @param Database $dbForConsole
* @param Mail $queueForMails
* @return void
*/
public function sendEmailAlert(int $attempts, mixed $statusCode, Document $webhook, Document $project, Database $dbForConsole, Mail $queueForMails): void
{
$memberships = $dbForConsole->find('memberships', [
Query::equal('teamInternalId', [$project->getAttribute('teamInternalId')]),
Query::limit(APP_LIMIT_SUBQUERY)
]);
$userIds = array_column(\array_map(fn ($membership) => $membership->getArrayCopy(), $memberships), 'userId');
$users = $dbForConsole->find('users', [
Query::equal('$id', $userIds),
]);
$projectId = $project->getId();
$webhookId = $webhook->getId();
$template = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-webhook-failed.tpl');
$template->setParam('{{webhook}}', $webhook->getAttribute('name'));
$template->setParam('{{project}}', $project->getAttribute('name'));
$template->setParam('{{url}}', $webhook->getAttribute('url'));
$template->setParam('{{error}}', $curlError ?? 'The server returned ' . $statusCode . ' status code');
$template->setParam('{{path}}', "/console/project-$projectId/settings/webhooks/$webhookId");
$template->setParam('{{attempts}}', $attempts);
$subject = 'Webhook deliveries have been paused';
$body = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base-styled.tpl');
$body
->setParam('{{subject}}', $subject)
->setParam('{{message}}', $template->render())
->setParam('{{year}}', date("Y"));
$queueForMails
->setSubject($subject)
->setBody($body->render());
foreach ($users as $user) {
$queueForMails
->setVariables(['user' => $user->getAttribute('name', '')])
->setName($user->getAttribute('name', ''))
->setRecipient($user->getAttribute('email'))
->trigger();
}
}
}
@@ -5,16 +5,12 @@ namespace Appwrite\Utopia\Database\Validator\Queries;
class Messages extends Base
{
public const ALLOWED_ATTRIBUTES = [
'topics',
'users',
'targets',
'providerId',
'scheduledAt',
'deliveredAt',
'deliveredTo',
'deliveryErrors',
'deliveredTotal',
'status',
'description',
'data'
'providerType',
];
/**
@@ -8,6 +8,7 @@ class Targets extends Base
'userId',
'providerId',
'identifier',
'providerType',
];
/**
+18 -1
View File
@@ -76,7 +76,24 @@ class Webhook extends Model
'default' => '',
'example' => 'ad3d581ca230e2b7059c545e5a',
])
;
->addRule('enabled', [
'type' => self::TYPE_BOOLEAN,
'description' => 'Indicates if this webhook is enabled.',
'default' => true,
'example' => true,
])
->addRule('logs', [
'type' => self::TYPE_STRING,
'description' => 'Webhook error logs from the most recent failure.',
'default' => '',
'example' => 'Failed to connect to remote server.',
])
->addRule('attempts', [
'type' => self::TYPE_INTEGER,
'description' => 'Number of consecutive failed webhook attempts.',
'default' => 0,
'example' => 10,
]);
}
/**