diff --git a/app/cli.php b/app/cli.php index fc658c5ad2..9ade97e90c 100644 --- a/app/cli.php +++ b/app/cli.php @@ -26,6 +26,9 @@ use Utopia\Pools\Group; use Utopia\Queue\Publisher; use Utopia\Registry\Registry; use Utopia\System\System; +use Utopia\Telemetry\Adapter\None as NoTelemetry; + +use function Swoole\Coroutine\run; // Overwriting runtimes to be architecture agnostic for CLI Config::setParam('runtimes', (new Runtimes('v4'))->getAll(supported: false)); @@ -200,7 +203,7 @@ CLI::setResource('getLogsDB', function (Group $pools, Cache $cache) { }; }, ['pools', 'cache']); -CLI::setResource('queueForStatsUsage', function (Connection $publisher) { +CLI::setResource('queueForStatsUsage', function (Publisher $publisher) { return new StatsUsage($publisher); }, ['publisher']); CLI::setResource('queueForStatsResources', function (Publisher $publisher) { @@ -264,6 +267,8 @@ CLI::setResource('logError', function (Registry $register) { CLI::setResource('executor', fn () => new Executor(fn (string $projectId, string $deploymentId) => System::getEnv('_APP_EXECUTOR_HOST'))); +CLI::setResource('telemetry', fn () => new NoTelemetry()); + $platform = new Appwrite(); $args = $platform->getEnv('argv'); @@ -293,4 +298,4 @@ $cli $cli->shutdown()->action(fn () => Timer::clearAll()); -$cli->run(); +run($cli->run(...)); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index edc9d55ddb..b19e12e291 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2813,11 +2813,12 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') ->param('type', null, new WhiteList([Database::INDEX_KEY, Database::INDEX_FULLTEXT, Database::INDEX_UNIQUE]), 'Index type.') ->param('attributes', null, new ArrayList(new Key(true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of attributes to index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' attributes are allowed, each 32 characters long.') ->param('orders', [], new ArrayList(new WhiteList(['ASC', 'DESC'], false, Database::VAR_STRING), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of index orders. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' orders are allowed.', true) + ->param('lengths', [], new ArrayList(new Nullable(new Integer()), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Length of index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE, optional:true) ->inject('response') ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->action(function (string $databaseId, string $collectionId, string $key, string $type, array $attributes, array $orders, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { + ->action(function (string $databaseId, string $collectionId, string $key, string $type, array $attributes, array $orders, array $lengths, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $db = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); @@ -2877,9 +2878,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') 'size' => 0 ]; - // lengths hidden by default - $lengths = []; - foreach ($attributes as $i => $attribute) { // find attribute metadata in collection document $attributeIndex = \array_search($attribute, array_column($oldAttributes, 'key')); @@ -2901,8 +2899,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') throw new Exception(Exception::ATTRIBUTE_NOT_AVAILABLE, 'Attribute not available: ' . $oldAttributes[$attributeIndex]['key']); } - $lengths[$i] = null; - + $lengths[$i] ??= null; if ($attributeArray === true) { $lengths[$i] = Database::ARRAY_INDEX_LENGTH; $orders[$i] = null; diff --git a/composer.json b/composer.json index 9e45f71c61..8b8bb811af 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.13.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.68.*", + "utopia-php/database": "0.69.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", @@ -59,7 +59,7 @@ "utopia-php/image": "0.8.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", - "utopia-php/messaging": "0.16.*", + "utopia-php/messaging": "0.17.*", "utopia-php/migration": "0.9.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", diff --git a/composer.lock b/composer.lock index cf15b7657e..5e86bfafd2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a5c4c6b723423bc6cb3a7344ab071b43", + "content-hash": "2c14e20244a06f508dd67cda717aefeb", "packages": [ { "name": "adhocore/jwt", @@ -1109,16 +1109,16 @@ }, { "name": "open-telemetry/api", - "version": "1.2.3", + "version": "1.3.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "199d7ddda88f5f5619fa73463f1a5a7149ccd1f1" + "reference": "4e3bb38e069876fb73c2ce85c89583bf2b28cd86" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/199d7ddda88f5f5619fa73463f1a5a7149ccd1f1", - "reference": "199d7ddda88f5f5619fa73463f1a5a7149ccd1f1", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/4e3bb38e069876fb73c2ce85c89583bf2b28cd86", + "reference": "4e3bb38e069876fb73c2ce85c89583bf2b28cd86", "shasum": "" }, "require": { @@ -1175,7 +1175,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-03-05T21:42:54+00:00" + "time": "2025-05-07T12:32:21+00:00" }, { "name": "open-telemetry/context", @@ -1238,16 +1238,16 @@ }, { "name": "open-telemetry/exporter-otlp", - "version": "1.2.1", + "version": "1.3.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/exporter-otlp.git", - "reference": "b7580440b7481a98da97aceabeb46e1b276c8747" + "reference": "19adf03d2b0f91f9e9b1c7f93db6c755c737cf6c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/b7580440b7481a98da97aceabeb46e1b276c8747", - "reference": "b7580440b7481a98da97aceabeb46e1b276c8747", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/19adf03d2b0f91f9e9b1c7f93db6c755c737cf6c", + "reference": "19adf03d2b0f91f9e9b1c7f93db6c755c737cf6c", "shasum": "" }, "require": { @@ -1298,7 +1298,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-03-06T23:21:56+00:00" + "time": "2025-05-12T00:36:35+00:00" }, { "name": "open-telemetry/gen-otlp-protobuf", @@ -1365,16 +1365,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.3.0", + "version": "1.4.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "05d9ceb6773b5bddcf485af6d4a6f543bbeb980b" + "reference": "939d3a28395c249a763676458140dad44b3a8011" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/05d9ceb6773b5bddcf485af6d4a6f543bbeb980b", - "reference": "05d9ceb6773b5bddcf485af6d4a6f543bbeb980b", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/939d3a28395c249a763676458140dad44b3a8011", + "reference": "939d3a28395c249a763676458140dad44b3a8011", "shasum": "" }, "require": { @@ -1451,7 +1451,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-05-01T23:20:43+00:00" + "time": "2025-05-07T12:32:21+00:00" }, { "name": "open-telemetry/sem-conv", @@ -3499,16 +3499,16 @@ }, { "name": "utopia-php/database", - "version": "0.68.1", + "version": "0.69.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "72b2e2c0b875028f7d9dd755f6d4524b693c6507" + "reference": "cc6538e05e25d930244ab938c966d32db0922e83" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/72b2e2c0b875028f7d9dd755f6d4524b693c6507", - "reference": "72b2e2c0b875028f7d9dd755f6d4524b693c6507", + "url": "https://api.github.com/repos/utopia-php/database/zipball/cc6538e05e25d930244ab938c966d32db0922e83", + "reference": "cc6538e05e25d930244ab938c966d32db0922e83", "shasum": "" }, "require": { @@ -3549,9 +3549,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.68.1" + "source": "https://github.com/utopia-php/database/tree/0.69.1" }, - "time": "2025-05-09T10:08:53+00:00" + "time": "2025-05-13T12:00:31+00:00" }, { "name": "utopia-php/domains", @@ -3902,16 +3902,16 @@ }, { "name": "utopia-php/messaging", - "version": "0.16.0", + "version": "0.17.0", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "5f3083697102b1821d6624938186761b1e09c54e" + "reference": "c51915d0e030db3a3add37f1561751d18b2d9a85" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/5f3083697102b1821d6624938186761b1e09c54e", - "reference": "5f3083697102b1821d6624938186761b1e09c54e", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/c51915d0e030db3a3add37f1561751d18b2d9a85", + "reference": "c51915d0e030db3a3add37f1561751d18b2d9a85", "shasum": "" }, "require": { @@ -3947,9 +3947,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.16.0" + "source": "https://github.com/utopia-php/messaging/tree/0.17.0" }, - "time": "2025-02-18T08:27:00+00:00" + "time": "2025-05-12T16:14:08+00:00" }, { "name": "utopia-php/migration", diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 4efa78ed4b..e495ce1d3f 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -33,11 +33,7 @@ class Migrate extends Action ->inject('dbForPlatform') ->inject('getProjectDB') ->inject('register') - ->callback(function ($version, $dbForPlatform, $getProjectDB, Registry $register) { - \Co\run(function () use ($version, $dbForPlatform, $getProjectDB, $register) { - $this->action($version, $dbForPlatform, $getProjectDB, $register); - }); - }); + ->callback($this->action(...)); } private function clearProjectsCache(Document $project) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index a3c36cb96e..8f7eab4d87 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -13,8 +13,9 @@ use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; use Utopia\Pools\Group; use Utopia\System\System; - -use function Swoole\Coroutine\run; +use Utopia\Telemetry\Adapter as Telemetry; +use Utopia\Telemetry\Gauge; +use Utopia\Telemetry\Histogram; abstract class ScheduleBase extends Action { @@ -23,6 +24,11 @@ abstract class ScheduleBase extends Action protected array $schedules = []; + private ?Histogram $collectSchedulesTelemetryDuration = null; + private ?Gauge $collectSchedulesTelemetryCount = null; + private ?Gauge $scheduleTelemetryCount = null; + private ?Histogram $enqueueDelayTelemetry = null; + abstract public static function getName(): string; abstract public static function getSupportedResource(): string; abstract public static function getCollectionId(): string; @@ -37,7 +43,8 @@ abstract class ScheduleBase extends Action ->inject('pools') ->inject('dbForPlatform') ->inject('getProjectDB') - ->callback(fn (Group $pools, Database $dbForPlatform, callable $getProjectDB) => $this->action($pools, $dbForPlatform, $getProjectDB)); + ->inject('telemetry') + ->callback($this->action(...)); } protected function updateProjectAccess(Document $project, Database $dbForPlatform): void @@ -56,11 +63,47 @@ abstract class ScheduleBase extends Action * 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute * 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutine sleeps until exact time before sending request to worker. */ - public function action(Group $pools, Database $dbForPlatform, callable $getProjectDB): void + public function action(Group $pools, Database $dbForPlatform, callable $getProjectDB, Telemetry $telemetry): void { Console::title(\ucfirst(static::getSupportedResource()) . ' scheduler V1'); Console::success(APP_NAME . ' ' . \ucfirst(static::getSupportedResource()) . ' scheduler v1 has started'); + $this->scheduleTelemetryCount = $telemetry->createGauge('task.schedule.count'); + $this->collectSchedulesTelemetryDuration = $telemetry->createHistogram('task.schedule.collect_schedules.duration', 's'); + $this->collectSchedulesTelemetryCount = $telemetry->createGauge('task.schedule.collect_schedules.count'); + $this->enqueueDelayTelemetry = $telemetry->createHistogram('task.schedule.enqueue_delay', 's'); + + // start with "0" to load all active documents. + $lastSyncUpdate = "0"; + $this->collectSchedules($pools, $dbForPlatform, $getProjectDB, $lastSyncUpdate); + + Console::success("Starting timers at " . DateTime::now()); + /** + * The timer synchronize $schedules copy with database collection. + */ + Timer::tick(static::UPDATE_TIMER * 1000, function () use ($pools, $dbForPlatform, $getProjectDB, &$lastSyncUpdate) { + $time = DateTime::now(); + Console::log("Sync tick: Running at $time"); + $this->collectSchedules($pools, $dbForPlatform, $getProjectDB, $lastSyncUpdate); + }); + + while (true) { + try { + go(fn () => $this->enqueueResources($pools, $dbForPlatform, $getProjectDB)); + $this->scheduleTelemetryCount->record(count($this->schedules), ['resourceType' => static::getSupportedResource()]); + sleep(static::ENQUEUE_TIMER); + } catch (\Throwable $th) { + Console::error('Failed to enqueue resources: ' . $th->getMessage()); + } + + } + } + + private function collectSchedules(Group $pools, Database $dbForPlatform, callable $getProjectDB, string &$lastSyncUpdate): void + { + // If we haven't synced yet, load all active schedules + $initialLoad = $lastSyncUpdate === "0"; + /** * Extract only necessary attributes to lower memory used. * @@ -68,7 +111,7 @@ abstract class ScheduleBase extends Action * @throws Exception * @var Document $schedule */ - $getSchedule = function (Document $schedule) use ($dbForPlatform, $getProjectDB): array { + $getSchedule = function (Document $schedule) use ($pools, $dbForPlatform, $getProjectDB): array { $project = $dbForPlatform->getDocument('projects', $schedule->getAttribute('projectId')); $resource = $getProjectDB($project)->getDocument( @@ -76,6 +119,8 @@ abstract class ScheduleBase extends Action $schedule->getAttribute('resourceId') ); + $pools->reclaim(); + return [ '$internalId' => $schedule->getInternalId(), '$id' => $schedule->getId(), @@ -88,12 +133,12 @@ abstract class ScheduleBase extends Action ]; }; - $lastSyncUpdate = DateTime::now(); + $loadStart = microtime(true); + $time = DateTime::now(); $limit = 10_000; $sum = $limit; $total = 0; - $loadStart = \microtime(true); $latestDocument = null; while ($sum === $limit) { @@ -110,105 +155,57 @@ abstract class ScheduleBase extends Action $regions[] = 'default'; } - $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ + $paginationQueries = [ + ...$paginationQueries, Query::equal('region', $regions), Query::equal('resourceType', [static::getSupportedResource()]), - Query::equal('active', [true]), - ])); + ]; - $sum = \count($results); + if ($initialLoad) { + $paginationQueries[] = Query::equal('active', [true]); + } else { + $paginationQueries[] = Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate); + } + + $results = $dbForPlatform->find('schedules', $paginationQueries); + + $sum = count($results); $total = $total + $sum; foreach ($results as $document) { - try { - $this->schedules[$document->getInternalId()] = $getSchedule($document); - } catch (\Throwable $th) { - $collectionId = static::getCollectionId(); - Console::error("Failed to load schedule for project {$document['projectId']} {$collectionId} {$document['resourceId']}"); - Console::error($th->getMessage()); + $localDocument = $this->schedules[$document->getInternalId()] ?? null; + + if ($localDocument !== null) { + if (!$document['active']) { + Console::info("Removing: {$document['resourceType']}::{$document['resourceId']}"); + unset($this->schedules[$document->getInternalId()]); + } elseif (strtotime($localDocument['resourceUpdatedAt']) !== strtotime($document['resourceUpdatedAt'])) { + Console::info("Updating: {$document['resourceType']}::{$document['resourceId']}"); + $this->schedules[$document->getInternalId()] = $getSchedule($document); + } + } else { + try { + $this->schedules[$document->getInternalId()] = $getSchedule($document); + } catch (\Throwable $th) { + $collectionId = static::getCollectionId(); + Console::error("Failed to load schedule for project {$document['projectId']} {$collectionId} {$document['resourceId']}"); + Console::error($th->getMessage()); + } } } $latestDocument = \end($results); } - $pools->reclaim(); + $lastSyncUpdate = $time; + $duration = microtime(true) - $loadStart; + $this->collectSchedulesTelemetryDuration->record($duration, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); + $this->collectSchedulesTelemetryCount->record($total, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); + Console::success("{$total} resources were loaded in " . $duration . " seconds"); + } - Console::success("{$total} resources were loaded in " . (\microtime(true) - $loadStart) . " seconds"); - - Console::success("Starting timers at " . DateTime::now()); - - run(function () use ($dbForPlatform, &$lastSyncUpdate, $getSchedule, $pools, $getProjectDB) { - /** - * The timer synchronize $schedules copy with database collection. - */ - Timer::tick(static::UPDATE_TIMER * 1000, function () use ($dbForPlatform, &$lastSyncUpdate, $getSchedule, $pools) { - $time = DateTime::now(); - $timerStart = \microtime(true); - - $limit = 1000; - $sum = $limit; - $total = 0; - $latestDocument = null; - - Console::log("Sync tick: Running at $time"); - - while ($sum === $limit) { - $paginationQueries = [Query::limit($limit)]; - - if ($latestDocument) { - $paginationQueries[] = Query::cursorAfter($latestDocument); - } - - // Temporarly accepting both 'fra' and 'default' - // When all migrated, only use _APP_REGION with 'default' as default value - $regions = [System::getEnv('_APP_REGION', 'default')]; - if (!in_array('default', $regions)) { - $regions[] = 'default'; - } - - $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', $regions), - Query::equal('resourceType', [static::getSupportedResource()]), - Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), - ])); - - $sum = count($results); - $total = $total + $sum; - - foreach ($results as $document) { - $localDocument = $this->schedules[$document->getInternalId()] ?? null; - - // Check if resource has been updated since last sync - $org = $localDocument !== null ? \strtotime($localDocument['resourceUpdatedAt']) : null; - $new = \strtotime($document['resourceUpdatedAt']); - - if (!$document['active']) { - Console::info("Removing: {$document['resourceType']}::{$document['resourceId']}"); - unset($this->schedules[$document->getInternalId()]); - } elseif ($new !== $org) { - Console::info("Updating: {$document['resourceType']}::{$document['resourceId']}"); - $this->schedules[$document->getInternalId()] = $getSchedule($document); - } - } - - $latestDocument = \end($results); - } - - $lastSyncUpdate = $time; - $timerEnd = \microtime(true); - - $pools->reclaim(); - - Console::log("Sync tick: {$total} schedules were updated in " . ($timerEnd - $timerStart) . " seconds"); - }); - - Timer::tick( - static::ENQUEUE_TIMER * 1000, - fn () => $this->enqueueResources($pools, $dbForPlatform, $getProjectDB) - ); - - $this->enqueueResources($pools, $dbForPlatform, $getProjectDB); - }); + protected function recordEnqueueDelay(\DateTime $expectedExecutionSchedule): void + { + $this->enqueueDelayTelemetry->record(time() - $expectedExecutionSchedule->getTimestamp(), ['resourceType' => static::getSupportedResource()]); } } diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 7cd76b480d..89d1609a33 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -59,7 +59,7 @@ class ScheduleExecutions extends ScheduleBase $this->updateProjectAccess($schedule['project'], $dbForPlatform); - \go(function () use ($queueForFunctions, $schedule, $delay, $data) { + \go(function () use ($queueForFunctions, $schedule, $scheduledAt, $delay, $data) { Co::sleep($delay); $queueForFunctions->setType('schedule') @@ -74,6 +74,8 @@ class ScheduleExecutions extends ScheduleBase ->setProject($schedule['project']) ->setUserId($data['userId'] ?? '') ->trigger(); + + $this->recordEnqueueDelay($scheduledAt); }); $dbForPlatform->deleteDocument( diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index 5b8e3027a7..6788748f3d 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -46,7 +46,13 @@ class ScheduleFunctions extends ScheduleBase $delayedExecutions = []; // Group executions with same delay to share one coroutine foreach ($this->schedules as $key => $schedule) { - $cron = new CronExpression($schedule['schedule']); + try { + $cron = new CronExpression($schedule['schedule']); + } catch (\InvalidArgumentException) { + // ignore invalid cron expressions + continue; + } + $nextDate = $cron->getNextRunDate(); $next = DateTime::format($nextDate); @@ -66,17 +72,18 @@ class ScheduleFunctions extends ScheduleBase $delayedExecutions[$delay] = []; } - $delayedExecutions[$delay][] = $key; + $delayedExecutions[$delay][] = ['key' => $key, 'nextDate' => $nextDate]; } - foreach ($delayedExecutions as $delay => $scheduleKeys) { - \go(function () use ($delay, $scheduleKeys, $pools, $dbForPlatform) { + foreach ($delayedExecutions as $delay => $schedules) { + \go(function () use ($delay, $schedules, $pools, $dbForPlatform) { \sleep($delay); // in seconds $queue = $pools->get('publisher')->pop(); $connection = $queue->getResource(); - foreach ($scheduleKeys as $scheduleKey) { + foreach ($schedules as $delayConfig) { + $scheduleKey = $delayConfig['key']; // Ensure schedule was not deleted if (!\array_key_exists($scheduleKey, $this->schedules)) { return; @@ -95,6 +102,8 @@ class ScheduleFunctions extends ScheduleBase ->setPath('/') ->setProject($schedule['project']) ->trigger(); + + $this->recordEnqueueDelay($delayConfig['nextDate']); } $queue->reclaim(); diff --git a/src/Appwrite/Platform/Tasks/ScheduleMessages.php b/src/Appwrite/Platform/Tasks/ScheduleMessages.php index 201d5eab53..a15df6ed5b 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleMessages.php +++ b/src/Appwrite/Platform/Tasks/ScheduleMessages.php @@ -40,7 +40,7 @@ class ScheduleMessages extends ScheduleBase continue; } - \go(function () use ($schedule, $pools, $dbForPlatform) { + \go(function () use ($schedule, $scheduledAt, $pools, $dbForPlatform) { $queue = $pools->get('publisher')->pop(); $connection = $queue->getResource(); $queueForMessaging = new Messaging($connection); @@ -59,7 +59,7 @@ class ScheduleMessages extends ScheduleBase ); $queue->reclaim(); - + $this->recordEnqueueDelay($scheduledAt); unset($this->schedules[$schedule['$internalId']]); }); } diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index cf2b8bfc84..c9eca2a1e0 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -24,6 +24,7 @@ use Utopia\Messaging\Adapter\Push\FCM; use Utopia\Messaging\Adapter\SMS as SMSAdapter; use Utopia\Messaging\Adapter\SMS\Fast2SMS; use Utopia\Messaging\Adapter\SMS\GEOSMS; +use Utopia\Messaging\Adapter\SMS\Inforu; use Utopia\Messaging\Adapter\SMS\Mock; use Utopia\Messaging\Adapter\SMS\Msg91; use Utopia\Messaging\Adapter\SMS\Telesign; @@ -455,6 +456,10 @@ class Messaging extends Action $credentials['messageId'] ?? '', $credentials['useDLT'] ?? true ), + 'inforu' => new Inforu( + $credentials['senderId'] ?? '', + $credentials['apiKey'] ?? '', + ), default => null }; } @@ -780,6 +785,10 @@ class Messaging extends Action 'messageId' => $dsn->getParam('messageId'), 'useDLT' => $dsn->getParam('useDLT'), ], + 'inforu' => [ + 'senderId' => $user, + 'apiKey' => $password, + ], default => null }, 'options' => match ($host) { diff --git a/src/Appwrite/Utopia/Request.php b/src/Appwrite/Utopia/Request.php index 480fce58b0..c50dea2713 100644 --- a/src/Appwrite/Utopia/Request.php +++ b/src/Appwrite/Utopia/Request.php @@ -3,6 +3,7 @@ namespace Appwrite\Utopia; use Appwrite\Auth\Auth; +use Appwrite\SDK\Method; use Appwrite\Utopia\Request\Filter; use Swoole\Http\Request as SwooleRequest; use Utopia\Database\Validator\Authorization; @@ -29,35 +30,49 @@ class Request extends UtopiaRequest { $parameters = parent::getParams(); - if ($this->hasFilters() && self::hasRoute()) { - $methods = self::getRoute()->getLabel('sdk', null); + if (!$this->hasFilters() || !self::hasRoute()) { + return $parameters; + } - if (!\is_array($methods)) { - $methods = [$methods]; - } + $methods = self::getRoute()->getLabel('sdk', null); - $params = []; - - foreach ($methods as $method) { - /** @var \Appwrite\SDK\Method $method */ - if (empty($method)) { - $endpointIdentifier = 'unknown.unknown'; - } else { - $endpointIdentifier = $method->getNamespace() . '.' . $method->getMethodName(); - } - - $params += $method->getParameters(); - } - - if (!empty($params)) { - $parameters = array_filter($parameters, function ($key) use ($params) { - return array_key_exists($key, $params); - }, \ARRAY_FILTER_USE_KEY); - } + if (empty($methods)) { + return $parameters; + } + if (!\is_array($methods)) { + $id = $methods->getNamespace() . '.' . $methods->getMethodName(); foreach ($this->getFilters() as $filter) { - $parameters = $filter->parse($parameters, $endpointIdentifier); + $parameters = $filter->parse($parameters, $id); } + return $parameters; + } + + $matched = null; + foreach ($methods as $method) { + /** @var Method|null $method */ + if ($method === null) { + continue; + } + + // Find the method that matches the parameters passed + $methodParamNames = \array_map(fn ($param) => $param->getName(), $method->getParameters()); + $invalidParams = \array_diff(\array_keys($parameters), $methodParamNames); + + // No params defined, or all params are valid + if (empty($methodParamNames) || empty($invalidParams)) { + $matched = $method; + break; + } + } + + $id = $matched !== null + ? $matched->getNamespace() . '.' . $matched->getMethodName() + : 'unknown.unknown'; + + // Apply filters + foreach ($this->getFilters() as $filter) { + $parameters = $filter->parse($parameters, $id); } return $parameters; diff --git a/src/Appwrite/Utopia/Response/Model/Index.php b/src/Appwrite/Utopia/Response/Model/Index.php index 2d795ad439..fcd978b5be 100644 --- a/src/Appwrite/Utopia/Response/Model/Index.php +++ b/src/Appwrite/Utopia/Response/Model/Index.php @@ -41,6 +41,13 @@ class Index extends Model 'example' => [], 'array' => true, ]) + ->addRule('lengths', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Index attributes length.', + 'default' => [], + 'example' => [], + 'array' => true, + ]) ->addRule('orders', [ 'type' => self::TYPE_STRING, 'description' => 'Index orders.', diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 1f19c514d1..2b60dee856 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -1422,9 +1422,102 @@ trait DatabasesBase return $data; } + /** - * @depends testCreateIndexes - */ + * @depends testCreateAttributes + */ + public function testGetIndexByKeyWithLengths(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['moviesId']; + + // Test case for valid lengths + $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'lengthTestIndex', + 'type' => 'key', + 'attributes' => ['title','description'], + 'lengths' => [128,200] + ]); + $this->assertEquals(202, $create['headers']['status-code']); + + // Fetch index and check correct lengths + $index = $this->client->call(Client::METHOD_GET, "/databases/{$databaseId}/collections/{$collectionId}/indexes/lengthTestIndex", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $index['headers']['status-code']); + $this->assertEquals('lengthTestIndex', $index['body']['key']); + $this->assertEquals([128, 200], $index['body']['lengths']); + + // Test case for lengths array overriding + // set a length for an array attribute, it should get overriden with Database::ARRAY_INDEX_LENGTH + $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'lengthOverrideTestIndex', + 'type' => 'key', + 'attributes' => ['actors'], + 'lengths' => [120] + ]); + $this->assertEquals(202, $create['headers']['status-code']); + + $index = $this->client->call(Client::METHOD_GET, "/databases/{$databaseId}/collections/{$collectionId}/indexes/lengthOverrideTestIndex", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals([Database::ARRAY_INDEX_LENGTH], $index['body']['lengths']); + + // Test case for count of lengths greater than attributes (should throw 400) + $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'lengthCountExceededIndex', + 'type' => 'key', + 'attributes' => ['title'], + 'lengths' => [128, 128] + ]); + $this->assertEquals(400, $create['headers']['status-code']); + + // Test case for lengths exceeding total of 768 + $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'lengthTooLargeIndex', + 'type' => 'key', + 'attributes' => ['title','description','tagline','actors'], + 'lengths' => [256,256,256,20] + ]); + + $this->assertEquals(400, $create['headers']['status-code']); + + // Test case for negative length values + $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'negativeLengthIndex', + 'type' => 'key', + 'attributes' => ['title'], + 'lengths' => [-1] + ]); + $this->assertEquals(400, $create['headers']['status-code']); + } + /** + * @depends testCreateIndexes + */ public function testListIndexes(array $data): void { $databaseId = $data['databaseId']; diff --git a/tests/unit/Utopia/RequestTest.php b/tests/unit/Utopia/RequestTest.php index 26273f154e..63655de21d 100644 --- a/tests/unit/Utopia/RequestTest.php +++ b/tests/unit/Utopia/RequestTest.php @@ -3,6 +3,7 @@ namespace Tests\Unit\Utopia; use Appwrite\SDK\Method; +use Appwrite\SDK\Parameter; use Appwrite\Utopia\Request; use PHPUnit\Framework\TestCase; use Swoole\Http\Request as SwooleRequest; @@ -57,4 +58,130 @@ class RequestTest extends TestCase $this->assertTrue($output['second']); $this->assertArrayNotHasKey('deleted', $output); } + + public function testGetParamsWithMultipleMethods(): void + { + $this->setupMultiMethodRoute(); + + // Pass only "foo", should match Method A + $this->request->setQueryString([ + 'foo' => 'valueFoo', + ]); + + $params = $this->request->getParams(); + + $this->assertArrayHasKey('foo', $params); + $this->assertSame('valueFoo', $params['foo']); + $this->assertArrayNotHasKey('baz', $params); + } + + public function testGetParamsWithAllRequired(): void + { + $this->setupMultiMethodRoute(); + + // Pass "foo" and "bar", should match Method A + $this->request->setQueryString([ + 'foo' => 'valueFoo', + 'bar' => 'valueBar', + ]); + + $params = $this->request->getParams(); + $this->assertArrayHasKey('foo', $params); + $this->assertSame('valueFoo', $params['foo']); + $this->assertArrayHasKey('bar', $params); + $this->assertSame('valueBar', $params['bar']); + $this->assertArrayNotHasKey('baz', $params); + } + + public function testGetParamsWithAllOptional(): void + { + $this->setupMultiMethodRoute(); + + // Pass only "bar", should match Method A + $this->request->setQueryString([ + 'bar' => 'valueBar', + ]); + + $params = $this->request->getParams(); + + $this->assertArrayHasKey('bar', $params); + $this->assertSame('valueBar', $params['bar']); + $this->assertArrayNotHasKey('foo', $params); + $this->assertArrayNotHasKey('baz', $params); + } + + public function testGetParamsMatchesMethodB(): void + { + $this->setupMultiMethodRoute(); + + // Pass only "baz", should match Method B + $this->request->setQueryString([ + 'baz' => 'valueBaz', + ]); + + $params = $this->request->getParams(); + + $this->assertArrayHasKey('baz', $params); + $this->assertSame('valueBaz', $params['baz']); + $this->assertArrayNotHasKey('foo', $params); + } + + public function testGetParamsFallbackForMixedAndUnknown(): void + { + $this->setupMultiMethodRoute(); + + // Mixed and unknown should fallback to raw params + $this->request->setQueryString([ + 'foo' => 'valueFoo', + 'baz' => 'valueBaz', + 'extra' => 'unexpected', + ]); + + $params = $this->request->getParams(); + + $this->assertArrayHasKey('foo', $params); + $this->assertSame('valueFoo', $params['foo']); + $this->assertArrayHasKey('baz', $params); + $this->assertSame('valueBaz', $params['baz']); + $this->assertArrayHasKey('extra', $params); + $this->assertSame('unexpected', $params['extra']); + } + + /** + * Helper to attach a route with multiple SDK methods to the request. + */ + private function setupMultiMethodRoute(): void + { + $route = new Route(Request::METHOD_GET, '/multi'); + + $methodA = new Method( + namespace: 'namespace', + group: 'group', + name: 'methodA', + description: 'desc', + auth: [], + responses: [], + parameters: [ + new Parameter('foo'), + new Parameter('bar', optional: true), + ], + ); + + $methodB = new Method( + namespace: 'namespace', + group: 'group', + name: 'methodB', + description: 'desc', + auth: [], + responses: [], + parameters: [ + new Parameter('baz'), + ], + ); + + $route->label('sdk', [$methodA, $methodB]); + $this->request->addFilter(new First()); + $this->request->addFilter(new Second()); + $this->request->setRoute($route); + } }