From 3203ea5d43ef1839ec15ed41be51265cfc3c4c14 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 12 May 2025 17:30:36 +1200 Subject: [PATCH 01/15] Fix request filters with multi-method routes --- composer.lock | 12 +-- src/Appwrite/Utopia/Request.php | 56 ++++++++----- tests/unit/Utopia/RequestTest.php | 127 ++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 27 deletions(-) diff --git a/composer.lock b/composer.lock index b56b7b387d..cf15b7657e 100644 --- a/composer.lock +++ b/composer.lock @@ -4769,16 +4769,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.15", + "version": "0.40.16", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "65c708b931b29b3e01c5cc7504a734ce2cc3dc95" + "reference": "f1f506da74033f0cb5a11e3dffcfd1ee8daf237d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/65c708b931b29b3e01c5cc7504a734ce2cc3dc95", - "reference": "65c708b931b29b3e01c5cc7504a734ce2cc3dc95", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/f1f506da74033f0cb5a11e3dffcfd1ee8daf237d", + "reference": "f1f506da74033f0cb5a11e3dffcfd1ee8daf237d", "shasum": "" }, "require": { @@ -4814,9 +4814,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.15" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.16" }, - "time": "2025-04-25T08:50:44+00:00" + "time": "2025-05-09T12:06:09+00:00" }, { "name": "doctrine/annotations", diff --git a/src/Appwrite/Utopia/Request.php b/src/Appwrite/Utopia/Request.php index 480fce58b0..3e8b7a0a5b 100644 --- a/src/Appwrite/Utopia/Request.php +++ b/src/Appwrite/Utopia/Request.php @@ -3,6 +3,7 @@ namespace Appwrite\Utopia; use Appwrite\Auth\Auth; +use Appwrite\SDK\Method; use Appwrite\Utopia\Request\Filter; use Swoole\Http\Request as SwooleRequest; use Utopia\Database\Validator\Authorization; @@ -29,37 +30,50 @@ class Request extends UtopiaRequest { $parameters = parent::getParams(); - if ($this->hasFilters() && self::hasRoute()) { - $methods = self::getRoute()->getLabel('sdk', null); + if (!$this->hasFilters() || !self::hasRoute()) { + return $parameters; + } - if (!\is_array($methods)) { - $methods = [$methods]; + $methods = self::getRoute()->getLabel('sdk', null); + $methods = \is_array($methods) ? $methods : [$methods]; + $matched = null; + + foreach ($methods as $method) { + /** @var Method|null $method */ + if ($method === null) { + continue; } - $params = []; + // Find the method that matches the parameters passed + $methodParamNames = \array_map(fn($param) => $param->getName(), $method->getParameters()); + $invalidParams = \array_diff(\array_keys($parameters), $methodParamNames); - foreach ($methods as $method) { - /** @var \Appwrite\SDK\Method $method */ - if (empty($method)) { - $endpointIdentifier = 'unknown.unknown'; - } else { - $endpointIdentifier = $method->getNamespace() . '.' . $method->getMethodName(); - } - - $params += $method->getParameters(); + // No params defined, or all params are valid + if (empty($methodParamNames) || empty($invalidParams)) { + $matched = $method; + break; } + } - if (!empty($params)) { - $parameters = array_filter($parameters, function ($key) use ($params) { - return array_key_exists($key, $params); - }, \ARRAY_FILTER_USE_KEY); - } + $endpointIdentifier = $matched !== null + ? $matched->getNamespace() . '.' . $matched->getMethodName() + : 'unknown.unknown'; - foreach ($this->getFilters() as $filter) { - $parameters = $filter->parse($parameters, $endpointIdentifier); + // Filter params to valid keys + if ($matched !== null) { + $definedNames = \array_map(fn($param) => $param->getName(), $matched->getParameters()); + + // If matched method has explicit params, remove all other params + if (!empty($definedNames)) { + $parameters = \array_intersect_key($parameters, \array_flip($definedNames)); } } + // Apply filters + foreach ($this->getFilters() as $filter) { + $parameters = $filter->parse($parameters, $endpointIdentifier); + } + return $parameters; } diff --git a/tests/unit/Utopia/RequestTest.php b/tests/unit/Utopia/RequestTest.php index 26273f154e..63655de21d 100644 --- a/tests/unit/Utopia/RequestTest.php +++ b/tests/unit/Utopia/RequestTest.php @@ -3,6 +3,7 @@ namespace Tests\Unit\Utopia; use Appwrite\SDK\Method; +use Appwrite\SDK\Parameter; use Appwrite\Utopia\Request; use PHPUnit\Framework\TestCase; use Swoole\Http\Request as SwooleRequest; @@ -57,4 +58,130 @@ class RequestTest extends TestCase $this->assertTrue($output['second']); $this->assertArrayNotHasKey('deleted', $output); } + + public function testGetParamsWithMultipleMethods(): void + { + $this->setupMultiMethodRoute(); + + // Pass only "foo", should match Method A + $this->request->setQueryString([ + 'foo' => 'valueFoo', + ]); + + $params = $this->request->getParams(); + + $this->assertArrayHasKey('foo', $params); + $this->assertSame('valueFoo', $params['foo']); + $this->assertArrayNotHasKey('baz', $params); + } + + public function testGetParamsWithAllRequired(): void + { + $this->setupMultiMethodRoute(); + + // Pass "foo" and "bar", should match Method A + $this->request->setQueryString([ + 'foo' => 'valueFoo', + 'bar' => 'valueBar', + ]); + + $params = $this->request->getParams(); + $this->assertArrayHasKey('foo', $params); + $this->assertSame('valueFoo', $params['foo']); + $this->assertArrayHasKey('bar', $params); + $this->assertSame('valueBar', $params['bar']); + $this->assertArrayNotHasKey('baz', $params); + } + + public function testGetParamsWithAllOptional(): void + { + $this->setupMultiMethodRoute(); + + // Pass only "bar", should match Method A + $this->request->setQueryString([ + 'bar' => 'valueBar', + ]); + + $params = $this->request->getParams(); + + $this->assertArrayHasKey('bar', $params); + $this->assertSame('valueBar', $params['bar']); + $this->assertArrayNotHasKey('foo', $params); + $this->assertArrayNotHasKey('baz', $params); + } + + public function testGetParamsMatchesMethodB(): void + { + $this->setupMultiMethodRoute(); + + // Pass only "baz", should match Method B + $this->request->setQueryString([ + 'baz' => 'valueBaz', + ]); + + $params = $this->request->getParams(); + + $this->assertArrayHasKey('baz', $params); + $this->assertSame('valueBaz', $params['baz']); + $this->assertArrayNotHasKey('foo', $params); + } + + public function testGetParamsFallbackForMixedAndUnknown(): void + { + $this->setupMultiMethodRoute(); + + // Mixed and unknown should fallback to raw params + $this->request->setQueryString([ + 'foo' => 'valueFoo', + 'baz' => 'valueBaz', + 'extra' => 'unexpected', + ]); + + $params = $this->request->getParams(); + + $this->assertArrayHasKey('foo', $params); + $this->assertSame('valueFoo', $params['foo']); + $this->assertArrayHasKey('baz', $params); + $this->assertSame('valueBaz', $params['baz']); + $this->assertArrayHasKey('extra', $params); + $this->assertSame('unexpected', $params['extra']); + } + + /** + * Helper to attach a route with multiple SDK methods to the request. + */ + private function setupMultiMethodRoute(): void + { + $route = new Route(Request::METHOD_GET, '/multi'); + + $methodA = new Method( + namespace: 'namespace', + group: 'group', + name: 'methodA', + description: 'desc', + auth: [], + responses: [], + parameters: [ + new Parameter('foo'), + new Parameter('bar', optional: true), + ], + ); + + $methodB = new Method( + namespace: 'namespace', + group: 'group', + name: 'methodB', + description: 'desc', + auth: [], + responses: [], + parameters: [ + new Parameter('baz'), + ], + ); + + $route->label('sdk', [$methodA, $methodB]); + $this->request->addFilter(new First()); + $this->request->addFilter(new Second()); + $this->request->setRoute($route); + } } From d04e43c615ab4fff127849e4e49fdb56ae138256 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 12 May 2025 17:50:14 +1200 Subject: [PATCH 02/15] Improve matched case --- src/Appwrite/Utopia/Request.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Utopia/Request.php b/src/Appwrite/Utopia/Request.php index 3e8b7a0a5b..611be4c229 100644 --- a/src/Appwrite/Utopia/Request.php +++ b/src/Appwrite/Utopia/Request.php @@ -36,6 +36,11 @@ class Request extends UtopiaRequest $methods = self::getRoute()->getLabel('sdk', null); $methods = \is_array($methods) ? $methods : [$methods]; + + if (empty($methods)) { + return $parameters; + } + $matched = null; foreach ($methods as $method) { @@ -45,7 +50,7 @@ class Request extends UtopiaRequest } // Find the method that matches the parameters passed - $methodParamNames = \array_map(fn($param) => $param->getName(), $method->getParameters()); + $methodParamNames = \array_map(fn ($param) => $param->getName(), $method->getParameters()); $invalidParams = \array_diff(\array_keys($parameters), $methodParamNames); // No params defined, or all params are valid @@ -60,13 +65,8 @@ class Request extends UtopiaRequest : 'unknown.unknown'; // Filter params to valid keys - if ($matched !== null) { - $definedNames = \array_map(fn($param) => $param->getName(), $matched->getParameters()); - - // If matched method has explicit params, remove all other params - if (!empty($definedNames)) { - $parameters = \array_intersect_key($parameters, \array_flip($definedNames)); - } + if ($matched !== null && !empty($methodParamNames)) { + $parameters = \array_intersect_key($parameters, \array_flip($methodParamNames)); } // Apply filters From 0099eef8c86427f2af29d7a4babecc01374c2aa6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 12 May 2025 19:25:52 +1200 Subject: [PATCH 03/15] Faster short-path for single method routes --- src/Appwrite/Utopia/Request.php | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Utopia/Request.php b/src/Appwrite/Utopia/Request.php index 611be4c229..b80950bbb6 100644 --- a/src/Appwrite/Utopia/Request.php +++ b/src/Appwrite/Utopia/Request.php @@ -35,14 +35,20 @@ class Request extends UtopiaRequest } $methods = self::getRoute()->getLabel('sdk', null); - $methods = \is_array($methods) ? $methods : [$methods]; if (empty($methods)) { return $parameters; } - $matched = null; + if (!\is_array($methods)) { + $id = $methods->getNamespace() . '.' . $methods->getMethodName(); + foreach ($this->getFilters() as $filter) { + $parameters = $filter->parse($parameters, $id); + } + return $parameters; + } + $matched = null; foreach ($methods as $method) { /** @var Method|null $method */ if ($method === null) { @@ -60,7 +66,7 @@ class Request extends UtopiaRequest } } - $endpointIdentifier = $matched !== null + $id = $matched !== null ? $matched->getNamespace() . '.' . $matched->getMethodName() : 'unknown.unknown'; @@ -71,7 +77,7 @@ class Request extends UtopiaRequest // Apply filters foreach ($this->getFilters() as $filter) { - $parameters = $filter->parse($parameters, $endpointIdentifier); + $parameters = $filter->parse($parameters, $id); } return $parameters; From 800dd8bdacf669c64f5af95e8ec093af0a5e50f0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 12 May 2025 19:26:35 +1200 Subject: [PATCH 04/15] Remove redundant filter, action checks which parameter sets are allowed --- src/Appwrite/Utopia/Request.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Appwrite/Utopia/Request.php b/src/Appwrite/Utopia/Request.php index b80950bbb6..c50dea2713 100644 --- a/src/Appwrite/Utopia/Request.php +++ b/src/Appwrite/Utopia/Request.php @@ -70,11 +70,6 @@ class Request extends UtopiaRequest ? $matched->getNamespace() . '.' . $matched->getMethodName() : 'unknown.unknown'; - // Filter params to valid keys - if ($matched !== null && !empty($methodParamNames)) { - $parameters = \array_intersect_key($parameters, \array_flip($methodParamNames)); - } - // Apply filters foreach ($this->getFilters() as $filter) { $parameters = $filter->parse($parameters, $id); From 7c229ad87840bde2c4d7857da3d1544482d70471 Mon Sep 17 00:00:00 2001 From: Fabian Gruber <1951610+basert@users.noreply.github.com> Date: Mon, 12 May 2025 18:05:32 +0200 Subject: [PATCH 05/15] feat(scheduling): add telemetry for scheduler tasks (#9721) --- app/cli.php | 9 +- src/Appwrite/Platform/Tasks/Migrate.php | 6 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 185 +++++++++--------- .../Platform/Tasks/ScheduleExecutions.php | 2 + .../Platform/Tasks/ScheduleFunctions.php | 2 + .../Platform/Tasks/ScheduleMessages.php | 2 +- 6 files changed, 104 insertions(+), 102 deletions(-) diff --git a/app/cli.php b/app/cli.php index fc658c5ad2..9ade97e90c 100644 --- a/app/cli.php +++ b/app/cli.php @@ -26,6 +26,9 @@ use Utopia\Pools\Group; use Utopia\Queue\Publisher; use Utopia\Registry\Registry; use Utopia\System\System; +use Utopia\Telemetry\Adapter\None as NoTelemetry; + +use function Swoole\Coroutine\run; // Overwriting runtimes to be architecture agnostic for CLI Config::setParam('runtimes', (new Runtimes('v4'))->getAll(supported: false)); @@ -200,7 +203,7 @@ CLI::setResource('getLogsDB', function (Group $pools, Cache $cache) { }; }, ['pools', 'cache']); -CLI::setResource('queueForStatsUsage', function (Connection $publisher) { +CLI::setResource('queueForStatsUsage', function (Publisher $publisher) { return new StatsUsage($publisher); }, ['publisher']); CLI::setResource('queueForStatsResources', function (Publisher $publisher) { @@ -264,6 +267,8 @@ CLI::setResource('logError', function (Registry $register) { CLI::setResource('executor', fn () => new Executor(fn (string $projectId, string $deploymentId) => System::getEnv('_APP_EXECUTOR_HOST'))); +CLI::setResource('telemetry', fn () => new NoTelemetry()); + $platform = new Appwrite(); $args = $platform->getEnv('argv'); @@ -293,4 +298,4 @@ $cli $cli->shutdown()->action(fn () => Timer::clearAll()); -$cli->run(); +run($cli->run(...)); diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 4efa78ed4b..e495ce1d3f 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -33,11 +33,7 @@ class Migrate extends Action ->inject('dbForPlatform') ->inject('getProjectDB') ->inject('register') - ->callback(function ($version, $dbForPlatform, $getProjectDB, Registry $register) { - \Co\run(function () use ($version, $dbForPlatform, $getProjectDB, $register) { - $this->action($version, $dbForPlatform, $getProjectDB, $register); - }); - }); + ->callback($this->action(...)); } private function clearProjectsCache(Document $project) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index a3c36cb96e..093c2740ba 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Tasks; +use Swoole\Runtime; use Swoole\Timer; use Utopia\CLI\Console; use Utopia\Database\Database; @@ -13,8 +14,9 @@ use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; use Utopia\Pools\Group; use Utopia\System\System; - -use function Swoole\Coroutine\run; +use Utopia\Telemetry\Adapter as Telemetry; +use Utopia\Telemetry\Gauge; +use Utopia\Telemetry\Histogram; abstract class ScheduleBase extends Action { @@ -23,6 +25,11 @@ abstract class ScheduleBase extends Action protected array $schedules = []; + private ?Histogram $collectSchedulesTelemetryDuration = null; + private ?Gauge $collectSchedulesTelemetryCount = null; + private ?Gauge $scheduleTelemetryCount = null; + private ?Histogram $enqueueDelayTelemetry = null; + abstract public static function getName(): string; abstract public static function getSupportedResource(): string; abstract public static function getCollectionId(): string; @@ -37,7 +44,8 @@ abstract class ScheduleBase extends Action ->inject('pools') ->inject('dbForPlatform') ->inject('getProjectDB') - ->callback(fn (Group $pools, Database $dbForPlatform, callable $getProjectDB) => $this->action($pools, $dbForPlatform, $getProjectDB)); + ->inject('telemetry') + ->callback($this->action(...)); } protected function updateProjectAccess(Document $project, Database $dbForPlatform): void @@ -56,11 +64,44 @@ abstract class ScheduleBase extends Action * 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute * 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutine sleeps until exact time before sending request to worker. */ - public function action(Group $pools, Database $dbForPlatform, callable $getProjectDB): void + public function action(Group $pools, Database $dbForPlatform, callable $getProjectDB, Telemetry $telemetry): void { + Runtime::enableCoroutine(); + Console::title(\ucfirst(static::getSupportedResource()) . ' scheduler V1'); Console::success(APP_NAME . ' ' . \ucfirst(static::getSupportedResource()) . ' scheduler v1 has started'); + $this->scheduleTelemetryCount = $telemetry->createGauge('task.schedule.count'); + $this->collectSchedulesTelemetryDuration = $telemetry->createHistogram('task.schedule.collect_schedules.duration', 's'); + $this->collectSchedulesTelemetryCount = $telemetry->createGauge('task.schedule.collect_schedules.count'); + $this->enqueueDelayTelemetry = $telemetry->createHistogram('task.schedule.enqueue_delay', 's'); + + // start with "0" to load all active documents. + $lastSyncUpdate = "0"; + $this->collectSchedules($pools, $dbForPlatform, $getProjectDB, $lastSyncUpdate); + + Console::success("Starting timers at " . DateTime::now()); + /** + * The timer synchronize $schedules copy with database collection. + */ + Timer::tick(static::UPDATE_TIMER * 1000, function () use ($pools, $dbForPlatform, $getProjectDB, &$lastSyncUpdate) { + $time = DateTime::now(); + Console::log("Sync tick: Running at $time"); + $this->collectSchedules($pools, $dbForPlatform, $getProjectDB, $lastSyncUpdate); + }); + + while (true) { + $this->enqueueResources($pools, $dbForPlatform, $getProjectDB); + $this->scheduleTelemetryCount->record(count($this->schedules), ['resourceType' => static::getSupportedResource()]); + sleep(static::ENQUEUE_TIMER); + } + } + + private function collectSchedules(Group $pools, Database $dbForPlatform, callable $getProjectDB, ?string &$lastSyncUpdate): void + { + // If we haven't synced yet, load all active schedules + $initialLoad = $lastSyncUpdate === "0"; + /** * Extract only necessary attributes to lower memory used. * @@ -68,7 +109,7 @@ abstract class ScheduleBase extends Action * @throws Exception * @var Document $schedule */ - $getSchedule = function (Document $schedule) use ($dbForPlatform, $getProjectDB): array { + $getSchedule = function (Document $schedule) use ($pools, $dbForPlatform, $getProjectDB): array { $project = $dbForPlatform->getDocument('projects', $schedule->getAttribute('projectId')); $resource = $getProjectDB($project)->getDocument( @@ -76,6 +117,8 @@ abstract class ScheduleBase extends Action $schedule->getAttribute('resourceId') ); + $pools->reclaim(); + return [ '$internalId' => $schedule->getInternalId(), '$id' => $schedule->getId(), @@ -88,12 +131,12 @@ abstract class ScheduleBase extends Action ]; }; - $lastSyncUpdate = DateTime::now(); + $loadStart = microtime(true); + $time = DateTime::now(); $limit = 10_000; $sum = $limit; $total = 0; - $loadStart = \microtime(true); $latestDocument = null; while ($sum === $limit) { @@ -110,105 +153,59 @@ abstract class ScheduleBase extends Action $regions[] = 'default'; } - $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ + $paginationQueries = [ + ...$paginationQueries, Query::equal('region', $regions), Query::equal('resourceType', [static::getSupportedResource()]), - Query::equal('active', [true]), - ])); + ]; - $sum = \count($results); + if ($initialLoad) { + $paginationQueries[] = Query::equal('active', [true]); + } else { + $paginationQueries[] = Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate); + } + + $results = $dbForPlatform->find('schedules', $paginationQueries); + + $sum = count($results); $total = $total + $sum; foreach ($results as $document) { - try { - $this->schedules[$document->getInternalId()] = $getSchedule($document); - } catch (\Throwable $th) { - $collectionId = static::getCollectionId(); - Console::error("Failed to load schedule for project {$document['projectId']} {$collectionId} {$document['resourceId']}"); - Console::error($th->getMessage()); + $localDocument = $this->schedules[$document->getInternalId()] ?? null; + + if ($localDocument !== null) { + if (!$document['active']) { + Console::info("Removing: {$document['resourceType']}::{$document['resourceId']}"); + unset($this->schedules[$document->getInternalId()]); + } elseif (strtotime($localDocument['resourceUpdatedAt']) !== strtotime($document['resourceUpdatedAt'])) { + Console::info("Updating: {$document['resourceType']}::{$document['resourceId']}"); + $this->schedules[$document->getInternalId()] = $getSchedule($document); + } + } else { + try { + $this->schedules[$document->getInternalId()] = $getSchedule($document); + } catch (\Throwable $th) { + $collectionId = static::getCollectionId(); + Console::error("Failed to load schedule for project {$document['projectId']} {$collectionId} {$document['resourceId']}"); + Console::error($th->getMessage()); + } } } $latestDocument = \end($results); } - $pools->reclaim(); + $lastSyncUpdate = $time; + $duration = microtime(true) - $loadStart; + $this->collectSchedulesTelemetryDuration->record($duration, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); + $this->collectSchedulesTelemetryCount->record($total, ['initial' => $initialLoad, 'resourceType' => static::getSupportedResource()]); + Console::success("{$total} resources were loaded in " . $duration . " seconds"); + } - Console::success("{$total} resources were loaded in " . (\microtime(true) - $loadStart) . " seconds"); - - Console::success("Starting timers at " . DateTime::now()); - - run(function () use ($dbForPlatform, &$lastSyncUpdate, $getSchedule, $pools, $getProjectDB) { - /** - * The timer synchronize $schedules copy with database collection. - */ - Timer::tick(static::UPDATE_TIMER * 1000, function () use ($dbForPlatform, &$lastSyncUpdate, $getSchedule, $pools) { - $time = DateTime::now(); - $timerStart = \microtime(true); - - $limit = 1000; - $sum = $limit; - $total = 0; - $latestDocument = null; - - Console::log("Sync tick: Running at $time"); - - while ($sum === $limit) { - $paginationQueries = [Query::limit($limit)]; - - if ($latestDocument) { - $paginationQueries[] = Query::cursorAfter($latestDocument); - } - - // Temporarly accepting both 'fra' and 'default' - // When all migrated, only use _APP_REGION with 'default' as default value - $regions = [System::getEnv('_APP_REGION', 'default')]; - if (!in_array('default', $regions)) { - $regions[] = 'default'; - } - - $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', $regions), - Query::equal('resourceType', [static::getSupportedResource()]), - Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), - ])); - - $sum = count($results); - $total = $total + $sum; - - foreach ($results as $document) { - $localDocument = $this->schedules[$document->getInternalId()] ?? null; - - // Check if resource has been updated since last sync - $org = $localDocument !== null ? \strtotime($localDocument['resourceUpdatedAt']) : null; - $new = \strtotime($document['resourceUpdatedAt']); - - if (!$document['active']) { - Console::info("Removing: {$document['resourceType']}::{$document['resourceId']}"); - unset($this->schedules[$document->getInternalId()]); - } elseif ($new !== $org) { - Console::info("Updating: {$document['resourceType']}::{$document['resourceId']}"); - $this->schedules[$document->getInternalId()] = $getSchedule($document); - } - } - - $latestDocument = \end($results); - } - - $lastSyncUpdate = $time; - $timerEnd = \microtime(true); - - $pools->reclaim(); - - Console::log("Sync tick: {$total} schedules were updated in " . ($timerEnd - $timerStart) . " seconds"); - }); - - Timer::tick( - static::ENQUEUE_TIMER * 1000, - fn () => $this->enqueueResources($pools, $dbForPlatform, $getProjectDB) - ); - - $this->enqueueResources($pools, $dbForPlatform, $getProjectDB); - }); + protected function recordEnqueueDelay(string $expectedExecutionSchedule): void + { + $now = strtotime('now'); + $scheduledAt = strtotime($expectedExecutionSchedule); + $this->enqueueDelayTelemetry->record($now - $scheduledAt, ['resourceType' => static::getSupportedResource()]); } } diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 7cd76b480d..79e983f0c3 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -74,6 +74,8 @@ class ScheduleExecutions extends ScheduleBase ->setProject($schedule['project']) ->setUserId($data['userId'] ?? '') ->trigger(); + + $this->recordEnqueueDelay($schedule['schedule']); }); $dbForPlatform->deleteDocument( diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index 5b8e3027a7..abcfe132e3 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -95,6 +95,8 @@ class ScheduleFunctions extends ScheduleBase ->setPath('/') ->setProject($schedule['project']) ->trigger(); + + $this->recordEnqueueDelay($schedule['schedule']); } $queue->reclaim(); diff --git a/src/Appwrite/Platform/Tasks/ScheduleMessages.php b/src/Appwrite/Platform/Tasks/ScheduleMessages.php index 201d5eab53..9b962c99ee 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleMessages.php +++ b/src/Appwrite/Platform/Tasks/ScheduleMessages.php @@ -59,7 +59,7 @@ class ScheduleMessages extends ScheduleBase ); $queue->reclaim(); - + $this->recordEnqueueDelay($schedule['schedule']); unset($this->schedules[$schedule['$internalId']]); }); } From a970512e3cf92adabb33b7bcd23dfbae470a5a71 Mon Sep 17 00:00:00 2001 From: arnab Date: Tue, 13 May 2025 10:35:04 +0530 Subject: [PATCH 06/15] added length param, response model, e2e test --- app/controllers/api/databases.php | 13 ++++---- src/Appwrite/Utopia/Response/Model/Index.php | 7 ++++ .../e2e/Services/Databases/DatabasesBase.php | 33 ++++++++++++++++++- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 0bdb42ec1c..1b7108329f 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2812,12 +2812,13 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') ->param('key', null, new Key(), 'Index Key.') ->param('type', null, new WhiteList([Database::INDEX_KEY, Database::INDEX_FULLTEXT, Database::INDEX_UNIQUE]), 'Index type.') ->param('attributes', null, new ArrayList(new Key(true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of attributes to index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' attributes are allowed, each 32 characters long.') + ->param('lengths', [], new ArrayList(new Integer(), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Length of index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE, optional:true) ->param('orders', [], new ArrayList(new WhiteList(['ASC', 'DESC'], false, Database::VAR_STRING), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of index orders. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' orders are allowed.', true) ->inject('response') ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->action(function (string $databaseId, string $collectionId, string $key, string $type, array $attributes, array $orders, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { + ->action(function (string $databaseId, string $collectionId, string $key, string $type, array $attributes, array $lengths, array $orders, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $db = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); @@ -2877,9 +2878,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') 'size' => 0 ]; - // lengths hidden by default - $lengths = []; - foreach ($attributes as $i => $attribute) { // find attribute metadata in collection document $attributeIndex = \array_search($attribute, array_column($oldAttributes, 'key')); @@ -2901,10 +2899,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') throw new Exception(Exception::ATTRIBUTE_NOT_AVAILABLE, 'Attribute not available: ' . $oldAttributes[$attributeIndex]['key']); } - $lengths[$i] = null; - + $lengths[$i] = array_key_exists($i, $lengths) ? $lengths[$i] : null; if ($attributeArray === true) { - $lengths[$i] = Database::ARRAY_INDEX_LENGTH; + if ($lengths[$i] === null) { + $lengths[$i] = Database::ARRAY_INDEX_LENGTH; + } $orders[$i] = null; } } diff --git a/src/Appwrite/Utopia/Response/Model/Index.php b/src/Appwrite/Utopia/Response/Model/Index.php index 2d795ad439..fcd978b5be 100644 --- a/src/Appwrite/Utopia/Response/Model/Index.php +++ b/src/Appwrite/Utopia/Response/Model/Index.php @@ -41,6 +41,13 @@ class Index extends Model 'example' => [], 'array' => true, ]) + ->addRule('lengths', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Index attributes length.', + 'default' => [], + 'example' => [], + 'array' => true, + ]) ->addRule('orders', [ 'type' => self::TYPE_STRING, 'description' => 'Index orders.', diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 1f19c514d1..3ae88cca57 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -1422,9 +1422,40 @@ trait DatabasesBase return $data; } + /** - * @depends testCreateIndexes + * @depends testCreateAttributes */ + public function testGetIndexByKeyWithLengths(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['moviesId']; + + $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'lengthTestIndex', + 'type' => 'key', + 'attributes' => ['title','description'], + 'lengths' => [128,200] + ]); + + $this->assertEquals(202, $create['headers']['status-code']); + + $index = $this->client->call(Client::METHOD_GET, "/databases/{$databaseId}/collections/{$collectionId}/indexes/lengthTestIndex", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $index['headers']['status-code']); + $this->assertEquals('lengthTestIndex', $index['body']['key']); + $this->assertEquals([128,200], $index['body']['lengths']); + } + /** + * @depends testCreateIndexes + */ public function testListIndexes(array $data): void { $databaseId = $data['databaseId']; From cd806e80d76780d8fcf611cb2fbe3340772f25aa Mon Sep 17 00:00:00 2001 From: arnab Date: Tue, 13 May 2025 15:39:13 +0530 Subject: [PATCH 07/15] updated tests and validations --- app/controllers/api/databases.php | 19 +++++- .../e2e/Services/Databases/DatabasesBase.php | 68 +++++++++++++++++-- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 1b7108329f..3f25f65faf 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2812,13 +2812,13 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') ->param('key', null, new Key(), 'Index Key.') ->param('type', null, new WhiteList([Database::INDEX_KEY, Database::INDEX_FULLTEXT, Database::INDEX_UNIQUE]), 'Index type.') ->param('attributes', null, new ArrayList(new Key(true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of attributes to index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' attributes are allowed, each 32 characters long.') - ->param('lengths', [], new ArrayList(new Integer(), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Length of index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE, optional:true) ->param('orders', [], new ArrayList(new WhiteList(['ASC', 'DESC'], false, Database::VAR_STRING), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of index orders. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' orders are allowed.', true) + ->param('lengths', [], new ArrayList(new Nullable(new Integer()), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Length of index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE, optional:true) ->inject('response') ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->action(function (string $databaseId, string $collectionId, string $key, string $type, array $attributes, array $lengths, array $orders, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { + ->action(function (string $databaseId, string $collectionId, string $key, string $type, array $attributes, array $orders, array $lengths, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $db = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); @@ -2832,6 +2832,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') throw new Exception(Exception::COLLECTION_NOT_FOUND); } + if (count($lengths) > count($attributes)) { + throw new Exception(Exception::INDEX_LENGTHS_INVALID); + } + $count = $dbForProject->count('indexes', [ Query::equal('collectionInternalId', [$collection->getInternalId()]), Query::equal('databaseInternalId', [$db->getInternalId()]) @@ -2878,6 +2882,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') 'size' => 0 ]; + $totalIndexLength = array_sum($lengths); + if ($totalIndexLength > 768) { + throw new Exception(Exception::INDEX_LIMIT_EXCEEDED, 'Index total length crossing 768'); + } + foreach ($attributes as $i => $attribute) { // find attribute metadata in collection document $attributeIndex = \array_search($attribute, array_column($oldAttributes, 'key')); @@ -2899,7 +2908,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') throw new Exception(Exception::ATTRIBUTE_NOT_AVAILABLE, 'Attribute not available: ' . $oldAttributes[$attributeIndex]['key']); } - $lengths[$i] = array_key_exists($i, $lengths) ? $lengths[$i] : null; + if ($lengths[$i] < 0) { + throw new Exception(Exception::INDEX_INVALID, 'Negative index provided for ' . $oldAttributes[$attributeIndex]['key']); + } + + $lengths[$i] ??= null; if ($attributeArray === true) { if ($lengths[$i] === null) { $lengths[$i] = Database::ARRAY_INDEX_LENGTH; diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 3ae88cca57..025394362e 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -1424,13 +1424,14 @@ trait DatabasesBase /** - * @depends testCreateAttributes - */ + * @depends testCreateAttributes + */ public function testGetIndexByKeyWithLengths(array $data): void { $databaseId = $data['databaseId']; $collectionId = $data['moviesId']; + // Test case for valid lengths $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -1441,9 +1442,9 @@ trait DatabasesBase 'attributes' => ['title','description'], 'lengths' => [128,200] ]); - $this->assertEquals(202, $create['headers']['status-code']); + // Fetch index and check correct lengths $index = $this->client->call(Client::METHOD_GET, "/databases/{$databaseId}/collections/{$collectionId}/indexes/lengthTestIndex", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -1451,7 +1452,66 @@ trait DatabasesBase ]); $this->assertEquals(200, $index['headers']['status-code']); $this->assertEquals('lengthTestIndex', $index['body']['key']); - $this->assertEquals([128,200], $index['body']['lengths']); + $this->assertEquals([128, 200], $index['body']['lengths']); + + // Test case for lengths array overriding + $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'lengthOverrideTestIndex', + 'type' => 'key', + 'attributes' => ['title', 'description'], + 'lengths' => [null, 255] + ]); + $this->assertEquals(202, $create['headers']['status-code']); + $index = $this->client->call(Client::METHOD_GET, "/databases/{$databaseId}/collections/{$collectionId}/indexes/lengthOverrideTestIndex", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals([null, 255], $index['body']['lengths']); + + // Test case for count of lengths greater than attributes (should throw 400) + $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'lengthCountExceededIndex', + 'type' => 'key', + 'attributes' => ['title'], + 'lengths' => [128, 128] + ]); + $this->assertEquals(400, $create['headers']['status-code']); + + // Test case for lengths exceeding total of 768 + $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'lengthTooLargeIndex', + 'type' => 'key', + 'attributes' => ['title','description','tagline','actors'], + 'lengths' => [256,256,256,20] + ]); + + $this->assertEquals(400, $create['headers']['status-code']); + + // Test case for negative length values + $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'negativeLengthIndex', + 'type' => 'key', + 'attributes' => ['title'], + 'lengths' => [-1] + ]); + $this->assertEquals(400, $create['headers']['status-code']); } /** * @depends testCreateIndexes From 40642b2aad0b76160a64aa362741bbf20acc3482 Mon Sep 17 00:00:00 2001 From: Fabian Gruber <1951610+basert@users.noreply.github.com> Date: Tue, 13 May 2025 13:03:09 +0200 Subject: [PATCH 08/15] fix(schedules): enqueue delay telemetry in wrong format (#9749) --- src/Appwrite/Platform/Tasks/ScheduleBase.php | 6 ++---- src/Appwrite/Platform/Tasks/ScheduleExecutions.php | 4 ++-- src/Appwrite/Platform/Tasks/ScheduleFunctions.php | 11 ++++++----- src/Appwrite/Platform/Tasks/ScheduleMessages.php | 4 ++-- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index 093c2740ba..d9de41ea64 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -202,10 +202,8 @@ abstract class ScheduleBase extends Action Console::success("{$total} resources were loaded in " . $duration . " seconds"); } - protected function recordEnqueueDelay(string $expectedExecutionSchedule): void + protected function recordEnqueueDelay(\DateTime $expectedExecutionSchedule): void { - $now = strtotime('now'); - $scheduledAt = strtotime($expectedExecutionSchedule); - $this->enqueueDelayTelemetry->record($now - $scheduledAt, ['resourceType' => static::getSupportedResource()]); + $this->enqueueDelayTelemetry->record(time() - $expectedExecutionSchedule->getTimestamp(), ['resourceType' => static::getSupportedResource()]); } } diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 79e983f0c3..89d1609a33 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -59,7 +59,7 @@ class ScheduleExecutions extends ScheduleBase $this->updateProjectAccess($schedule['project'], $dbForPlatform); - \go(function () use ($queueForFunctions, $schedule, $delay, $data) { + \go(function () use ($queueForFunctions, $schedule, $scheduledAt, $delay, $data) { Co::sleep($delay); $queueForFunctions->setType('schedule') @@ -75,7 +75,7 @@ class ScheduleExecutions extends ScheduleBase ->setUserId($data['userId'] ?? '') ->trigger(); - $this->recordEnqueueDelay($schedule['schedule']); + $this->recordEnqueueDelay($scheduledAt); }); $dbForPlatform->deleteDocument( diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index abcfe132e3..689ba831b8 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -66,17 +66,18 @@ class ScheduleFunctions extends ScheduleBase $delayedExecutions[$delay] = []; } - $delayedExecutions[$delay][] = $key; + $delayedExecutions[$delay][] = ['key' => $key, 'nextDate' => $nextDate]; } - foreach ($delayedExecutions as $delay => $scheduleKeys) { - \go(function () use ($delay, $scheduleKeys, $pools, $dbForPlatform) { + foreach ($delayedExecutions as $delay => $schedules) { + \go(function () use ($delay, $schedules, $pools, $dbForPlatform) { \sleep($delay); // in seconds $queue = $pools->get('publisher')->pop(); $connection = $queue->getResource(); - foreach ($scheduleKeys as $scheduleKey) { + foreach ($schedules as $delayConfig) { + $scheduleKey = $delayConfig['key']; // Ensure schedule was not deleted if (!\array_key_exists($scheduleKey, $this->schedules)) { return; @@ -96,7 +97,7 @@ class ScheduleFunctions extends ScheduleBase ->setProject($schedule['project']) ->trigger(); - $this->recordEnqueueDelay($schedule['schedule']); + $this->recordEnqueueDelay($delayConfig['nextDate']); } $queue->reclaim(); diff --git a/src/Appwrite/Platform/Tasks/ScheduleMessages.php b/src/Appwrite/Platform/Tasks/ScheduleMessages.php index 9b962c99ee..a15df6ed5b 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleMessages.php +++ b/src/Appwrite/Platform/Tasks/ScheduleMessages.php @@ -40,7 +40,7 @@ class ScheduleMessages extends ScheduleBase continue; } - \go(function () use ($schedule, $pools, $dbForPlatform) { + \go(function () use ($schedule, $scheduledAt, $pools, $dbForPlatform) { $queue = $pools->get('publisher')->pop(); $connection = $queue->getResource(); $queueForMessaging = new Messaging($connection); @@ -59,7 +59,7 @@ class ScheduleMessages extends ScheduleBase ); $queue->reclaim(); - $this->recordEnqueueDelay($schedule['schedule']); + $this->recordEnqueueDelay($scheduledAt); unset($this->schedules[$schedule['$internalId']]); }); } From 4853e0803ce19b7f2bcdf7b27ebae8febf2c5460 Mon Sep 17 00:00:00 2001 From: Fabian Gruber <1951610+basert@users.noreply.github.com> Date: Tue, 13 May 2025 14:05:13 +0200 Subject: [PATCH 09/15] fix(schedules): better error handling (#9751) --- src/Appwrite/Platform/Tasks/ScheduleBase.php | 13 +++++++++---- src/Appwrite/Platform/Tasks/ScheduleFunctions.php | 8 +++++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index d9de41ea64..afd7d9d22a 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -91,13 +91,18 @@ abstract class ScheduleBase extends Action }); while (true) { - $this->enqueueResources($pools, $dbForPlatform, $getProjectDB); - $this->scheduleTelemetryCount->record(count($this->schedules), ['resourceType' => static::getSupportedResource()]); - sleep(static::ENQUEUE_TIMER); + try { + go(fn () => $this->enqueueResources($pools, $dbForPlatform, $getProjectDB)); + $this->scheduleTelemetryCount->record(count($this->schedules), ['resourceType' => static::getSupportedResource()]); + sleep(static::ENQUEUE_TIMER); + } catch (\Throwable $th) { + Console::error('Failed to enqueue resources: ' . $th->getMessage()); + } + } } - private function collectSchedules(Group $pools, Database $dbForPlatform, callable $getProjectDB, ?string &$lastSyncUpdate): void + private function collectSchedules(Group $pools, Database $dbForPlatform, callable $getProjectDB, string &$lastSyncUpdate): void { // If we haven't synced yet, load all active schedules $initialLoad = $lastSyncUpdate === "0"; diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index 689ba831b8..6788748f3d 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -46,7 +46,13 @@ class ScheduleFunctions extends ScheduleBase $delayedExecutions = []; // Group executions with same delay to share one coroutine foreach ($this->schedules as $key => $schedule) { - $cron = new CronExpression($schedule['schedule']); + try { + $cron = new CronExpression($schedule['schedule']); + } catch (\InvalidArgumentException) { + // ignore invalid cron expressions + continue; + } + $nextDate = $cron->getNextRunDate(); $next = DateTime::format($nextDate); From 41d7114e97282169147fb767e9d77633173c6665 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 13 May 2025 17:37:15 +0530 Subject: [PATCH 10/15] updated composer and index api --- app/controllers/api/databases.php | 13 --------- composer.json | 2 +- composer.lock | 44 +++++++++++++++---------------- 3 files changed, 23 insertions(+), 36 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 3f25f65faf..a56e0a05df 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2832,10 +2832,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - if (count($lengths) > count($attributes)) { - throw new Exception(Exception::INDEX_LENGTHS_INVALID); - } - $count = $dbForProject->count('indexes', [ Query::equal('collectionInternalId', [$collection->getInternalId()]), Query::equal('databaseInternalId', [$db->getInternalId()]) @@ -2882,11 +2878,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') 'size' => 0 ]; - $totalIndexLength = array_sum($lengths); - if ($totalIndexLength > 768) { - throw new Exception(Exception::INDEX_LIMIT_EXCEEDED, 'Index total length crossing 768'); - } - foreach ($attributes as $i => $attribute) { // find attribute metadata in collection document $attributeIndex = \array_search($attribute, array_column($oldAttributes, 'key')); @@ -2908,10 +2899,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') throw new Exception(Exception::ATTRIBUTE_NOT_AVAILABLE, 'Attribute not available: ' . $oldAttributes[$attributeIndex]['key']); } - if ($lengths[$i] < 0) { - throw new Exception(Exception::INDEX_INVALID, 'Negative index provided for ' . $oldAttributes[$attributeIndex]['key']); - } - $lengths[$i] ??= null; if ($attributeArray === true) { if ($lengths[$i] === null) { diff --git a/composer.json b/composer.json index 9e45f71c61..9b2cf7a1ab 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.13.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.68.*", + "utopia-php/database": "0.69.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index cf15b7657e..e6bcd919e2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a5c4c6b723423bc6cb3a7344ab071b43", + "content-hash": "1eeb5a0f3560aefd8f71bd0955e30360", "packages": [ { "name": "adhocore/jwt", @@ -1109,16 +1109,16 @@ }, { "name": "open-telemetry/api", - "version": "1.2.3", + "version": "1.3.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "199d7ddda88f5f5619fa73463f1a5a7149ccd1f1" + "reference": "4e3bb38e069876fb73c2ce85c89583bf2b28cd86" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/199d7ddda88f5f5619fa73463f1a5a7149ccd1f1", - "reference": "199d7ddda88f5f5619fa73463f1a5a7149ccd1f1", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/4e3bb38e069876fb73c2ce85c89583bf2b28cd86", + "reference": "4e3bb38e069876fb73c2ce85c89583bf2b28cd86", "shasum": "" }, "require": { @@ -1175,7 +1175,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-03-05T21:42:54+00:00" + "time": "2025-05-07T12:32:21+00:00" }, { "name": "open-telemetry/context", @@ -1238,16 +1238,16 @@ }, { "name": "open-telemetry/exporter-otlp", - "version": "1.2.1", + "version": "1.3.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/exporter-otlp.git", - "reference": "b7580440b7481a98da97aceabeb46e1b276c8747" + "reference": "19adf03d2b0f91f9e9b1c7f93db6c755c737cf6c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/b7580440b7481a98da97aceabeb46e1b276c8747", - "reference": "b7580440b7481a98da97aceabeb46e1b276c8747", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/19adf03d2b0f91f9e9b1c7f93db6c755c737cf6c", + "reference": "19adf03d2b0f91f9e9b1c7f93db6c755c737cf6c", "shasum": "" }, "require": { @@ -1298,7 +1298,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-03-06T23:21:56+00:00" + "time": "2025-05-12T00:36:35+00:00" }, { "name": "open-telemetry/gen-otlp-protobuf", @@ -1365,16 +1365,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.3.0", + "version": "1.4.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "05d9ceb6773b5bddcf485af6d4a6f543bbeb980b" + "reference": "939d3a28395c249a763676458140dad44b3a8011" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/05d9ceb6773b5bddcf485af6d4a6f543bbeb980b", - "reference": "05d9ceb6773b5bddcf485af6d4a6f543bbeb980b", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/939d3a28395c249a763676458140dad44b3a8011", + "reference": "939d3a28395c249a763676458140dad44b3a8011", "shasum": "" }, "require": { @@ -1451,7 +1451,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-05-01T23:20:43+00:00" + "time": "2025-05-07T12:32:21+00:00" }, { "name": "open-telemetry/sem-conv", @@ -3499,16 +3499,16 @@ }, { "name": "utopia-php/database", - "version": "0.68.1", + "version": "0.69.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "72b2e2c0b875028f7d9dd755f6d4524b693c6507" + "reference": "cc6538e05e25d930244ab938c966d32db0922e83" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/72b2e2c0b875028f7d9dd755f6d4524b693c6507", - "reference": "72b2e2c0b875028f7d9dd755f6d4524b693c6507", + "url": "https://api.github.com/repos/utopia-php/database/zipball/cc6538e05e25d930244ab938c966d32db0922e83", + "reference": "cc6538e05e25d930244ab938c966d32db0922e83", "shasum": "" }, "require": { @@ -3549,9 +3549,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.68.1" + "source": "https://github.com/utopia-php/database/tree/0.69.1" }, - "time": "2025-05-09T10:08:53+00:00" + "time": "2025-05-13T12:00:31+00:00" }, { "name": "utopia-php/domains", From f8fdecaa2dbb16a6727f67732f72e2b85dcc37ab Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 13 May 2025 17:40:04 +0530 Subject: [PATCH 11/15] made the code a bit simpler --- app/controllers/api/databases.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index a56e0a05df..b98c5f0215 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2901,9 +2901,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') $lengths[$i] ??= null; if ($attributeArray === true) { - if ($lengths[$i] === null) { - $lengths[$i] = Database::ARRAY_INDEX_LENGTH; - } + $lengths[$i] ??= Database::ARRAY_INDEX_LENGTH; $orders[$i] = null; } } From 9055645d06e246b8449cd94527b27d211f6f9fc8 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 13 May 2025 18:21:34 +0530 Subject: [PATCH 12/15] updated test for the array attribute index overriding --- tests/e2e/Services/Databases/DatabasesBase.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 025394362e..2b60dee856 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -1455,6 +1455,7 @@ trait DatabasesBase $this->assertEquals([128, 200], $index['body']['lengths']); // Test case for lengths array overriding + // set a length for an array attribute, it should get overriden with Database::ARRAY_INDEX_LENGTH $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -1462,16 +1463,17 @@ trait DatabasesBase ], [ 'key' => 'lengthOverrideTestIndex', 'type' => 'key', - 'attributes' => ['title', 'description'], - 'lengths' => [null, 255] + 'attributes' => ['actors'], + 'lengths' => [120] ]); $this->assertEquals(202, $create['headers']['status-code']); + $index = $this->client->call(Client::METHOD_GET, "/databases/{$databaseId}/collections/{$collectionId}/indexes/lengthOverrideTestIndex", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]); - $this->assertEquals([null, 255], $index['body']['lengths']); + $this->assertEquals([Database::ARRAY_INDEX_LENGTH], $index['body']['lengths']); // Test case for count of lengths greater than attributes (should throw 400) $create = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ From 4208c52c7f828f5118130a2a6c6e5f925b58dc62 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 13 May 2025 18:21:58 +0530 Subject: [PATCH 13/15] reverted overriding index size for index attribute --- app/controllers/api/databases.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index b98c5f0215..cac713948a 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2901,7 +2901,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') $lengths[$i] ??= null; if ($attributeArray === true) { - $lengths[$i] ??= Database::ARRAY_INDEX_LENGTH; + $lengths[$i] = Database::ARRAY_INDEX_LENGTH; $orders[$i] = null; } } From c0b7c47615300142e67cc75711e95bcfefdc78c5 Mon Sep 17 00:00:00 2001 From: Fabian Gruber <1951610+basert@users.noreply.github.com> Date: Wed, 14 May 2025 10:40:56 +0200 Subject: [PATCH 14/15] fix(schedules): disable coroutine until we have proper pool support (#9759) --- src/Appwrite/Platform/Tasks/ScheduleBase.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index afd7d9d22a..8f7eab4d87 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Tasks; -use Swoole\Runtime; use Swoole\Timer; use Utopia\CLI\Console; use Utopia\Database\Database; @@ -66,8 +65,6 @@ abstract class ScheduleBase extends Action */ public function action(Group $pools, Database $dbForPlatform, callable $getProjectDB, Telemetry $telemetry): void { - Runtime::enableCoroutine(); - Console::title(\ucfirst(static::getSupportedResource()) . ' scheduler V1'); Console::success(APP_NAME . ' ' . \ucfirst(static::getSupportedResource()) . ' scheduler v1 has started'); From f62979cfa8e4fc05e221a5e72977b7e48848ce04 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 14 May 2025 15:41:21 +0530 Subject: [PATCH 15/15] feat: inforu adapter (#9620) * chore: update utopia messaging library * chore: init inforu adapter * chore: use dev message * chore: update composer * chore: fix ordering * chore: use user for senderId --- composer.json | 2 +- composer.lock | 14 +++++++------- src/Appwrite/Platform/Workers/Messaging.php | 9 +++++++++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index 9b2cf7a1ab..8b8bb811af 100644 --- a/composer.json +++ b/composer.json @@ -59,7 +59,7 @@ "utopia-php/image": "0.8.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", - "utopia-php/messaging": "0.16.*", + "utopia-php/messaging": "0.17.*", "utopia-php/migration": "0.9.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", diff --git a/composer.lock b/composer.lock index e6bcd919e2..5e86bfafd2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1eeb5a0f3560aefd8f71bd0955e30360", + "content-hash": "2c14e20244a06f508dd67cda717aefeb", "packages": [ { "name": "adhocore/jwt", @@ -3902,16 +3902,16 @@ }, { "name": "utopia-php/messaging", - "version": "0.16.0", + "version": "0.17.0", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "5f3083697102b1821d6624938186761b1e09c54e" + "reference": "c51915d0e030db3a3add37f1561751d18b2d9a85" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/5f3083697102b1821d6624938186761b1e09c54e", - "reference": "5f3083697102b1821d6624938186761b1e09c54e", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/c51915d0e030db3a3add37f1561751d18b2d9a85", + "reference": "c51915d0e030db3a3add37f1561751d18b2d9a85", "shasum": "" }, "require": { @@ -3947,9 +3947,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.16.0" + "source": "https://github.com/utopia-php/messaging/tree/0.17.0" }, - "time": "2025-02-18T08:27:00+00:00" + "time": "2025-05-12T16:14:08+00:00" }, { "name": "utopia-php/migration", diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index cf2b8bfc84..c9eca2a1e0 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -24,6 +24,7 @@ use Utopia\Messaging\Adapter\Push\FCM; use Utopia\Messaging\Adapter\SMS as SMSAdapter; use Utopia\Messaging\Adapter\SMS\Fast2SMS; use Utopia\Messaging\Adapter\SMS\GEOSMS; +use Utopia\Messaging\Adapter\SMS\Inforu; use Utopia\Messaging\Adapter\SMS\Mock; use Utopia\Messaging\Adapter\SMS\Msg91; use Utopia\Messaging\Adapter\SMS\Telesign; @@ -455,6 +456,10 @@ class Messaging extends Action $credentials['messageId'] ?? '', $credentials['useDLT'] ?? true ), + 'inforu' => new Inforu( + $credentials['senderId'] ?? '', + $credentials['apiKey'] ?? '', + ), default => null }; } @@ -780,6 +785,10 @@ class Messaging extends Action 'messageId' => $dsn->getParam('messageId'), 'useDLT' => $dsn->getParam('useDLT'), ], + 'inforu' => [ + 'senderId' => $user, + 'apiKey' => $password, + ], default => null }, 'options' => match ($host) {