From a791ca23155dc7bf3c3860685fc2e42e6914ca34 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Thu, 11 Jun 2026 11:13:50 +0100 Subject: [PATCH 01/24] Revert distributed locking changes --- app/config/errors.php | 5 - app/config/variables.php | 9 - app/controllers/general.php | 27 ++- app/controllers/shared/api.php | 32 ++-- app/init/resources/request.php | 6 - composer.json | 6 - composer.lock | 2 +- docker-compose.yml | 1 - src/Appwrite/Extend/Exception.php | 1 - src/Appwrite/Locking/Lock.php | 235 ------------------------- tests/unit/Locking/LockTest.php | 280 ------------------------------ 11 files changed, 29 insertions(+), 575 deletions(-) delete mode 100644 src/Appwrite/Locking/Lock.php delete mode 100644 tests/unit/Locking/LockTest.php diff --git a/app/config/errors.php b/app/config/errors.php index a51a83e529..a548d75179 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -54,11 +54,6 @@ return [ 'description' => 'Rate limit for the current endpoint has been exceeded. Please try again after some time.', 'code' => 429, ], - Exception::GENERAL_RESOURCE_LOCKED => [ - 'name' => Exception::GENERAL_RESOURCE_LOCKED, - 'description' => 'The requested resource is currently being modified by another request. Please retry after a brief delay.', - 'code' => 409, - ], Exception::GENERAL_SMTP_DISABLED => [ 'name' => Exception::GENERAL_SMTP_DISABLED, 'description' => 'SMTP is disabled on your Appwrite instance. You can learn more about setting up SMTP in our docs.', diff --git a/app/config/variables.php b/app/config/variables.php index 23d4eaf315..90df9b4518 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -34,15 +34,6 @@ return [ 'question' => '', 'filter' => '' ], - [ - 'name' => '_APP_LOCKING_ENABLED', - 'description' => 'Enable distributed locking for platform writes. Locks coordinate concurrent updates across API pods so that read-modify-write operations on shared documents do not lose updates. By default, set to \'enabled\'. Set to \'disabled\' as an emergency kill switch — locks become no-ops (fail-open) and concurrent writes will race.', - 'introduction' => '1.9.3', - 'default' => 'enabled', - 'required' => false, - 'question' => '', - 'filter' => '' - ], [ 'name' => '_APP_OPTIONS_FORCE_HTTPS', 'description' => 'Allows you to force HTTPS connection to your API. This feature redirects any HTTP call to HTTPS and adds the \'Strict-Transport-Security\' header to all HTTP responses. By default, set to \'enabled\'. To disable, set to \'disabled\'. This feature will work only when your ports are set to default 80 and 443, and you have set up wildcard certificates with DNS challenge.', diff --git a/app/controllers/general.php b/app/controllers/general.php index 9e26af3198..94caf4760d 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -12,7 +12,6 @@ use Appwrite\Event\Message\Delete as DeleteMessage; use Appwrite\Event\Publisher\Certificate; use Appwrite\Event\Publisher\Delete as DeletePublisher; use Appwrite\Extend\Exception as AppwriteException; -use Appwrite\Locking\Lock; use Appwrite\Network\Cors; use Appwrite\Platform\Appwrite; use Appwrite\SDK\Method; @@ -76,7 +75,7 @@ use Utopia\Validator\Text; Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) +function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { $host = $request->getHostname(); if (!empty($previewHostname)) { @@ -145,7 +144,9 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S if (!$project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $lock->set('projects', $project->getId(), 'accessedAt', DateTime::now()); + $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ + 'accessedAt' => DateTime::now() + ]))); } /** @@ -839,9 +840,8 @@ Http::init() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('lock') ->inject('params') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock, array $params) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, array $params) { /* * Appwrite Router */ @@ -849,7 +849,7 @@ Http::init() $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!\in_array($hostname, $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { $utopia->match($request)?->route->label('router', true); } } @@ -1152,15 +1152,14 @@ Http::options() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('lock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { /* * Appwrite Router */ $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { $utopia->match($request)?->route->label('router', true); } } @@ -1555,14 +1554,13 @@ Http::get('/robots.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('lock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { $utopia->match($request)?->route->label('router', true); } } @@ -1590,14 +1588,13 @@ Http::get('/humans.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('lock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { $utopia->match($request)?->route->label('router', true); } } diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index e11ba433ee..4b5f7170ff 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -16,7 +16,6 @@ use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\EventProcessor; -use Appwrite\Locking\Lock; use Appwrite\Platform\Modules\Storage\Config\CacheControl; use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl; use Appwrite\SDK\Method; @@ -100,8 +99,7 @@ Http::init() ->inject('team') ->inject('apiKey') ->inject('authorization') - ->inject('lock') - ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Lock $lock) { + ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization) { /** * Handle user authentication and session validation. @@ -242,22 +240,20 @@ Http::init() $sdks[] = $sdk; $updates->setAttribute('sdks', $sdks); - $updates->setAttribute('accessedAt', DateTime::now()); + $updates->setAttribute('accessedAt', Datetime::now()); } } if (! $updates->isEmpty()) { - $lock->run('keys', $dbKey->getId(), function () use ($dbForPlatform, $dbKey, $updates, $apiKey, $project, $user, $team) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); - if (! empty($apiKey->getProjectId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); - } elseif (! empty($apiKey->getUserId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); - } elseif (! empty($apiKey->getTeamId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); - } - }); + if (! empty($apiKey->getProjectId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); + } elseif (! empty($apiKey->getUserId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); + } elseif (! empty($apiKey->getTeamId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); + } } $userClone = clone $user; @@ -388,7 +384,9 @@ Http::init() if ($project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $lock->set('projects', $project->getId(), 'accessedAt', DateTime::now()); + $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ + 'accessedAt' => DateTime::now() + ]))); } } @@ -405,7 +403,9 @@ Http::init() 'accessedAt' => $user->getAttribute('accessedAt') ])); } else { - $lock->set('users', $user->getId(), 'accessedAt', $user->getAttribute('accessedAt')); + $authorization->skip(fn () => $dbForPlatform->updateDocument('users', $user->getId(), new Document([ + 'accessedAt' => $user->getAttribute('accessedAt') + ]))); } } } diff --git a/app/init/resources/request.php b/app/init/resources/request.php index fa480034f0..982cd13088 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -14,7 +14,6 @@ use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\GraphQL\Schema; -use Appwrite\Locking\Lock; use Appwrite\Network\Cors; use Appwrite\Network\Platform; use Appwrite\Network\Validator\Origin; @@ -45,7 +44,6 @@ use Utopia\Domains\Domain; use Utopia\Http\Http; use Utopia\Locale\Locale; use Utopia\Logger\Log; -use Utopia\Logger\Logger; use Utopia\Pools\Group; use Utopia\Queue\Publisher; use Utopia\Queue\Queue; @@ -67,10 +65,6 @@ return function (Container $context): void { $context->set('logger', fn ($register) => $register->get('logger'), ['register']); - $context->set('lock', function (\Redis $redis, Telemetry $telemetry, Database $dbForPlatform, Authorization $authorization, Log $log, ?Logger $logger, Document $project): Lock { - return new Lock($redis, $telemetry, $dbForPlatform, $authorization, $log, $logger, $project); - }, ['redis', 'telemetry', 'dbForPlatform', 'authorization', 'log', 'logger', 'project']); - $context->set('authorization', fn () => new Authorization(), []); $context->set('store', fn (): Store => new Store(), []); diff --git a/composer.json b/composer.json index 0e45b48ffe..de5a1ba601 100644 --- a/composer.json +++ b/composer.json @@ -100,12 +100,6 @@ "league/csv": "9.14.*", "enshrined/svg-sanitize": "0.22.*" }, - "repositories": [ - { - "type": "vcs", - "url": "https://github.com/utopia-php/lock" - } - ], "require-dev": { "ext-fileinfo": "*", "appwrite/sdk-generator": "^2.0", diff --git a/composer.lock b/composer.lock index 5fb89d8d4c..1e57ede67b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9f9908fb48b239cd403e78fb9e1cdbb8", + "content-hash": "1842adc3102a8d4af14a013751acd366", "packages": [ { "name": "adhocore/jwt", diff --git a/docker-compose.yml b/docker-compose.yml index 3a954cebd1..212b0f1f47 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -212,7 +212,6 @@ services: - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - _APP_LOGGING_CONFIG - - _APP_LOCKING_ENABLED - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 349cff11de..ca114550b7 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -45,7 +45,6 @@ class Exception extends \Exception public const string GENERAL_SERVICE_DISABLED = 'general_service_disabled'; public const string GENERAL_UNAUTHORIZED_SCOPE = 'general_unauthorized_scope'; public const string GENERAL_RATE_LIMIT_EXCEEDED = 'general_rate_limit_exceeded'; - public const string GENERAL_RESOURCE_LOCKED = 'general_resource_locked'; public const string GENERAL_SMTP_DISABLED = 'general_smtp_disabled'; public const string GENERAL_PHONE_DISABLED = 'general_phone_disabled'; public const string GENERAL_ARGUMENT_INVALID = 'general_argument_invalid'; diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php deleted file mode 100644 index 9ef6d70c03..0000000000 --- a/src/Appwrite/Locking/Lock.php +++ /dev/null @@ -1,235 +0,0 @@ - */ - private static array $lastReportAt = []; - - public function __construct( - private readonly \Redis $redis, - Telemetry $telemetry, - private readonly Database $dbForPlatform, - private readonly Authorization $authorization, - private readonly Log $log, - private readonly ?Logger $logger, - Document $project, - ) { - $this->enabled = System::getEnv('_APP_LOCKING_ENABLED', 'enabled') !== 'disabled'; - $this->attempts = $telemetry->createCounter('lock.attempts', null, 'Distributed lock acquire outcomes'); - $sequence = $project->getSequence(); - $this->projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; - } - - /** - * Throttled single-attribute write under a per-attribute skip-on-contention - * lock with authorization bypass. For idempotent timestamp-style updates - * (accessedAt, mcpAccessedAt) where regional pods writing the same value - * would thrash the platform DB. - */ - public function set( - string $collection, - string $id, - string $attribute, - string $value, - ): void { - $key = "lock:platform:{$this->projectInternalId}:{$collection}:{$id}:{$attribute}"; - $this->execute($key, $collection, function () use ($collection, $id, $attribute, $value) { - $this->authorization->skip(fn () => $this->dbForPlatform->updateDocument( - $collection, - $id, - new Document([$attribute => $value]) - )); - }); - } - - /** - * Skip-on-contention lock around an arbitrary callback for a platform - * document. For idempotent multi-statement writes that don't fit `set`. - */ - public function run(string $collection, string $id, Closure $fn): void - { - $key = "lock:platform:{$this->projectInternalId}:{$collection}:{$id}"; - $this->execute($key, $collection, $fn); - } - - /** - * Block-then-409 lock around an arbitrary callback for a platform document. - * For read-modify-write endpoints where silently dropping a concurrent - * request would lose user data. - */ - public function runOrFail(string $collection, string $id, Closure $fn): mixed - { - $key = "lock:platform:{$this->projectInternalId}:{$collection}:{$id}"; - - return $this->execute($key, $collection, $fn, ttl: self::FAIL_TTL_SECONDS, orFail: true); - } - - /** - * Generic lock primitive with full control over key, TTL, contention - * behavior, and wait timeout. Escape hatch for non-platform keys - * (cache, queue, edge) and for unusual TTL/timeout requirements. - * - * Caller may pass `target` for telemetry; otherwise it's extracted by - * position from the key (best-effort for keys following the standard - * `lock:platform:{project}:{target}:...` shape). - */ - public function withKey( - string $key, - Closure $fn, - int $ttl = self::SKIP_TTL_SECONDS, - bool $orFail = false, - float $waitTimeout = self::FAIL_WAIT_SECONDS, - ?string $target = null, - ): mixed { - return $this->execute( - $key, - $target ?? self::inferTargetFromKey($key), - $fn, - ttl: $ttl, - orFail: $orFail, - waitTimeout: $waitTimeout, - ); - } - - private function execute( - string $key, - string $target, - Closure $fn, - int $ttl = self::SKIP_TTL_SECONDS, - bool $orFail = false, - float $waitTimeout = self::FAIL_WAIT_SECONDS, - ): mixed { - if (! $this->enabled) { - return $fn(); - } - - $lock = new DistributedLock($this->redis, $key, $ttl); - $labels = ['target' => $target, 'project' => $this->projectInternalId]; - - try { - $acquired = $orFail ? $lock->acquire($waitTimeout) : $lock->tryAcquire(); - } catch (\RedisException $e) { - $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); - $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); - - return $fn(); - } - - if (! $acquired) { - if ($orFail) { - $this->attempts->add(1, ['outcome' => self::OUTCOME_CONTENDED, ...$labels]); - // No custom message — the lock key embeds collection + document id. - throw new Exception(Exception::GENERAL_RESOURCE_LOCKED); - } - $this->attempts->add(1, ['outcome' => self::OUTCOME_SKIPPED, ...$labels]); - - return null; - } - - $this->attempts->add(1, ['outcome' => self::OUTCOME_ACQUIRED, ...$labels]); - try { - return $fn(); - } finally { - try { - $lock->release(); - } catch (Throwable $e) { - $this->attempts->add(1, ['outcome' => self::OUTCOME_RELEASE_ERROR, ...$labels]); - $this->reportError(self::OUTCOME_RELEASE_ERROR, $key, $target, $e); - } - } - } - - /** - * Best-effort target extraction for telemetry. Assumes the standard - * `lock:platform:{project}:{target}:...` shape. For non-platform keys - * passed via withKey(), callers should pass `target` explicitly. - */ - private static function inferTargetFromKey(string $key): string - { - $parts = explode(':', $key, 5); - - return $parts[3] ?? 'unknown'; - } - - /** - * Rate-limited to one push per REPORT_RATE_LIMIT_SECONDS per (action, target) - * so a sustained backend outage doesn't flood Sentry across the pod fleet. - */ - private function reportError(string $action, string $key, string $target, Throwable $e): void - { - Console::warning("Lock {$action} for {$key}: {$e->getMessage()}"); - - if ($this->logger === null) { - return; - } - - $bucket = $action.':'.$target; - $now = time(); - if ((self::$lastReportAt[$bucket] ?? 0) + self::REPORT_RATE_LIMIT_SECONDS > $now) { - return; - } - self::$lastReportAt[$bucket] = $now; - - $this->log->setNamespace('http'); - $this->log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); - $this->log->setVersion(APP_VERSION_STABLE); - $this->log->setType(Log::TYPE_WARNING); - $this->log->setMessage('Distributed lock '.$action.': '.$e->getMessage()); - $this->log->setAction("lock.{$action}"); - $this->log->setEnvironment(System::getEnv('_APP_ENV', 'development') === 'production' - ? Log::ENVIRONMENT_PRODUCTION - : Log::ENVIRONMENT_STAGING); - $this->log->addTag('lock.target', $target); - $this->log->addTag('lock.project', $this->projectInternalId); - // Strip trailing document ID to keep aggregator cardinality bounded. - $this->log->addTag('lock.key_pattern', preg_replace('/:[^:]+$/', ':*', $key)); - $this->log->addTag('code', $e->getCode()); - $this->log->addExtra('file', $e->getFile()); - $this->log->addExtra('line', $e->getLine()); - $this->log->addExtra('trace', $e->getTraceAsString()); - - try { - $this->logger->addLog($this->log); - } catch (Throwable) { - } - } -} diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php deleted file mode 100644 index 4a0436acb3..0000000000 --- a/tests/unit/Locking/LockTest.php +++ /dev/null @@ -1,280 +0,0 @@ -getRedisConnection(); - - $this->redis = new Redis(); - $this->redis->connect($host, $port, 1.0); - - $this->project = new Document([ - '$id' => 'test-project', - '$sequence' => self::PROJECT_SEQUENCE, - ]); - $this->authorization = new Authorization(); - $this->log = new Log(); - - $this->cleanupKeys(); - } - - /** - * @return array{0: string, 1: int} - */ - private function getRedisConnection(): array - { - $dsn = \getenv('_APP_CONNECTIONS_CACHE') ?: ''; - - if ($dsn !== '') { - $dsn = \explode(',', $dsn, 2)[0]; - $dsn = \explode('=', $dsn, 2)[1] ?? $dsn; - $parsed = new DSN($dsn); - - return [$parsed->getHost(), (int) ($parsed->getPort() ?? '6379')]; - } - - return [ - \getenv('_APP_REDIS_HOST') ?: 'redis', - (int) (\getenv('_APP_REDIS_PORT') ?: 6379), - ]; - } - - protected function tearDown(): void - { - if (isset($this->redis) && $this->redis->isConnected()) { - $this->cleanupKeys(); - } - } - - private function cleanupKeys(): void - { - foreach ($this->redis->keys(self::KEY_PREFIX.'*') as $key) { - $this->redis->del($key); - } - // also clean keys produced by withKey() tests - foreach ($this->redis->keys('lock:test:*') as $key) { - $this->redis->del($key); - } - } - - private function makeLock(?Database $db = null, ?Authorization $auth = null): Lock - { - return new Lock( - $this->redis, - new NoTelemetry(), - $db ?? $this->createStub(Database::class), - $auth ?? $this->authorization, - $this->log, - null, - $this->project, - ); - } - - public function test_set_uses_per_attribute_key_and_auth_skipped_update(): void - { - $captured = null; - $db = $this->createMock(Database::class); - $db->expects($this->once()) - ->method('updateDocument') - ->with('projects', 'p1', $this->callback(function (Document $doc) use (&$captured) { - $captured = $doc->getArrayCopy(); - - return true; - })) - ->willReturnArgument(2); - - $lock = $this->makeLock($db); - $lock->set('projects', 'p1', 'accessedAt', '2024-06-01 12:00:00'); - - $this->assertSame(['accessedAt' => '2024-06-01 12:00:00'], $captured); - $this->assertSame(0, $this->redis->exists(self::KEY_PREFIX.'projects:p1:accessedAt')); - } - - public function test_set_skips_on_contention(): void - { - $key = self::KEY_PREFIX.'projects:p1:accessedAt'; - $this->redis->set($key, 'other-owner', ['NX', 'EX' => 30]); - - $db = $this->createMock(Database::class); - $db->expects($this->never())->method('updateDocument'); - - $lock = $this->makeLock($db); - $lock->set('projects', 'p1', 'accessedAt', '2024-06-01 12:00:00'); - - $this->assertSame('other-owner', $this->redis->get($key)); - } - - public function test_set_different_attributes_do_not_compete(): void - { - // Hold accessedAt - $heldKey = self::KEY_PREFIX.'projects:p1:accessedAt'; - $this->redis->set($heldKey, 'other-owner', ['NX', 'EX' => 30]); - - // mcpAccessedAt should still be acquirable - $db = $this->createMock(Database::class); - $db->expects($this->once()) - ->method('updateDocument') - ->with('projects', 'p1', $this->isInstanceOf(Document::class)) - ->willReturnArgument(2); - - $lock = $this->makeLock($db); - $lock->set('projects', 'p1', 'mcpAccessedAt', '2024-06-01 12:00:00'); - - $this->assertSame('other-owner', $this->redis->get($heldKey)); - } - - public function test_run_uses_per_document_key_and_invokes_callback(): void - { - $called = false; - $lock = $this->makeLock(); - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - - $this->assertTrue($called); - $this->assertSame(0, $this->redis->exists(self::KEY_PREFIX.'keys:k1')); - } - - public function test_run_skips_on_contention(): void - { - $key = self::KEY_PREFIX.'keys:k1'; - $this->redis->set($key, 'other-owner', ['NX', 'EX' => 30]); - - $called = false; - $lock = $this->makeLock(); - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - - $this->assertFalse($called); - $this->assertSame('other-owner', $this->redis->get($key)); - } - - public function test_run_or_fail_throws_on_contention(): void - { - $key = self::KEY_PREFIX.'projects:p1'; - $this->redis->set($key, 'other-owner', ['NX', 'EX' => 30]); - - $lock = $this->makeLock(); - - $this->expectException(Exception::class); - try { - $lock->runOrFail('projects', 'p1', fn () => 'never-runs'); - } catch (Exception $e) { - $this->assertSame(Exception::GENERAL_RESOURCE_LOCKED, $e->getType()); - throw $e; - } - } - - public function test_run_or_fail_returns_callback_value_when_uncontended(): void - { - $lock = $this->makeLock(); - $result = $lock->runOrFail('projects', 'p1', fn () => 'ok'); - - $this->assertSame('ok', $result); - $this->assertSame(0, $this->redis->exists(self::KEY_PREFIX.'projects:p1')); - } - - public function test_with_key_uses_raw_key(): void - { - $custom = 'lock:test:custom-key'; - $called = false; - $lock = $this->makeLock(); - $lock->withKey($custom, function () use (&$called) { - $called = true; - }); - - $this->assertTrue($called); - $this->assertSame(0, $this->redis->exists($custom)); - } - - public function test_with_key_or_fail_flag_throws_on_contention(): void - { - $custom = 'lock:test:contended'; - $this->redis->set($custom, 'other', ['NX', 'EX' => 30]); - - $lock = $this->makeLock(); - $this->expectException(Exception::class); - $lock->withKey($custom, fn () => null, ttl: 5, orFail: true, waitTimeout: 0.1); - } - - public function test_disabled_mode_runs_callback_unlocked(): void - { - $previous = \getenv('_APP_LOCKING_ENABLED'); - \putenv('_APP_LOCKING_ENABLED=disabled'); - try { - // Even when the key is already held, the callback must still run. - $key = self::KEY_PREFIX.'keys:k1'; - $this->redis->set($key, 'other-owner', ['NX', 'EX' => 30]); - - $called = false; - $lock = $this->makeLock(); - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - - $this->assertTrue($called); - $this->assertSame('other-owner', $this->redis->get($key)); - } finally { - $previous === false ? \putenv('_APP_LOCKING_ENABLED') : \putenv('_APP_LOCKING_ENABLED='.$previous); - } - } - - public function test_project_without_sequence_falls_back_to_unknown(): void - { - $emptyProject = new Document(); - $lock = new Lock( - $this->redis, - new NoTelemetry(), - $this->createStub(Database::class), - $this->authorization, - $this->log, - null, - $emptyProject, - ); - - // Pre-acquire the lock at the 'unknown' projectInternalId path. - $key = 'lock:platform:unknown:keys:k1'; - $this->redis->set($key, 'held', ['NX', 'EX' => 30]); - - $called = false; - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - $this->assertFalse($called, 'Lock without project sequence should hash to the unknown bucket'); - - $this->redis->del($key); - } -} From 6138529eb654aa1834e78c3e4b5f9abd8a13f58e Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Thu, 11 Jun 2026 11:22:39 +0100 Subject: [PATCH 02/24] Fix DateTime casing in API controller --- app/controllers/shared/api.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 4b5f7170ff..a7b5df1e1b 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -240,7 +240,7 @@ Http::init() $sdks[] = $sdk; $updates->setAttribute('sdks', $sdks); - $updates->setAttribute('accessedAt', Datetime::now()); + $updates->setAttribute('accessedAt', DateTime::now()); } } From 0733c32c2890307580e2ab35ce107889c374e42d Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Thu, 11 Jun 2026 11:41:59 +0100 Subject: [PATCH 03/24] Reintroduce platform write locking --- app/config/errors.php | 5 + app/config/variables.php | 9 + app/controllers/general.php | 27 ++- app/controllers/shared/api.php | 32 +-- app/init/resources/request.php | 22 ++ docker-compose.yml | 1 + src/Appwrite/Extend/Exception.php | 1 + src/Appwrite/Locking/Lock.php | 238 +++++++++++++++++++ src/Appwrite/Locking/PlatformLock.php | 42 ++++ tests/unit/Locking/LockTest.php | 297 ++++++++++++++++++++++++ tests/unit/Locking/PlatformLockTest.php | 164 +++++++++++++ 11 files changed, 811 insertions(+), 27 deletions(-) create mode 100644 src/Appwrite/Locking/Lock.php create mode 100644 src/Appwrite/Locking/PlatformLock.php create mode 100644 tests/unit/Locking/LockTest.php create mode 100644 tests/unit/Locking/PlatformLockTest.php diff --git a/app/config/errors.php b/app/config/errors.php index a548d75179..a51a83e529 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -54,6 +54,11 @@ return [ 'description' => 'Rate limit for the current endpoint has been exceeded. Please try again after some time.', 'code' => 429, ], + Exception::GENERAL_RESOURCE_LOCKED => [ + 'name' => Exception::GENERAL_RESOURCE_LOCKED, + 'description' => 'The requested resource is currently being modified by another request. Please retry after a brief delay.', + 'code' => 409, + ], Exception::GENERAL_SMTP_DISABLED => [ 'name' => Exception::GENERAL_SMTP_DISABLED, 'description' => 'SMTP is disabled on your Appwrite instance. You can learn more about setting up SMTP in our docs.', diff --git a/app/config/variables.php b/app/config/variables.php index 90df9b4518..6fef4ecbe0 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -34,6 +34,15 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_LOCKING_ENABLED', + 'description' => 'Enable distributed locking for platform writes. Locks coordinate concurrent updates across API pods so read-modify-write operations on shared documents do not lose updates. By default, set to \'enabled\'. Set to \'disabled\' as an emergency kill switch; locks become no-ops and concurrent writes will race.', + 'introduction' => '1.9.3', + 'default' => 'enabled', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_OPTIONS_FORCE_HTTPS', 'description' => 'Allows you to force HTTPS connection to your API. This feature redirects any HTTP call to HTTPS and adds the \'Strict-Transport-Security\' header to all HTTP responses. By default, set to \'enabled\'. To disable, set to \'disabled\'. This feature will work only when your ports are set to default 80 and 443, and you have set up wildcard certificates with DNS challenge.', diff --git a/app/controllers/general.php b/app/controllers/general.php index 94caf4760d..2af3cb1d6c 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -12,6 +12,7 @@ use Appwrite\Event\Message\Delete as DeleteMessage; use Appwrite\Event\Publisher\Certificate; use Appwrite\Event\Publisher\Delete as DeletePublisher; use Appwrite\Extend\Exception as AppwriteException; +use Appwrite\Locking\PlatformLock; use Appwrite\Network\Cors; use Appwrite\Platform\Appwrite; use Appwrite\SDK\Method; @@ -75,7 +76,7 @@ use Utopia\Validator\Text; Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) +function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformLock $platformLock) { $host = $request->getHostname(); if (!empty($previewHostname)) { @@ -144,9 +145,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S if (!$project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ - 'accessedAt' => DateTime::now() - ]))); + $platformLock->set('projects', $project->getId(), 'accessedAt', DateTime::now()); } /** @@ -840,8 +839,9 @@ Http::init() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') + ->inject('platformLock') ->inject('params') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, array $params) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformLock $platformLock, array $params) { /* * Appwrite Router */ @@ -849,7 +849,7 @@ Http::init() $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!\in_array($hostname, $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformLock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1152,14 +1152,15 @@ Http::options() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { + ->inject('platformLock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformLock $platformLock) { /* * Appwrite Router */ $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformLock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1554,13 +1555,14 @@ Http::get('/robots.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { + ->inject('platformLock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformLock $platformLock) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformLock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1588,13 +1590,14 @@ Http::get('/humans.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { + ->inject('platformLock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformLock $platformLock) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformLock)) { $utopia->match($request)?->route->label('router', true); } } diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index a7b5df1e1b..b6c252227c 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -16,6 +16,8 @@ use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\EventProcessor; +use Appwrite\Locking\Lock; +use Appwrite\Locking\PlatformLock; use Appwrite\Platform\Modules\Storage\Config\CacheControl; use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl; use Appwrite\SDK\Method; @@ -99,7 +101,9 @@ Http::init() ->inject('team') ->inject('apiKey') ->inject('authorization') - ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization) { + ->inject('lock') + ->inject('platformLock') + ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Lock $lock, PlatformLock $platformLock) { /** * Handle user authentication and session validation. @@ -245,15 +249,17 @@ Http::init() } if (! $updates->isEmpty()) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); + $lock->run('keys', $dbKey->getId(), function () use ($dbForPlatform, $dbKey, $updates, $apiKey, $project, $user, $team) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); - if (! empty($apiKey->getProjectId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); - } elseif (! empty($apiKey->getUserId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); - } elseif (! empty($apiKey->getTeamId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); - } + if (! empty($apiKey->getProjectId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); + } elseif (! empty($apiKey->getUserId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); + } elseif (! empty($apiKey->getTeamId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); + } + }); } $userClone = clone $user; @@ -384,9 +390,7 @@ Http::init() if ($project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ - 'accessedAt' => DateTime::now() - ]))); + $platformLock->set('projects', $project->getId(), 'accessedAt', DateTime::now()); } } @@ -403,9 +407,7 @@ Http::init() 'accessedAt' => $user->getAttribute('accessedAt') ])); } else { - $authorization->skip(fn () => $dbForPlatform->updateDocument('users', $user->getId(), new Document([ - 'accessedAt' => $user->getAttribute('accessedAt') - ]))); + $platformLock->set('users', $user->getId(), 'accessedAt', $user->getAttribute('accessedAt')); } } } diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 982cd13088..24a4c0e127 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -14,6 +14,8 @@ use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\GraphQL\Schema; +use Appwrite\Locking\Lock; +use Appwrite\Locking\PlatformLock; use Appwrite\Network\Cors; use Appwrite\Network\Platform; use Appwrite\Network\Validator\Origin; @@ -43,7 +45,9 @@ use Utopia\DI\Container; use Utopia\Domains\Domain; use Utopia\Http\Http; use Utopia\Locale\Locale; +use Utopia\Lock\Distributed as DistributedLock; use Utopia\Logger\Log; +use Utopia\Logger\Logger; use Utopia\Pools\Group; use Utopia\Queue\Publisher; use Utopia\Queue\Queue; @@ -65,6 +69,24 @@ return function (Container $context): void { $context->set('logger', fn ($register) => $register->get('logger'), ['register']); + $context->set('lock', function (Group $pools, Telemetry $telemetry, Log $log, ?Logger $logger, Document $project): Lock { + return new Lock( + fn (string $key, int $ttl, Closure $callback): mixed => $pools->get('lock')->use( + fn (\Redis $redis): mixed => $callback(new DistributedLock($redis, $key, $ttl)) + ), + $telemetry, + $log, + $logger, + $project + ); + }, ['pools', 'telemetry', 'log', 'logger', 'project']); + + $context->set('platformLock', fn (Lock $lock, Database $dbForPlatform, Authorization $authorization): PlatformLock => new PlatformLock( + $lock, + $dbForPlatform, + $authorization, + ), ['lock', 'dbForPlatform', 'authorization']); + $context->set('authorization', fn () => new Authorization(), []); $context->set('store', fn (): Store => new Store(), []); diff --git a/docker-compose.yml b/docker-compose.yml index 212b0f1f47..3a954cebd1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -212,6 +212,7 @@ services: - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - _APP_LOGGING_CONFIG + - _APP_LOCKING_ENABLED - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index ca114550b7..349cff11de 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -45,6 +45,7 @@ class Exception extends \Exception public const string GENERAL_SERVICE_DISABLED = 'general_service_disabled'; public const string GENERAL_UNAUTHORIZED_SCOPE = 'general_unauthorized_scope'; public const string GENERAL_RATE_LIMIT_EXCEEDED = 'general_rate_limit_exceeded'; + public const string GENERAL_RESOURCE_LOCKED = 'general_resource_locked'; public const string GENERAL_SMTP_DISABLED = 'general_smtp_disabled'; public const string GENERAL_PHONE_DISABLED = 'general_phone_disabled'; public const string GENERAL_ARGUMENT_INVALID = 'general_argument_invalid'; diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php new file mode 100644 index 0000000000..7bd7e7db28 --- /dev/null +++ b/src/Appwrite/Locking/Lock.php @@ -0,0 +1,238 @@ + */ + private static array $lastReportAt = []; + + /** + * @var Closure(string, int, Closure(UtopiaLock): mixed): mixed + */ + private readonly Closure $withLock; + + public function __construct( + Closure $withLock, + Telemetry $telemetry, + private readonly Log $log, + private readonly ?Logger $logger, + Document $project, + ) { + $this->withLock = $withLock; + $this->enabled = System::getEnv('_APP_LOCKING_ENABLED', 'enabled') !== 'disabled'; + $this->attempts = $telemetry->createCounter('lock.attempts', null, 'Distributed lock acquire outcomes'); + $sequence = $project->getSequence(); + $this->projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; + } + + /** + * Skip-on-contention lock around an arbitrary callback for a platform + * document. For idempotent multi-statement writes that don't fit `set`. + */ + public function run(string $collection, string $id, Closure $fn): void + { + $this->execute($this->key($collection, $id), $collection, $fn); + } + + /** + * Block-then-409 lock around an arbitrary callback for a platform document. + * For read-modify-write endpoints where silently dropping a concurrent + * request would lose user data. + */ + public function runOrFail(string $collection, string $id, Closure $fn): mixed + { + return $this->execute($this->key($collection, $id), $collection, $fn, ttl: self::FAIL_TTL_SECONDS, orFail: true); + } + + /** + * Generic lock primitive with full control over key, TTL, contention + * behavior, and wait timeout. Escape hatch for non-platform keys + * (cache, queue, edge) and for unusual TTL/timeout requirements. + * + * Caller may pass `target` for telemetry; otherwise it's extracted by + * position from the key (best-effort for keys following the standard + * `lock:platform:{project}:{target}:...` shape). + */ + public function withKey( + string $key, + Closure $fn, + int $ttl = self::SKIP_TTL_SECONDS, + bool $orFail = false, + float $waitTimeout = self::FAIL_WAIT_SECONDS, + ?string $target = null, + ): mixed { + return $this->execute( + $key, + $target ?? self::inferTargetFromKey($key), + $fn, + ttl: $ttl, + orFail: $orFail, + waitTimeout: $waitTimeout, + ); + } + + private function execute( + string $key, + string $target, + Closure $fn, + int $ttl = self::SKIP_TTL_SECONDS, + bool $orFail = false, + float $waitTimeout = self::FAIL_WAIT_SECONDS, + ): mixed { + if (! $this->enabled) { + return $fn(); + } + + $labels = ['target' => $target, 'project' => $this->projectInternalId]; + $lockCallbackStarted = false; + + try { + return ($this->withLock)($key, $ttl, function (UtopiaLock $lock) use ($key, $target, $fn, $orFail, $waitTimeout, $labels, &$lockCallbackStarted) { + $lockCallbackStarted = true; + + try { + $acquired = $orFail ? $lock->acquire($waitTimeout) : $lock->tryAcquire(); + } catch (\RedisException $e) { + $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); + $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); + + return $fn(); + } + + if (! $acquired) { + if ($orFail) { + $this->attempts->add(1, ['outcome' => self::OUTCOME_CONTENDED, ...$labels]); + // No custom message: the lock key embeds collection and document ID. + throw new Exception(Exception::GENERAL_RESOURCE_LOCKED); + } + $this->attempts->add(1, ['outcome' => self::OUTCOME_SKIPPED, ...$labels]); + + return; + } + + $this->attempts->add(1, ['outcome' => self::OUTCOME_ACQUIRED, ...$labels]); + try { + return $fn(); + } finally { + try { + $lock->release(); + } catch (Throwable $e) { + $this->attempts->add(1, ['outcome' => self::OUTCOME_RELEASE_ERROR, ...$labels]); + $this->reportError(self::OUTCOME_RELEASE_ERROR, $key, $target, $e); + } + } + }); + } catch (\RedisException $e) { + if ($lockCallbackStarted) { + throw $e; + } + + $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); + $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); + + return $fn(); + } + } + + /** + * Best-effort target extraction for telemetry. Assumes the standard + * `lock:platform:{project}:{target}:...` shape. For non-platform keys + * passed via withKey(), callers should pass `target` explicitly. + */ + private static function inferTargetFromKey(string $key): string + { + $parts = explode(':', $key, 5); + + return $parts[3] ?? 'unknown'; + } + + /** + * Shared platform lock key builder. Exposed so higher-level decorators can + * keep the platform key shape centralized while adding narrower scopes. + */ + public function key(string $collection, string $id, ?string $attribute = null): string + { + $key = "lock:platform:{$this->projectInternalId}:{$collection}:{$id}"; + + return $attribute === null ? $key : "{$key}:{$attribute}"; + } + + /** + * Rate-limited to one push per REPORT_RATE_LIMIT_SECONDS per (action, target) + * so a sustained backend outage doesn't flood Sentry across the pod fleet. + */ + private function reportError(string $action, string $key, string $target, Throwable $e): void + { + Console::warning("Lock {$action} for {$key}: {$e->getMessage()}"); + + if ($this->logger === null) { + return; + } + + $bucket = $action.':'.$target; + $now = time(); + if ((self::$lastReportAt[$bucket] ?? 0) + self::REPORT_RATE_LIMIT_SECONDS > $now) { + return; + } + self::$lastReportAt[$bucket] = $now; + + $this->log->setNamespace('http'); + $this->log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); + $this->log->setVersion(APP_VERSION_STABLE); + $this->log->setType(Log::TYPE_WARNING); + $this->log->setMessage('Distributed lock '.$action.': '.$e->getMessage()); + $this->log->setAction("lock.{$action}"); + $this->log->setEnvironment(System::getEnv('_APP_ENV', 'development') === 'production' + ? Log::ENVIRONMENT_PRODUCTION + : Log::ENVIRONMENT_STAGING); + $this->log->addTag('lock.target', $target); + $this->log->addTag('lock.project', $this->projectInternalId); + // Strip trailing document ID to keep aggregator cardinality bounded. + $this->log->addTag('lock.key_pattern', preg_replace('/:[^:]+$/', ':*', $key)); + $this->log->addTag('code', $e->getCode()); + $this->log->addExtra('file', $e->getFile()); + $this->log->addExtra('line', $e->getLine()); + $this->log->addExtra('trace', $e->getTraceAsString()); + + try { + $this->logger->addLog($this->log); + } catch (Throwable) { + } + } +} diff --git a/src/Appwrite/Locking/PlatformLock.php b/src/Appwrite/Locking/PlatformLock.php new file mode 100644 index 0000000000..9d5c8f7844 --- /dev/null +++ b/src/Appwrite/Locking/PlatformLock.php @@ -0,0 +1,42 @@ +lock->withKey( + $this->lock->key($collection, $id, $attribute), + function () use ($collection, $id, $attribute, $value): void { + $this->authorization->skip(fn () => $this->dbForPlatform->updateDocument( + $collection, + $id, + new Document([$attribute => $value]) + )); + }, + target: $collection, + ); + } +} diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php new file mode 100644 index 0000000000..1bc408e324 --- /dev/null +++ b/tests/unit/Locking/LockTest.php @@ -0,0 +1,297 @@ + + */ + private array $heldLocks = []; + + private Document $project; + + private Log $log; + + /** + * Project sequence used for every test; lock keys are scoped under it + * so cleanup is bounded. + */ + private const PROJECT_SEQUENCE = '42'; + + private const KEY_PREFIX = 'lock:platform:'.self::PROJECT_SEQUENCE.':'; + + protected function setUp(): void + { + Config::setParam('errors', require __DIR__.'/../../../app/config/errors.php'); + + $this->heldLocks = []; + $this->project = new Document([ + '$id' => 'test-project', + '$sequence' => self::PROJECT_SEQUENCE, + ]); + $this->log = new Log(); + } + + private function makeLock(): Lock + { + return new Lock( + $this->withLock(), + new NoTelemetry(), + $this->log, + null, + $this->project, + ); + } + + private function withLock(): \Closure + { + return fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new MemoryLock($key, $this->heldLocks)); + } + + public function test_run_uses_per_document_key_and_invokes_callback(): void + { + $called = false; + $lock = $this->makeLock(); + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertTrue($called); + $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); + } + + public function test_run_skips_on_contention(): void + { + $key = self::KEY_PREFIX.'keys:k1'; + $this->heldLocks[$key] = true; + + $called = false; + $lock = $this->makeLock(); + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertFalse($called); + $this->assertArrayHasKey($key, $this->heldLocks); + } + + public function test_backend_acquire_error_runs_callback_unlocked(): void + { + $lock = new Lock( + fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new FailingLock(new \RedisException('redis unavailable'))), + new NoTelemetry(), + $this->log, + null, + $this->project, + ); + + $called = false; + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertTrue($called); + } + + public function test_pool_checkout_error_runs_callback_unlocked(): void + { + $lock = new Lock( + fn (string $key, int $ttl, \Closure $callback): mixed => throw new \RedisException('redis unavailable'), + new NoTelemetry(), + $this->log, + null, + $this->project, + ); + + $called = false; + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertTrue($called); + } + + public function test_run_or_fail_throws_on_contention(): void + { + $key = self::KEY_PREFIX.'projects:p1'; + $this->heldLocks[$key] = true; + + $lock = $this->makeLock(); + + $this->expectException(Exception::class); + try { + $lock->runOrFail('projects', 'p1', fn () => 'never-runs'); + } catch (Exception $e) { + $this->assertSame(Exception::GENERAL_RESOURCE_LOCKED, $e->getType()); + throw $e; + } + } + + public function test_run_or_fail_returns_callback_value_when_uncontended(): void + { + $lock = $this->makeLock(); + $result = $lock->runOrFail('projects', 'p1', fn () => 'ok'); + + $this->assertSame('ok', $result); + $this->assertArrayNotHasKey(self::KEY_PREFIX.'projects:p1', $this->heldLocks); + } + + public function test_with_key_uses_raw_key(): void + { + $custom = 'lock:test:custom-key'; + $called = false; + $lock = $this->makeLock(); + $lock->withKey($custom, function () use (&$called) { + $called = true; + }); + + $this->assertTrue($called); + $this->assertArrayNotHasKey($custom, $this->heldLocks); + } + + public function test_with_key_or_fail_flag_throws_on_contention(): void + { + $custom = 'lock:test:contended'; + $this->heldLocks[$custom] = true; + + $lock = $this->makeLock(); + $this->expectException(Exception::class); + $lock->withKey($custom, fn () => null, ttl: 5, orFail: true, waitTimeout: 0.1); + } + + public function test_disabled_mode_runs_callback_unlocked(): void + { + $previous = \getenv('_APP_LOCKING_ENABLED'); + \putenv('_APP_LOCKING_ENABLED=disabled'); + try { + $key = self::KEY_PREFIX.'keys:k1'; + $this->heldLocks[$key] = true; + + $called = false; + $lock = $this->makeLock(); + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertTrue($called); + $this->assertArrayHasKey($key, $this->heldLocks); + } finally { + $previous === false ? \putenv('_APP_LOCKING_ENABLED') : \putenv('_APP_LOCKING_ENABLED='.$previous); + } + } + + public function test_project_without_sequence_falls_back_to_unknown(): void + { + $emptyProject = new Document(); + $lock = new Lock( + $this->withLock(), + new NoTelemetry(), + $this->log, + null, + $emptyProject, + ); + + $key = 'lock:platform:unknown:keys:k1'; + $this->heldLocks[$key] = true; + + $called = false; + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + $this->assertFalse($called, 'Lock without project sequence should hash to the unknown bucket'); + + unset($this->heldLocks[$key]); + } +} + +final class MemoryLock implements UtopiaLock +{ + private bool $acquired = false; + + /** + * @param array $heldLocks + */ + public function __construct( + private readonly string $key, + private array &$heldLocks, + ) { + } + + public function acquire(float $timeout = 0.0): bool + { + return $this->tryAcquire(); + } + + public function tryAcquire(): bool + { + if (isset($this->heldLocks[$this->key])) { + return false; + } + + $this->heldLocks[$this->key] = true; + $this->acquired = true; + + return true; + } + + public function release(): void + { + if (! $this->acquired) { + return; + } + + unset($this->heldLocks[$this->key]); + $this->acquired = false; + } + + public function withLock(callable $callback, float $timeout = 0.0): mixed + { + if (! $this->acquire($timeout)) { + return null; + } + + try { + return $callback(); + } finally { + $this->release(); + } + } +} + +final class FailingLock implements UtopiaLock +{ + public function __construct(private readonly \RedisException $error) + { + } + + public function acquire(float $timeout = 0.0): bool + { + throw $this->error; + } + + public function tryAcquire(): bool + { + throw $this->error; + } + + public function release(): void + { + } + + public function withLock(callable $callback, float $timeout = 0.0): mixed + { + throw $this->error; + } +} diff --git a/tests/unit/Locking/PlatformLockTest.php b/tests/unit/Locking/PlatformLockTest.php new file mode 100644 index 0000000000..1c66ab40c0 --- /dev/null +++ b/tests/unit/Locking/PlatformLockTest.php @@ -0,0 +1,164 @@ + + */ + private array $heldLocks = []; + + private Document $project; + + private Authorization $authorization; + + private const PROJECT_SEQUENCE = '42'; + + private const KEY_PREFIX = 'lock:platform:'.self::PROJECT_SEQUENCE.':'; + + protected function setUp(): void + { + Config::setParam('errors', require __DIR__.'/../../../app/config/errors.php'); + + $this->heldLocks = []; + $this->project = new Document([ + '$id' => 'test-project', + '$sequence' => self::PROJECT_SEQUENCE, + ]); + $this->authorization = new Authorization(); + } + + private function makePlatformLock(?Database $db = null): PlatformLock + { + return new PlatformLock( + new Lock( + fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new PlatformMemoryLock($key, $this->heldLocks)), + new NoTelemetry(), + new Log(), + null, + $this->project, + ), + $db ?? $this->createStub(Database::class), + $this->authorization, + ); + } + + public function test_set_uses_per_attribute_key_and_auth_skipped_update(): void + { + $captured = null; + $db = $this->createMock(Database::class); + $db->expects($this->once()) + ->method('updateDocument') + ->with('projects', 'p1', $this->callback(function (Document $doc) use (&$captured) { + $captured = $doc->getArrayCopy(); + + return true; + })) + ->willReturnArgument(2); + + $lock = $this->makePlatformLock($db); + $lock->set('projects', 'p1', 'accessedAt', '2024-06-01 12:00:00'); + + $this->assertSame(['accessedAt' => '2024-06-01 12:00:00'], $captured); + $this->assertArrayNotHasKey(self::KEY_PREFIX.'projects:p1:accessedAt', $this->heldLocks); + } + + public function test_set_skips_on_contention(): void + { + $key = self::KEY_PREFIX.'projects:p1:accessedAt'; + $this->heldLocks[$key] = true; + + $db = $this->createMock(Database::class); + $db->expects($this->never())->method('updateDocument'); + + $lock = $this->makePlatformLock($db); + $lock->set('projects', 'p1', 'accessedAt', '2024-06-01 12:00:00'); + + $this->assertArrayHasKey($key, $this->heldLocks); + } + + public function test_set_different_attributes_do_not_compete(): void + { + $heldKey = self::KEY_PREFIX.'projects:p1:accessedAt'; + $this->heldLocks[$heldKey] = true; + + $db = $this->createMock(Database::class); + $db->expects($this->once()) + ->method('updateDocument') + ->with('projects', 'p1', $this->isInstanceOf(Document::class)) + ->willReturnArgument(2); + + $lock = $this->makePlatformLock($db); + $lock->set('projects', 'p1', 'mcpAccessedAt', '2024-06-01 12:00:00'); + + $this->assertArrayHasKey($heldKey, $this->heldLocks); + } +} + +final class PlatformMemoryLock implements UtopiaLock +{ + private bool $acquired = false; + + /** + * @param array $heldLocks + */ + public function __construct( + private readonly string $key, + private array &$heldLocks, + ) { + } + + public function acquire(float $timeout = 0.0): bool + { + return $this->tryAcquire(); + } + + public function tryAcquire(): bool + { + if (isset($this->heldLocks[$this->key])) { + return false; + } + + $this->heldLocks[$this->key] = true; + $this->acquired = true; + + return true; + } + + public function release(): void + { + if (! $this->acquired) { + return; + } + + unset($this->heldLocks[$this->key]); + $this->acquired = false; + } + + public function withLock(callable $callback, float $timeout = 0.0): mixed + { + if (! $this->acquire($timeout)) { + return null; + } + + try { + return $callback(); + } finally { + $this->release(); + } + } +} From 5d453a53ab9f2a1a63cbdd2603573640b2604c83 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Thu, 11 Jun 2026 12:39:30 +0100 Subject: [PATCH 04/24] Add lock resource wiring test --- .../unit/Locking/RequestLockResourceTest.php | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/unit/Locking/RequestLockResourceTest.php diff --git a/tests/unit/Locking/RequestLockResourceTest.php b/tests/unit/Locking/RequestLockResourceTest.php new file mode 100644 index 0000000000..d48f5b3b8c --- /dev/null +++ b/tests/unit/Locking/RequestLockResourceTest.php @@ -0,0 +1,77 @@ +markTestSkipped('Redis extension is required to exercise the registered lock resource.'); + } + + $pool = new Pool(new Stack(), 'lock', 1, fn () => new \Redis()); + $pools = new RecordingPoolGroup(); + $pools->add($pool); + + $db = $this->createMock(Database::class); + $db->expects($this->once()) + ->method('updateDocument') + ->with('projects', 'p1', $this->callback(fn (Document $document): bool => $document->getArrayCopy() === [ + 'accessedAt' => '2026-06-11 10:00:00', + ])) + ->willReturnArgument(2); + + $context = new Container(); + $registrar = require __DIR__.'/../../../app/init/resources/request.php'; + $registrar($context); + + $context + ->set('pools', fn () => $pools) + ->set('telemetry', fn () => new NoTelemetry()) + ->set('logger', fn () => null) + ->set('project', fn () => new Document([ + '$id' => 'test-project', + '$sequence' => '42', + ])) + ->set('dbForPlatform', fn () => $db); + + $lock = $context->get('lock'); + $platformLock = $context->get('platformLock'); + + $this->assertInstanceOf(Lock::class, $lock); + $this->assertInstanceOf(PlatformLock::class, $platformLock); + + $platformLock->set('projects', 'p1', 'accessedAt', '2026-06-11 10:00:00'); + + $this->assertSame(['lock'], $pools->names); + } +} + +final class RecordingPoolGroup extends Group +{ + /** + * @var list + */ + public array $names = []; + + public function get(string $name): Pool + { + $this->names[] = $name; + + return parent::get($name); + } +} From 5a03bb02d20da366cfe1aba166afdf153b77db78 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Tue, 16 Jun 2026 14:33:36 +0100 Subject: [PATCH 05/24] Align distributed lock behavior --- app/config/variables.php | 18 ++ app/controllers/general.php | 38 ++-- app/controllers/shared/api.php | 24 ++- app/init/registers.php | 15 +- app/init/resources/request.php | 11 +- composer.lock | 17 +- docker-compose.yml | 2 + src/Appwrite/Locking/Lock.php | 87 +++------- src/Appwrite/Locking/PlatformLock.php | 42 ----- tests/unit/Locking/LockTest.php | 107 ++++++------ tests/unit/Locking/PlatformLockTest.php | 164 ------------------ .../unit/Locking/RequestLockResourceTest.php | 32 ++-- 12 files changed, 174 insertions(+), 383 deletions(-) delete mode 100644 src/Appwrite/Locking/PlatformLock.php delete mode 100644 tests/unit/Locking/PlatformLockTest.php diff --git a/app/config/variables.php b/app/config/variables.php index 6fef4ecbe0..64d70f436b 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -487,6 +487,24 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_CONNECTIONS_LOCKS', + 'description' => 'Connection DSN for distributed locks. By default, locks use the Redis server configured by _APP_REDIS_*.', + 'introduction' => '1.9.3', + 'default' => '', + 'required' => false, + 'question' => '', + 'filter' => '' + ], + [ + 'name' => '_APP_POOL_SIZE_LOCK', + 'description' => 'Maximum number of connections in the distributed lock connection pool. By default, set to 2.', + 'introduction' => '1.9.3', + 'default' => '2', + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], ], [ diff --git a/app/controllers/general.php b/app/controllers/general.php index 2af3cb1d6c..2013fcb33d 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -12,7 +12,7 @@ use Appwrite\Event\Message\Delete as DeleteMessage; use Appwrite\Event\Publisher\Certificate; use Appwrite\Event\Publisher\Delete as DeletePublisher; use Appwrite\Extend\Exception as AppwriteException; -use Appwrite\Locking\PlatformLock; +use Appwrite\Locking\Lock; use Appwrite\Network\Cors; use Appwrite\Platform\Appwrite; use Appwrite\SDK\Method; @@ -76,7 +76,7 @@ use Utopia\Validator\Text; Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformLock $platformLock) +function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { $host = $request->getHostname(); if (!empty($previewHostname)) { @@ -145,7 +145,15 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S if (!$project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $platformLock->set('projects', $project->getId(), 'accessedAt', DateTime::now()); + $lock->withKey( + $lock->key('projects', $project->getId(), 'accessedAt'), + fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( + 'projects', + $project->getId(), + new Document(['accessedAt' => DateTime::now()]) + )), + target: 'projects' + ); } /** @@ -839,9 +847,9 @@ Http::init() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('platformLock') + ->inject('lock') ->inject('params') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformLock $platformLock, array $params) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock, array $params) { /* * Appwrite Router */ @@ -849,7 +857,7 @@ Http::init() $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!\in_array($hostname, $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformLock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1152,15 +1160,15 @@ Http::options() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('platformLock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformLock $platformLock) { + ->inject('lock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { /* * Appwrite Router */ $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformLock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1555,14 +1563,14 @@ Http::get('/robots.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('platformLock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformLock $platformLock) { + ->inject('lock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformLock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1590,14 +1598,14 @@ Http::get('/humans.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('platformLock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformLock $platformLock) { + ->inject('lock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformLock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { $utopia->match($request)?->route->label('router', true); } } diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index b6c252227c..1351e63f53 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -17,7 +17,6 @@ use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\EventProcessor; use Appwrite\Locking\Lock; -use Appwrite\Locking\PlatformLock; use Appwrite\Platform\Modules\Storage\Config\CacheControl; use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl; use Appwrite\SDK\Method; @@ -102,8 +101,7 @@ Http::init() ->inject('apiKey') ->inject('authorization') ->inject('lock') - ->inject('platformLock') - ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Lock $lock, PlatformLock $platformLock) { + ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Lock $lock) { /** * Handle user authentication and session validation. @@ -390,7 +388,15 @@ Http::init() if ($project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $platformLock->set('projects', $project->getId(), 'accessedAt', DateTime::now()); + $lock->withKey( + $lock->key('projects', $project->getId(), 'accessedAt'), + fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( + 'projects', + $project->getId(), + new Document(['accessedAt' => DateTime::now()]) + )), + target: 'projects' + ); } } @@ -407,7 +413,15 @@ Http::init() 'accessedAt' => $user->getAttribute('accessedAt') ])); } else { - $platformLock->set('users', $user->getId(), 'accessedAt', $user->getAttribute('accessedAt')); + $lock->withKey( + $lock->key('users', $user->getId(), 'accessedAt'), + fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( + 'users', + $user->getId(), + new Document(['accessedAt' => $user->getAttribute('accessedAt')]) + )), + target: 'users' + ); } } } diff --git a/app/init/registers.php b/app/init/registers.php index e7ede8b025..4043b58485 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -236,9 +236,10 @@ $register->set('pools', function () { ], 'lock' => [ 'type' => 'lock', - 'dsns' => $fallbackForRedis, + 'dsns' => System::getEnv('_APP_CONNECTIONS_LOCKS', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], + 'size' => max(1, (int) System::getEnv('_APP_POOL_SIZE_LOCK', 2)), ], ]; @@ -256,6 +257,7 @@ $register->set('pools', function () { $type = $connection['type']; $multiple = $connection['multiple']; $schemes = $connection['schemes']; + $poolSizeForConnection = $connection['size'] ?? $poolSize; $config = []; $dsns = explode(',', $connection['dsns']); foreach ($dsns as &$dsn) { @@ -339,8 +341,8 @@ $register->set('pools', function () { // PubSub workers hold one long-lived subscribed connection and also need // spare capacity for publishes from the same process. $connectionPoolSize = match ($type) { - 'pubsub' => max(2, $poolSize), - default => $poolSize, + 'pubsub' => max(2, $poolSizeForConnection), + default => $poolSizeForConnection, }; $pool = new Pool($poolAdapter, $name, $connectionPoolSize, function () use ($type, $resource, $dsn) { @@ -390,6 +392,13 @@ $register->set('pools', function () { } }); + if ($type === 'cache' || $type === 'lock') { + $pool + ->setRetryAttempts(1) + ->setReconnectAttempts(1) + ->setSynchronizationTimeout(1); + } + $group->add($pool); } diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 24a4c0e127..30bd301dc0 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -15,7 +15,6 @@ use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\GraphQL\Schema; use Appwrite\Locking\Lock; -use Appwrite\Locking\PlatformLock; use Appwrite\Network\Cors; use Appwrite\Network\Platform; use Appwrite\Network\Validator\Origin; @@ -71,8 +70,8 @@ return function (Container $context): void { $context->set('lock', function (Group $pools, Telemetry $telemetry, Log $log, ?Logger $logger, Document $project): Lock { return new Lock( - fn (string $key, int $ttl, Closure $callback): mixed => $pools->get('lock')->use( - fn (\Redis $redis): mixed => $callback(new DistributedLock($redis, $key, $ttl)) + fn (string $key, int $ttl, Closure $callback, float $timeout): mixed => $pools->get('lock')->use( + fn (\Redis $redis): mixed => (new DistributedLock($redis, $key, $ttl))->withLock($callback, $timeout) ), $telemetry, $log, @@ -81,12 +80,6 @@ return function (Container $context): void { ); }, ['pools', 'telemetry', 'log', 'logger', 'project']); - $context->set('platformLock', fn (Lock $lock, Database $dbForPlatform, Authorization $authorization): PlatformLock => new PlatformLock( - $lock, - $dbForPlatform, - $authorization, - ), ['lock', 'dbForPlatform', 'authorization']); - $context->set('authorization', fn () => new Authorization(), []); $context->set('store', fn (): Store => new Store(), []); diff --git a/composer.lock b/composer.lock index 1e57ede67b..12f10b4702 100644 --- a/composer.lock +++ b/composer.lock @@ -4260,25 +4260,22 @@ }, { "name": "utopia-php/lock", - "version": "0.2.0", + "version": "0.2.1", "source": { "type": "git", "url": "https://github.com/utopia-php/lock.git", - "reference": "49317c9493d8f747e4299aa24c22862aa5f6e106" + "reference": "9ef48d7094f5c68d1736f664f6e9b4b1ec656f21" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/lock/zipball/49317c9493d8f747e4299aa24c22862aa5f6e106", - "reference": "49317c9493d8f747e4299aa24c22862aa5f6e106", + "url": "https://api.github.com/repos/utopia-php/lock/zipball/9ef48d7094f5c68d1736f664f6e9b4b1ec656f21", + "reference": "9ef48d7094f5c68d1736f664f6e9b4b1ec656f21", "shasum": "" }, "require": { "php": ">=8.3" }, "require-dev": { - "laravel/pint": "1.*", - "phpstan/phpstan": "2.*", - "phpunit/phpunit": "11.*", "swoole/ide-helper": "*" }, "suggest": { @@ -4302,12 +4299,12 @@ "email": "team@appwrite.io" } ], - "description": "Mutex, semaphore, file and distributed locks for PHP — one interface, four backends.", + "description": "A simple lock library to coordinate access to shared resources across coroutines, processes and hosts", "support": { "issues": "https://github.com/utopia-php/lock/issues", - "source": "https://github.com/utopia-php/lock/tree/0.2.0" + "source": "https://github.com/utopia-php/lock/tree/0.2.1" }, - "time": "2026-04-24T10:47:56+00:00" + "time": "2026-06-10T15:55:36+00:00" }, { "name": "utopia-php/logger", diff --git a/docker-compose.yml b/docker-compose.yml index 3a954cebd1..7d79031616 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -155,6 +155,8 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_LOCKS + - _APP_POOL_SIZE_LOCK - _APP_DB_ADAPTER - _APP_DB_HOST - _APP_DB_PORT diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php index 7bd7e7db28..df48bcc6f5 100644 --- a/src/Appwrite/Locking/Lock.php +++ b/src/Appwrite/Locking/Lock.php @@ -7,7 +7,7 @@ use Closure; use Throwable; use Utopia\Console; use Utopia\Database\Document; -use Utopia\Lock\Lock as UtopiaLock; +use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\System\System; @@ -15,24 +15,18 @@ use Utopia\Telemetry\Adapter as Telemetry; final class Lock { - private const SKIP_TTL_SECONDS = 5; + private const TTL_SECONDS = 600; - private const FAIL_TTL_SECONDS = 10; - - private const FAIL_WAIT_SECONDS = 3.0; + private const WAIT_SECONDS = 0.0; private const REPORT_RATE_LIMIT_SECONDS = 60; private const OUTCOME_ACQUIRED = 'acquired'; - private const OUTCOME_SKIPPED = 'skipped'; - private const OUTCOME_CONTENDED = 'contended'; private const OUTCOME_BACKEND_ERROR = 'backend_error'; - private const OUTCOME_RELEASE_ERROR = 'release_error'; - private readonly bool $enabled; private readonly mixed $attempts; @@ -43,7 +37,7 @@ final class Lock private static array $lastReportAt = []; /** - * @var Closure(string, int, Closure(UtopiaLock): mixed): mixed + * @var Closure(string, int, Closure(): mixed, float): mixed */ private readonly Closure $withLock; @@ -62,8 +56,7 @@ final class Lock } /** - * Skip-on-contention lock around an arbitrary callback for a platform - * document. For idempotent multi-statement writes that don't fit `set`. + * Distributed lock around an arbitrary callback for a platform document. */ public function run(string $collection, string $id, Closure $fn): void { @@ -71,19 +64,17 @@ final class Lock } /** - * Block-then-409 lock around an arbitrary callback for a platform document. - * For read-modify-write endpoints where silently dropping a concurrent - * request would lose user data. + * Distributed lock around a platform document that returns the callback result. */ public function runOrFail(string $collection, string $id, Closure $fn): mixed { - return $this->execute($this->key($collection, $id), $collection, $fn, ttl: self::FAIL_TTL_SECONDS, orFail: true); + return $this->execute($this->key($collection, $id), $collection, $fn); } /** * Generic lock primitive with full control over key, TTL, contention - * behavior, and wait timeout. Escape hatch for non-platform keys - * (cache, queue, edge) and for unusual TTL/timeout requirements. + * behavior, and wait timeout. Escape hatch for non-platform keys (cache, + * queue, edge) and for unusual TTL/timeout requirements. * * Caller may pass `target` for telemetry; otherwise it's extracted by * position from the key (best-effort for keys following the standard @@ -92,9 +83,8 @@ final class Lock public function withKey( string $key, Closure $fn, - int $ttl = self::SKIP_TTL_SECONDS, - bool $orFail = false, - float $waitTimeout = self::FAIL_WAIT_SECONDS, + int $ttl = self::TTL_SECONDS, + float $waitTimeout = self::WAIT_SECONDS, ?string $target = null, ): mixed { return $this->execute( @@ -102,7 +92,6 @@ final class Lock $target ?? self::inferTargetFromKey($key), $fn, ttl: $ttl, - orFail: $orFail, waitTimeout: $waitTimeout, ); } @@ -111,62 +100,30 @@ final class Lock string $key, string $target, Closure $fn, - int $ttl = self::SKIP_TTL_SECONDS, - bool $orFail = false, - float $waitTimeout = self::FAIL_WAIT_SECONDS, + int $ttl = self::TTL_SECONDS, + float $waitTimeout = self::WAIT_SECONDS, ): mixed { if (! $this->enabled) { return $fn(); } $labels = ['target' => $target, 'project' => $this->projectInternalId]; - $lockCallbackStarted = false; try { - return ($this->withLock)($key, $ttl, function (UtopiaLock $lock) use ($key, $target, $fn, $orFail, $waitTimeout, $labels, &$lockCallbackStarted) { - $lockCallbackStarted = true; - - try { - $acquired = $orFail ? $lock->acquire($waitTimeout) : $lock->tryAcquire(); - } catch (\RedisException $e) { - $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); - $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); - - return $fn(); - } - - if (! $acquired) { - if ($orFail) { - $this->attempts->add(1, ['outcome' => self::OUTCOME_CONTENDED, ...$labels]); - // No custom message: the lock key embeds collection and document ID. - throw new Exception(Exception::GENERAL_RESOURCE_LOCKED); - } - $this->attempts->add(1, ['outcome' => self::OUTCOME_SKIPPED, ...$labels]); - - return; - } - + return ($this->withLock)($key, $ttl, function () use ($fn, $labels) { $this->attempts->add(1, ['outcome' => self::OUTCOME_ACQUIRED, ...$labels]); - try { - return $fn(); - } finally { - try { - $lock->release(); - } catch (Throwable $e) { - $this->attempts->add(1, ['outcome' => self::OUTCOME_RELEASE_ERROR, ...$labels]); - $this->reportError(self::OUTCOME_RELEASE_ERROR, $key, $target, $e); - } - } - }); - } catch (\RedisException $e) { - if ($lockCallbackStarted) { - throw $e; - } + return $fn(); + }, $waitTimeout); + } catch (LockContention) { + $this->attempts->add(1, ['outcome' => self::OUTCOME_CONTENDED, ...$labels]); + // No custom message: the lock key embeds collection and document ID. + throw new Exception(Exception::GENERAL_RESOURCE_LOCKED); + } catch (\RedisException $e) { $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); - return $fn(); + throw $e; } } diff --git a/src/Appwrite/Locking/PlatformLock.php b/src/Appwrite/Locking/PlatformLock.php deleted file mode 100644 index 9d5c8f7844..0000000000 --- a/src/Appwrite/Locking/PlatformLock.php +++ /dev/null @@ -1,42 +0,0 @@ -lock->withKey( - $this->lock->key($collection, $id, $attribute), - function () use ($collection, $id, $attribute, $value): void { - $this->authorization->skip(fn () => $this->dbForPlatform->updateDocument( - $collection, - $id, - new Document([$attribute => $value]) - )); - }, - target: $collection, - ); - } -} diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php index 1bc408e324..ca9b6ba999 100644 --- a/tests/unit/Locking/LockTest.php +++ b/tests/unit/Locking/LockTest.php @@ -9,6 +9,7 @@ use Appwrite\Locking\Lock; use PHPUnit\Framework\TestCase; use Utopia\Config\Config; use Utopia\Database\Document; +use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Lock\Lock as UtopiaLock; use Utopia\Logger\Log; use Utopia\Telemetry\Adapter\None as NoTelemetry; @@ -57,7 +58,7 @@ final class LockTest extends TestCase private function withLock(): \Closure { - return fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new MemoryLock($key, $this->heldLocks)); + return fn (string $key, int $ttl, \Closure $callback, float $timeout): mixed => (new MemoryLock($key, $this->heldLocks))->withLock($callback, $timeout); } public function test_run_uses_per_document_key_and_invokes_callback(): void @@ -72,25 +73,34 @@ final class LockTest extends TestCase $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); } - public function test_run_skips_on_contention(): void + public function test_run_throws_on_contention(): void { $key = self::KEY_PREFIX.'keys:k1'; $this->heldLocks[$key] = true; $called = false; $lock = $this->makeLock(); - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - $this->assertFalse($called); - $this->assertArrayHasKey($key, $this->heldLocks); + try { + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + $this->fail('Expected lock contention to throw.'); + } catch (Exception $e) { + $this->assertSame(Exception::GENERAL_RESOURCE_LOCKED, $e->getType()); + $this->assertFalse($called); + $this->assertArrayHasKey($key, $this->heldLocks); + } } - public function test_backend_acquire_error_runs_callback_unlocked(): void + public function test_backend_error_does_not_run_callback_unlocked(): void { + if (! \class_exists(\RedisException::class)) { + $this->markTestSkipped('Redis extension is required to simulate RedisException.'); + } + $lock = new Lock( - fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new FailingLock(new \RedisException('redis unavailable'))), + fn (string $key, int $ttl, \Closure $callback, float $timeout): mixed => throw new \RedisException('redis unavailable'), new NoTelemetry(), $this->log, null, @@ -98,17 +108,24 @@ final class LockTest extends TestCase ); $called = false; - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - - $this->assertTrue($called); + try { + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + $this->fail('Expected Redis backend errors to bubble.'); + } catch (\RedisException) { + $this->assertFalse($called); + } } - public function test_pool_checkout_error_runs_callback_unlocked(): void + public function test_pool_checkout_error_does_not_run_callback_unlocked(): void { + if (! \class_exists(\RedisException::class)) { + $this->markTestSkipped('Redis extension is required to simulate RedisException.'); + } + $lock = new Lock( - fn (string $key, int $ttl, \Closure $callback): mixed => throw new \RedisException('redis unavailable'), + fn (string $key, int $ttl, \Closure $callback, float $timeout): mixed => throw new \RedisException('redis unavailable'), new NoTelemetry(), $this->log, null, @@ -116,11 +133,14 @@ final class LockTest extends TestCase ); $called = false; - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - - $this->assertTrue($called); + try { + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + $this->fail('Expected Redis backend errors to bubble.'); + } catch (\RedisException) { + $this->assertFalse($called); + } } public function test_run_or_fail_throws_on_contention(): void @@ -161,14 +181,14 @@ final class LockTest extends TestCase $this->assertArrayNotHasKey($custom, $this->heldLocks); } - public function test_with_key_or_fail_flag_throws_on_contention(): void + public function test_with_key_throws_on_contention(): void { $custom = 'lock:test:contended'; $this->heldLocks[$custom] = true; $lock = $this->makeLock(); $this->expectException(Exception::class); - $lock->withKey($custom, fn () => null, ttl: 5, orFail: true, waitTimeout: 0.1); + $lock->withKey($custom, fn () => null, ttl: 5, waitTimeout: 0.1); } public function test_disabled_mode_runs_callback_unlocked(): void @@ -207,10 +227,15 @@ final class LockTest extends TestCase $this->heldLocks[$key] = true; $called = false; - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - $this->assertFalse($called, 'Lock without project sequence should hash to the unknown bucket'); + try { + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + $this->fail('Expected unknown-bucket contention to throw.'); + } catch (Exception $e) { + $this->assertSame(Exception::GENERAL_RESOURCE_LOCKED, $e->getType()); + $this->assertFalse($called, 'Lock without project sequence should hash to the unknown bucket'); + } unset($this->heldLocks[$key]); } @@ -259,7 +284,7 @@ final class MemoryLock implements UtopiaLock public function withLock(callable $callback, float $timeout = 0.0): mixed { if (! $this->acquire($timeout)) { - return null; + throw new LockContention("Failed to acquire distributed lock: {$this->key}"); } try { @@ -269,29 +294,3 @@ final class MemoryLock implements UtopiaLock } } } - -final class FailingLock implements UtopiaLock -{ - public function __construct(private readonly \RedisException $error) - { - } - - public function acquire(float $timeout = 0.0): bool - { - throw $this->error; - } - - public function tryAcquire(): bool - { - throw $this->error; - } - - public function release(): void - { - } - - public function withLock(callable $callback, float $timeout = 0.0): mixed - { - throw $this->error; - } -} diff --git a/tests/unit/Locking/PlatformLockTest.php b/tests/unit/Locking/PlatformLockTest.php deleted file mode 100644 index 1c66ab40c0..0000000000 --- a/tests/unit/Locking/PlatformLockTest.php +++ /dev/null @@ -1,164 +0,0 @@ - - */ - private array $heldLocks = []; - - private Document $project; - - private Authorization $authorization; - - private const PROJECT_SEQUENCE = '42'; - - private const KEY_PREFIX = 'lock:platform:'.self::PROJECT_SEQUENCE.':'; - - protected function setUp(): void - { - Config::setParam('errors', require __DIR__.'/../../../app/config/errors.php'); - - $this->heldLocks = []; - $this->project = new Document([ - '$id' => 'test-project', - '$sequence' => self::PROJECT_SEQUENCE, - ]); - $this->authorization = new Authorization(); - } - - private function makePlatformLock(?Database $db = null): PlatformLock - { - return new PlatformLock( - new Lock( - fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new PlatformMemoryLock($key, $this->heldLocks)), - new NoTelemetry(), - new Log(), - null, - $this->project, - ), - $db ?? $this->createStub(Database::class), - $this->authorization, - ); - } - - public function test_set_uses_per_attribute_key_and_auth_skipped_update(): void - { - $captured = null; - $db = $this->createMock(Database::class); - $db->expects($this->once()) - ->method('updateDocument') - ->with('projects', 'p1', $this->callback(function (Document $doc) use (&$captured) { - $captured = $doc->getArrayCopy(); - - return true; - })) - ->willReturnArgument(2); - - $lock = $this->makePlatformLock($db); - $lock->set('projects', 'p1', 'accessedAt', '2024-06-01 12:00:00'); - - $this->assertSame(['accessedAt' => '2024-06-01 12:00:00'], $captured); - $this->assertArrayNotHasKey(self::KEY_PREFIX.'projects:p1:accessedAt', $this->heldLocks); - } - - public function test_set_skips_on_contention(): void - { - $key = self::KEY_PREFIX.'projects:p1:accessedAt'; - $this->heldLocks[$key] = true; - - $db = $this->createMock(Database::class); - $db->expects($this->never())->method('updateDocument'); - - $lock = $this->makePlatformLock($db); - $lock->set('projects', 'p1', 'accessedAt', '2024-06-01 12:00:00'); - - $this->assertArrayHasKey($key, $this->heldLocks); - } - - public function test_set_different_attributes_do_not_compete(): void - { - $heldKey = self::KEY_PREFIX.'projects:p1:accessedAt'; - $this->heldLocks[$heldKey] = true; - - $db = $this->createMock(Database::class); - $db->expects($this->once()) - ->method('updateDocument') - ->with('projects', 'p1', $this->isInstanceOf(Document::class)) - ->willReturnArgument(2); - - $lock = $this->makePlatformLock($db); - $lock->set('projects', 'p1', 'mcpAccessedAt', '2024-06-01 12:00:00'); - - $this->assertArrayHasKey($heldKey, $this->heldLocks); - } -} - -final class PlatformMemoryLock implements UtopiaLock -{ - private bool $acquired = false; - - /** - * @param array $heldLocks - */ - public function __construct( - private readonly string $key, - private array &$heldLocks, - ) { - } - - public function acquire(float $timeout = 0.0): bool - { - return $this->tryAcquire(); - } - - public function tryAcquire(): bool - { - if (isset($this->heldLocks[$this->key])) { - return false; - } - - $this->heldLocks[$this->key] = true; - $this->acquired = true; - - return true; - } - - public function release(): void - { - if (! $this->acquired) { - return; - } - - unset($this->heldLocks[$this->key]); - $this->acquired = false; - } - - public function withLock(callable $callback, float $timeout = 0.0): mixed - { - if (! $this->acquire($timeout)) { - return null; - } - - try { - return $callback(); - } finally { - $this->release(); - } - } -} diff --git a/tests/unit/Locking/RequestLockResourceTest.php b/tests/unit/Locking/RequestLockResourceTest.php index d48f5b3b8c..56f8d7e29c 100644 --- a/tests/unit/Locking/RequestLockResourceTest.php +++ b/tests/unit/Locking/RequestLockResourceTest.php @@ -5,9 +5,7 @@ declare(strict_types=1); namespace Tests\Unit\Locking; use Appwrite\Locking\Lock; -use Appwrite\Locking\PlatformLock; use PHPUnit\Framework\TestCase; -use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\DI\Container; use Utopia\Pools\Adapter\Stack; @@ -17,24 +15,20 @@ use Utopia\Telemetry\Adapter\None as NoTelemetry; final class RequestLockResourceTest extends TestCase { - public function test_request_resources_wire_platform_lock_to_lock_pool(): void + public function test_request_resources_wire_lock_to_lock_pool(): void { if (! \class_exists(\Redis::class)) { $this->markTestSkipped('Redis extension is required to exercise the registered lock resource.'); } - $pool = new Pool(new Stack(), 'lock', 1, fn () => new \Redis()); + $redis = $this->createMock(\Redis::class); + $redis->method('set')->willReturn(true); + $redis->method('eval')->willReturn(1); + + $pool = new Pool(new Stack(), 'lock', 1, fn () => $redis); $pools = new RecordingPoolGroup(); $pools->add($pool); - $db = $this->createMock(Database::class); - $db->expects($this->once()) - ->method('updateDocument') - ->with('projects', 'p1', $this->callback(fn (Document $document): bool => $document->getArrayCopy() === [ - 'accessedAt' => '2026-06-11 10:00:00', - ])) - ->willReturnArgument(2); - $context = new Container(); $registrar = require __DIR__.'/../../../app/init/resources/request.php'; $registrar($context); @@ -47,16 +41,22 @@ final class RequestLockResourceTest extends TestCase '$id' => 'test-project', '$sequence' => '42', ])) - ->set('dbForPlatform', fn () => $db); + ->set('dbForPlatform', fn () => null); $lock = $context->get('lock'); - $platformLock = $context->get('platformLock'); $this->assertInstanceOf(Lock::class, $lock); - $this->assertInstanceOf(PlatformLock::class, $platformLock); - $platformLock->set('projects', 'p1', 'accessedAt', '2026-06-11 10:00:00'); + $ran = false; + $lock->withKey( + $lock->key('projects', 'p1', 'accessedAt'), + function () use (&$ran): void { + $ran = true; + }, + target: 'projects' + ); + $this->assertTrue($ran); $this->assertSame(['lock'], $pools->names); } } From 518cd68e6fc961ea62d39954424ce07a6fa32d64 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Tue, 16 Jun 2026 16:12:52 +0100 Subject: [PATCH 06/24] Align lock timing with cloud --- src/Appwrite/Locking/Lock.php | 7 +++++-- tests/unit/Locking/LockTest.php | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php index df48bcc6f5..dc04c532ef 100644 --- a/src/Appwrite/Locking/Lock.php +++ b/src/Appwrite/Locking/Lock.php @@ -15,9 +15,12 @@ use Utopia\Telemetry\Adapter as Telemetry; final class Lock { - private const TTL_SECONDS = 600; + /** + * Keep defaults aligned with Cloud's distributed database worker lock. + */ + private const TTL_SECONDS = 1800; - private const WAIT_SECONDS = 0.0; + private const WAIT_SECONDS = 1500.0; private const REPORT_RATE_LIMIT_SECONDS = 60; diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php index ca9b6ba999..1e5801e7fa 100644 --- a/tests/unit/Locking/LockTest.php +++ b/tests/unit/Locking/LockTest.php @@ -73,6 +73,30 @@ final class LockTest extends TestCase $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); } + public function test_run_uses_cloud_lock_ttl_and_acquire_timeout(): void + { + $ttl = null; + $timeout = null; + + $lock = new Lock( + function (string $key, int $lockTtl, \Closure $callback, float $lockTimeout) use (&$ttl, &$timeout): mixed { + $ttl = $lockTtl; + $timeout = $lockTimeout; + + return (new MemoryLock($key, $this->heldLocks))->withLock($callback, $lockTimeout); + }, + new NoTelemetry(), + $this->log, + null, + $this->project, + ); + + $lock->run('keys', 'k1', fn () => null); + + $this->assertSame(1800, $ttl); + $this->assertSame(1500.0, $timeout); + } + public function test_run_throws_on_contention(): void { $key = self::KEY_PREFIX.'keys:k1'; From 81beba0ee256e5b1f6f69566bad3b36fd651313c Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Tue, 16 Jun 2026 16:38:13 +0100 Subject: [PATCH 07/24] Align lock pool default --- app/config/variables.php | 4 ++-- app/init/registers.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index 64d70f436b..ff068fa285 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -498,9 +498,9 @@ return [ ], [ 'name' => '_APP_POOL_SIZE_LOCK', - 'description' => 'Maximum number of connections in the distributed lock connection pool. By default, set to 2.', + 'description' => 'Maximum number of connections in the distributed lock connection pool. By default, set to 10.', 'introduction' => '1.9.3', - 'default' => '2', + 'default' => '10', 'required' => false, 'question' => '', 'filter' => '' diff --git a/app/init/registers.php b/app/init/registers.php index 4043b58485..db736eacc0 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -239,7 +239,7 @@ $register->set('pools', function () { 'dsns' => System::getEnv('_APP_CONNECTIONS_LOCKS', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], - 'size' => max(1, (int) System::getEnv('_APP_POOL_SIZE_LOCK', 2)), + 'size' => max(1, (int) System::getEnv('_APP_POOL_SIZE_LOCK', 10)), ], ]; From 9ce916eb6a9b4f329ea1063a18bbf788c1456adb Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 17 Jun 2026 00:01:20 +0100 Subject: [PATCH 08/24] Refine platform metadata locks --- app/controllers/general.php | 38 +-- app/controllers/shared/api.php | 24 +- app/init/resources/request.php | 11 +- src/Appwrite/Locking/Lock.php | 159 +++++++++-- src/Appwrite/Locking/PlatformDBLock.php | 40 +++ tests/unit/Locking/LockTest.php | 348 ++++++++++++++++++++---- 6 files changed, 503 insertions(+), 117 deletions(-) create mode 100644 src/Appwrite/Locking/PlatformDBLock.php diff --git a/app/controllers/general.php b/app/controllers/general.php index 814c6f38c6..6642e219d1 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -12,7 +12,7 @@ use Appwrite\Event\Message\Delete as DeleteMessage; use Appwrite\Event\Publisher\Certificate; use Appwrite\Event\Publisher\Delete as DeletePublisher; use Appwrite\Extend\Exception as AppwriteException; -use Appwrite\Locking\Lock; +use Appwrite\Locking\PlatformDBLock; use Appwrite\Network\Cors; use Appwrite\Platform\Appwrite; use Appwrite\SDK\AuthType; @@ -77,7 +77,7 @@ use Utopia\Validator\Text; Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) +function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { $host = $request->getHostname(); if (!empty($previewHostname)) { @@ -146,15 +146,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S if (!$project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $lock->withKey( - $lock->key('projects', $project->getId(), 'accessedAt'), - fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( - 'projects', - $project->getId(), - new Document(['accessedAt' => DateTime::now()]) - )), - target: 'projects' - ); + $platformDBLock->tryUpdateAttribute('projects', $project->getId(), 'accessedAt', DateTime::now(), $project); } /** @@ -848,9 +840,9 @@ Http::init() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('lock') + ->inject('platformDBLock') ->inject('params') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock, array $params) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock, array $params) { /* * Appwrite Router */ @@ -858,7 +850,7 @@ Http::init() $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!\in_array($hostname, $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1161,15 +1153,15 @@ Http::options() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('lock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { + ->inject('platformDBLock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { /* * Appwrite Router */ $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1564,14 +1556,14 @@ Http::get('/robots.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('lock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { + ->inject('platformDBLock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1599,14 +1591,14 @@ Http::get('/humans.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('lock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { + ->inject('platformDBLock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { $utopia->match($request)?->route->label('router', true); } } diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 1351e63f53..7ee6610292 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -17,6 +17,7 @@ use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\EventProcessor; use Appwrite\Locking\Lock; +use Appwrite\Locking\PlatformDBLock; use Appwrite\Platform\Modules\Storage\Config\CacheControl; use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl; use Appwrite\SDK\Method; @@ -101,7 +102,8 @@ Http::init() ->inject('apiKey') ->inject('authorization') ->inject('lock') - ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Lock $lock) { + ->inject('platformDBLock') + ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Lock $lock, PlatformDBLock $platformDBLock) { /** * Handle user authentication and session validation. @@ -388,15 +390,7 @@ Http::init() if ($project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $lock->withKey( - $lock->key('projects', $project->getId(), 'accessedAt'), - fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( - 'projects', - $project->getId(), - new Document(['accessedAt' => DateTime::now()]) - )), - target: 'projects' - ); + $platformDBLock->tryUpdateAttribute('projects', $project->getId(), 'accessedAt', DateTime::now()); } } @@ -413,15 +407,7 @@ Http::init() 'accessedAt' => $user->getAttribute('accessedAt') ])); } else { - $lock->withKey( - $lock->key('users', $user->getId(), 'accessedAt'), - fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( - 'users', - $user->getId(), - new Document(['accessedAt' => $user->getAttribute('accessedAt')]) - )), - target: 'users' - ); + $platformDBLock->tryUpdateAttribute('users', $user->getId(), 'accessedAt', $user->getAttribute('accessedAt')); } } } diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 30bd301dc0..fade92113c 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -15,6 +15,7 @@ use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\GraphQL\Schema; use Appwrite\Locking\Lock; +use Appwrite\Locking\PlatformDBLock; use Appwrite\Network\Cors; use Appwrite\Network\Platform; use Appwrite\Network\Validator\Origin; @@ -70,8 +71,8 @@ return function (Container $context): void { $context->set('lock', function (Group $pools, Telemetry $telemetry, Log $log, ?Logger $logger, Document $project): Lock { return new Lock( - fn (string $key, int $ttl, Closure $callback, float $timeout): mixed => $pools->get('lock')->use( - fn (\Redis $redis): mixed => (new DistributedLock($redis, $key, $ttl))->withLock($callback, $timeout) + fn (string $key, int $ttl, Closure $callback): mixed => $pools->get('lock')->use( + fn (\Redis $redis): mixed => $callback(new DistributedLock($redis, $key, $ttl)) ), $telemetry, $log, @@ -80,6 +81,12 @@ return function (Container $context): void { ); }, ['pools', 'telemetry', 'log', 'logger', 'project']); + $context->set('platformDBLock', fn (Lock $lock, Database $dbForPlatform, Authorization $authorization): PlatformDBLock => new PlatformDBLock( + $lock, + $dbForPlatform, + $authorization + ), ['lock', 'dbForPlatform', 'authorization']); + $context->set('authorization', fn () => new Authorization(), []); $context->set('store', fn (): Store => new Store(), []); diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php index dc04c532ef..8a2061425b 100644 --- a/src/Appwrite/Locking/Lock.php +++ b/src/Appwrite/Locking/Lock.php @@ -7,7 +7,7 @@ use Closure; use Throwable; use Utopia\Console; use Utopia\Database\Document; -use Utopia\Lock\Exception\Contention as LockContention; +use Utopia\Lock\Lock as UtopiaLock; use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\System\System; @@ -15,21 +15,26 @@ use Utopia\Telemetry\Adapter as Telemetry; final class Lock { - /** - * Keep defaults aligned with Cloud's distributed database worker lock. - */ - private const TTL_SECONDS = 1800; + private const FAIL_TTL_SECONDS = 10; - private const WAIT_SECONDS = 1500.0; + private const FAIL_WAIT_SECONDS = 3.0; + + private const SKIP_TTL_SECONDS = 5; + + private const SKIP_WAIT_SECONDS = 0.0; private const REPORT_RATE_LIMIT_SECONDS = 60; private const OUTCOME_ACQUIRED = 'acquired'; + private const OUTCOME_SKIPPED = 'skipped'; + private const OUTCOME_CONTENDED = 'contended'; private const OUTCOME_BACKEND_ERROR = 'backend_error'; + private const OUTCOME_RELEASE_ERROR = 'release_error'; + private readonly bool $enabled; private readonly mixed $attempts; @@ -40,18 +45,18 @@ final class Lock private static array $lastReportAt = []; /** - * @var Closure(string, int, Closure(): mixed, float): mixed + * @var Closure(string, int, Closure(UtopiaLock): mixed): mixed */ - private readonly Closure $withLock; + private readonly Closure $useLock; public function __construct( - Closure $withLock, + Closure $useLock, Telemetry $telemetry, private readonly Log $log, private readonly ?Logger $logger, Document $project, ) { - $this->withLock = $withLock; + $this->useLock = $useLock; $this->enabled = System::getEnv('_APP_LOCKING_ENABLED', 'enabled') !== 'disabled'; $this->attempts = $telemetry->createCounter('lock.attempts', null, 'Distributed lock acquire outcomes'); $sequence = $project->getSequence(); @@ -59,11 +64,13 @@ final class Lock } /** - * Distributed lock around an arbitrary callback for a platform document. + * Try-once lock around an arbitrary callback for a platform document. + * Idempotent metadata writes should not make HTTP requests wait behind + * another pod doing the same update. */ public function run(string $collection, string $id, Closure $fn): void { - $this->execute($this->key($collection, $id), $collection, $fn); + $this->tryWithKey($this->key($collection, $id), $fn, target: $collection); } /** @@ -86,8 +93,8 @@ final class Lock public function withKey( string $key, Closure $fn, - int $ttl = self::TTL_SECONDS, - float $waitTimeout = self::WAIT_SECONDS, + int $ttl = self::FAIL_TTL_SECONDS, + float $waitTimeout = self::FAIL_WAIT_SECONDS, ?string $target = null, ): mixed { return $this->execute( @@ -96,6 +103,27 @@ final class Lock $fn, ttl: $ttl, waitTimeout: $waitTimeout, + skipOnContention: false, + ); + } + + /** + * Try-once lock around a callback. On contention, skip the callback and + * return null; this is for best-effort timestamp/access metadata writes. + */ + public function tryWithKey( + string $key, + Closure $fn, + int $ttl = self::SKIP_TTL_SECONDS, + ?string $target = null, + ): mixed { + return $this->execute( + $key, + $target ?? self::inferTargetFromKey($key), + $fn, + ttl: $ttl, + waitTimeout: self::SKIP_WAIT_SECONDS, + skipOnContention: true, ); } @@ -103,8 +131,9 @@ final class Lock string $key, string $target, Closure $fn, - int $ttl = self::TTL_SECONDS, - float $waitTimeout = self::WAIT_SECONDS, + int $ttl = self::FAIL_TTL_SECONDS, + float $waitTimeout = self::FAIL_WAIT_SECONDS, + bool $skipOnContention = false, ): mixed { if (! $this->enabled) { return $fn(); @@ -113,20 +142,65 @@ final class Lock $labels = ['target' => $target, 'project' => $this->projectInternalId]; try { - return ($this->withLock)($key, $ttl, function () use ($fn, $labels) { - $this->attempts->add(1, ['outcome' => self::OUTCOME_ACQUIRED, ...$labels]); - - return $fn(); - }, $waitTimeout); - } catch (LockContention) { - $this->attempts->add(1, ['outcome' => self::OUTCOME_CONTENDED, ...$labels]); - // No custom message: the lock key embeds collection and document ID. - throw new Exception(Exception::GENERAL_RESOURCE_LOCKED); + return ($this->useLock)($key, $ttl, function (UtopiaLock $lock) use ($fn, $key, $labels, $skipOnContention, $target, $waitTimeout): mixed { + return $this->executeWithLock($lock, $key, $target, $labels, $fn, $waitTimeout, $skipOnContention); + }); + } catch (CallbackRedisException $e) { + throw $e->getRedisException(); } catch (\RedisException $e) { $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); - throw $e; + return $fn(); + } + } + + /** + * @param array{target: string, project: string} $labels + */ + private function executeWithLock( + UtopiaLock $lock, + string $key, + string $target, + array $labels, + Closure $fn, + float $waitTimeout, + bool $skipOnContention, + ): mixed { + try { + $acquired = $lock->acquire($waitTimeout); + } catch (\RedisException $e) { + $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); + $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); + + return $fn(); + } + + if (! $acquired) { + if ($skipOnContention) { + $this->attempts->add(1, ['outcome' => self::OUTCOME_SKIPPED, ...$labels]); + + return null; + } + $this->attempts->add(1, ['outcome' => self::OUTCOME_CONTENDED, ...$labels]); + // No custom message: the lock key embeds collection and document ID. + throw new Exception(Exception::GENERAL_RESOURCE_LOCKED); + } + + $this->attempts->add(1, ['outcome' => self::OUTCOME_ACQUIRED, ...$labels]); + try { + try { + return $fn(); + } catch (\RedisException $e) { + throw new CallbackRedisException($e); + } + } finally { + try { + $lock->release(); + } catch (Throwable $e) { + $this->attempts->add(1, ['outcome' => self::OUTCOME_RELEASE_ERROR, ...$labels]); + $this->reportError(self::OUTCOME_RELEASE_ERROR, $key, $target, $e); + } } } @@ -148,7 +222,24 @@ final class Lock */ public function key(string $collection, string $id, ?string $attribute = null): string { - $key = "lock:platform:{$this->projectInternalId}:{$collection}:{$id}"; + return self::buildKey($this->projectInternalId, $collection, $id, $attribute); + } + + /** + * Build a key for a project document that was resolved after this request + * lock was constructed, such as router/custom-domain resolution. + */ + public function keyForProject(Document $project, string $collection, string $id, ?string $attribute = null): string + { + $sequence = $project->getSequence(); + $projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; + + return self::buildKey($projectInternalId, $collection, $id, $attribute); + } + + private static function buildKey(string $projectInternalId, string $collection, string $id, ?string $attribute = null): string + { + $key = "lock:platform:{$projectInternalId}:{$collection}:{$id}"; return $attribute === null ? $key : "{$key}:{$attribute}"; } @@ -196,3 +287,17 @@ final class Lock } } } + +final class CallbackRedisException extends \RuntimeException +{ + public function __construct(\RedisException $previous) + { + parent::__construct($previous->getMessage(), $previous->getCode(), $previous); + } + + public function getRedisException(): \RedisException + { + /** @var \RedisException */ + return parent::getPrevious(); + } +} diff --git a/src/Appwrite/Locking/PlatformDBLock.php b/src/Appwrite/Locking/PlatformDBLock.php new file mode 100644 index 0000000000..643bd2c7f7 --- /dev/null +++ b/src/Appwrite/Locking/PlatformDBLock.php @@ -0,0 +1,40 @@ +lock->key($collection, $id, $attribute) + : $this->lock->keyForProject($project, $collection, $id, $attribute); + + /** @var Document|null */ + return $this->lock->tryWithKey( + $key, + fn () => $this->authorization->skip(fn () => $this->dbForPlatform->updateDocument( + $collection, + $id, + new Document([$attribute => $value]) + )), + target: $collection + ); + } +} diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php index 1e5801e7fa..968ee0d722 100644 --- a/tests/unit/Locking/LockTest.php +++ b/tests/unit/Locking/LockTest.php @@ -6,9 +6,12 @@ namespace Tests\Unit\Locking; use Appwrite\Extend\Exception; use Appwrite\Locking\Lock; +use Appwrite\Locking\PlatformDBLock; use PHPUnit\Framework\TestCase; use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Validator\Authorization; use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Lock\Lock as UtopiaLock; use Utopia\Logger\Log; @@ -58,7 +61,7 @@ final class LockTest extends TestCase private function withLock(): \Closure { - return fn (string $key, int $ttl, \Closure $callback, float $timeout): mixed => (new MemoryLock($key, $this->heldLocks))->withLock($callback, $timeout); + return fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new MemoryLock($key, $this->heldLocks)); } public function test_run_uses_per_document_key_and_invokes_callback(): void @@ -73,17 +76,16 @@ final class LockTest extends TestCase $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); } - public function test_run_uses_cloud_lock_ttl_and_acquire_timeout(): void + public function test_run_uses_short_try_once_lock_window(): void { $ttl = null; $timeout = null; $lock = new Lock( - function (string $key, int $lockTtl, \Closure $callback, float $lockTimeout) use (&$ttl, &$timeout): mixed { + function (string $key, int $lockTtl, \Closure $callback) use (&$ttl, &$timeout): mixed { $ttl = $lockTtl; - $timeout = $lockTimeout; - return (new MemoryLock($key, $this->heldLocks))->withLock($callback, $lockTimeout); + return $callback(new InspectingLock(new MemoryLock($key, $this->heldLocks), $timeout)); }, new NoTelemetry(), $this->log, @@ -93,11 +95,11 @@ final class LockTest extends TestCase $lock->run('keys', 'k1', fn () => null); - $this->assertSame(1800, $ttl); - $this->assertSame(1500.0, $timeout); + $this->assertSame(5, $ttl); + $this->assertSame(0.0, $timeout); } - public function test_run_throws_on_contention(): void + public function test_run_skips_on_contention(): void { $key = self::KEY_PREFIX.'keys:k1'; $this->heldLocks[$key] = true; @@ -105,26 +107,45 @@ final class LockTest extends TestCase $called = false; $lock = $this->makeLock(); - try { - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - $this->fail('Expected lock contention to throw.'); - } catch (Exception $e) { - $this->assertSame(Exception::GENERAL_RESOURCE_LOCKED, $e->getType()); - $this->assertFalse($called); - $this->assertArrayHasKey($key, $this->heldLocks); - } + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertFalse($called); + $this->assertArrayHasKey($key, $this->heldLocks); } - public function test_backend_error_does_not_run_callback_unlocked(): void + public function test_run_or_fail_uses_short_http_lock_ttl_and_acquire_timeout(): void + { + $ttl = null; + $timeout = null; + + $lock = new Lock( + function (string $key, int $lockTtl, \Closure $callback) use (&$ttl, &$timeout): mixed { + $ttl = $lockTtl; + + return $callback(new InspectingLock(new MemoryLock($key, $this->heldLocks), $timeout)); + }, + new NoTelemetry(), + $this->log, + null, + $this->project, + ); + + $lock->runOrFail('keys', 'k1', fn () => null); + + $this->assertSame(10, $ttl); + $this->assertSame(3.0, $timeout); + } + + public function test_best_effort_backend_error_runs_callback_unlocked(): void { if (! \class_exists(\RedisException::class)) { $this->markTestSkipped('Redis extension is required to simulate RedisException.'); } $lock = new Lock( - fn (string $key, int $ttl, \Closure $callback, float $timeout): mixed => throw new \RedisException('redis unavailable'), + fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingAcquireLock(new \RedisException('redis unavailable'))), new NoTelemetry(), $this->log, null, @@ -132,24 +153,21 @@ final class LockTest extends TestCase ); $called = false; - try { - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - $this->fail('Expected Redis backend errors to bubble.'); - } catch (\RedisException) { - $this->assertFalse($called); - } + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertTrue($called); } - public function test_pool_checkout_error_does_not_run_callback_unlocked(): void + public function test_run_or_fail_backend_error_runs_callback_unlocked(): void { if (! \class_exists(\RedisException::class)) { $this->markTestSkipped('Redis extension is required to simulate RedisException.'); } $lock = new Lock( - fn (string $key, int $ttl, \Closure $callback, float $timeout): mixed => throw new \RedisException('redis unavailable'), + fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingAcquireLock(new \RedisException('redis unavailable'))), new NoTelemetry(), $this->log, null, @@ -157,14 +175,62 @@ final class LockTest extends TestCase ); $called = false; - try { - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - $this->fail('Expected Redis backend errors to bubble.'); - } catch (\RedisException) { - $this->assertFalse($called); + $lock->runOrFail('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertTrue($called); + } + + public function test_best_effort_pool_checkout_error_runs_callback_unlocked(): void + { + if (! \class_exists(\RedisException::class)) { + $this->markTestSkipped('Redis extension is required to simulate RedisException.'); } + + $lock = new Lock( + fn (string $key, int $ttl, \Closure $callback): mixed => throw new \RedisException('pool unavailable'), + new NoTelemetry(), + $this->log, + null, + $this->project, + ); + + $called = false; + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertTrue($called); + } + + public function test_callback_redis_exception_is_not_swallowed(): void + { + if (! \class_exists(\RedisException::class)) { + $this->markTestSkipped('Redis extension is required to simulate RedisException.'); + } + + $lock = $this->makeLock(); + + $this->expectException(\RedisException::class); + $lock->run('keys', 'k1', fn () => throw new \RedisException('callback failed')); + } + + public function test_release_error_after_callback_is_logged_but_not_thrown(): void + { + if (! \class_exists(\RedisException::class)) { + $this->markTestSkipped('Redis extension is required to simulate RedisException.'); + } + + $lock = new Lock( + fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingReleaseLock(new MemoryLock($key, $this->heldLocks), new \RedisException('release failed'))), + new NoTelemetry(), + $this->log, + null, + $this->project, + ); + + $this->assertSame('ok', $lock->runOrFail('keys', 'k1', fn () => 'ok')); } public function test_run_or_fail_throws_on_contention(): void @@ -205,6 +271,20 @@ final class LockTest extends TestCase $this->assertArrayNotHasKey($custom, $this->heldLocks); } + public function test_try_with_key_skips_on_contention(): void + { + $custom = 'lock:test:contended'; + $this->heldLocks[$custom] = true; + + $lock = $this->makeLock(); + $called = false; + $lock->tryWithKey($custom, function () use (&$called): void { + $called = true; + }, ttl: 5); + + $this->assertFalse($called); + } + public function test_with_key_throws_on_contention(): void { $custom = 'lock:test:contended'; @@ -251,18 +331,90 @@ final class LockTest extends TestCase $this->heldLocks[$key] = true; $called = false; - try { - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - $this->fail('Expected unknown-bucket contention to throw.'); - } catch (Exception $e) { - $this->assertSame(Exception::GENERAL_RESOURCE_LOCKED, $e->getType()); - $this->assertFalse($called, 'Lock without project sequence should hash to the unknown bucket'); - } + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertFalse($called, 'Lock without project sequence should hash to the unknown bucket'); unset($this->heldLocks[$key]); } + + public function test_key_for_project_uses_given_project_sequence(): void + { + $lock = new Lock( + $this->withLock(), + new NoTelemetry(), + $this->log, + null, + new Document(), + ); + + $project = new Document([ + '$id' => 'routed-project', + '$sequence' => '84', + ]); + + $this->assertSame( + 'lock:platform:84:projects:routed-project:accessedAt', + $lock->keyForProject($project, 'projects', 'routed-project', 'accessedAt') + ); + } + + public function test_platform_db_lock_updates_attribute_under_attribute_key(): void + { + $dbForPlatform = $this->createMock(Database::class); + $dbForPlatform + ->expects($this->once()) + ->method('updateDocument') + ->with( + 'projects', + 'p1', + $this->callback(fn (Document $document): bool => $document->getAttribute('accessedAt') === 'now') + ) + ->willReturn(new Document(['$id' => 'p1'])); + + $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); + + $this->assertSame( + 'p1', + $platformDBLock->tryUpdateAttribute('projects', 'p1', 'accessedAt', 'now')?->getId() + ); + $this->assertArrayNotHasKey(self::KEY_PREFIX.'projects:p1:accessedAt', $this->heldLocks); + } + + public function test_platform_db_lock_skips_routed_project_update_on_contention(): void + { + $lock = new Lock( + $this->withLock(), + new NoTelemetry(), + $this->log, + null, + new Document(), + ); + + $project = new Document([ + '$id' => 'routed-project', + '$sequence' => '84', + ]); + + $this->heldLocks['lock:platform:84:projects:routed-project:accessedAt'] = true; + + $dbForPlatform = $this->createMock(Database::class); + $dbForPlatform + ->expects($this->never()) + ->method('updateDocument'); + + $platformDBLock = new PlatformDBLock($lock, $dbForPlatform, new Authorization()); + + $this->assertNull($platformDBLock->tryUpdateAttribute( + 'projects', + 'routed-project', + 'accessedAt', + 'now', + $project + )); + } } final class MemoryLock implements UtopiaLock @@ -318,3 +470,107 @@ final class MemoryLock implements UtopiaLock } } } + +final class InspectingLock implements UtopiaLock +{ + public function __construct( + private readonly UtopiaLock $lock, + private ?float &$timeout, + ) { + } + + public function acquire(float $timeout = 0.0): bool + { + $this->timeout = $timeout; + + return $this->lock->acquire($timeout); + } + + public function tryAcquire(): bool + { + return $this->lock->tryAcquire(); + } + + public function release(): void + { + $this->lock->release(); + } + + public function withLock(callable $callback, float $timeout = 0.0): mixed + { + if (! $this->acquire($timeout)) { + throw new LockContention('Failed to acquire distributed lock'); + } + + try { + return $callback(); + } finally { + $this->release(); + } + } +} + +final class ThrowingAcquireLock implements UtopiaLock +{ + public function __construct(private readonly \Throwable $throwable) + { + } + + public function acquire(float $timeout = 0.0): bool + { + throw $this->throwable; + } + + public function tryAcquire(): bool + { + throw $this->throwable; + } + + public function release(): void + { + } + + public function withLock(callable $callback, float $timeout = 0.0): mixed + { + throw $this->throwable; + } +} + +final class ThrowingReleaseLock implements UtopiaLock +{ + public function __construct( + private readonly UtopiaLock $lock, + private readonly \Throwable $throwable, + ) { + } + + public function acquire(float $timeout = 0.0): bool + { + return $this->lock->acquire($timeout); + } + + public function tryAcquire(): bool + { + return $this->lock->tryAcquire(); + } + + public function release(): void + { + $this->lock->release(); + + throw $this->throwable; + } + + public function withLock(callable $callback, float $timeout = 0.0): mixed + { + if (! $this->acquire($timeout)) { + throw new LockContention('Failed to acquire distributed lock'); + } + + try { + return $callback(); + } finally { + $this->release(); + } + } +} From e339a157a1de16068a5cee5098452aee3cad61d1 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 1 Jul 2026 12:27:53 +0100 Subject: [PATCH 09/24] Harden distributed lock pool handling and add document-level helper - Fail open when the lock pool is exhausted: Pool::pop() throws a bare \Exception (not \RedisException), which previously escaped as a 500. Catch the literal class only so business exceptions keep propagating. - Scale the lock pool per-worker like every other pool; _APP_POOL_SIZE_LOCK now overrides the per-worker size instead of a flat value. - Keep the lock fail-fast tuning scoped to the lock pool and apply it after the group-wide reconnect defaults, which were overwriting it. Leaves the shared cache pool untouched. - Add PlatformDBLock::tryUpdateDocument for multi-attribute sparse updates, locking at the document key. - Add unit tests for the document helper and the pool-exhaustion fail-open. --- app/controllers/shared/api.php | 6 +- app/init/registers.php | 19 +++-- src/Appwrite/Locking/Lock.php | 13 +++ src/Appwrite/Locking/PlatformDBLock.php | 51 ++++++++++-- tests/unit/Locking/LockTest.php | 100 ++++++++++++++++++++++++ 5 files changed, 172 insertions(+), 17 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 0c0215f0c0..aef29b542b 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -16,7 +16,6 @@ use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\EventProcessor; -use Appwrite\Locking\Lock; use Appwrite\Locking\PlatformDBLock; use Appwrite\Platform\Modules\Storage\Config\CacheControl; use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl; @@ -62,11 +61,10 @@ Http::init() ->inject('team') ->inject('apiKey') ->inject('authorization') - ->inject('lock') ->inject('platformDBLock') ->inject('impersonatorUser') ->inject('targetUser') - ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Lock $lock, PlatformDBLock $platformDBLock, Document $impersonatorUser, User $targetUser) { + ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, PlatformDBLock $platformDBLock, Document $impersonatorUser, User $targetUser) { /** * Handle user authentication and session validation. @@ -212,7 +210,7 @@ Http::init() } if (! $updates->isEmpty()) { - $lock->run('keys', $dbKey->getId(), function () use ($dbForPlatform, $dbKey, $updates, $apiKey, $project, $user, $team) { + $platformDBLock->tryRun('keys', $dbKey->getId(), function () use ($dbForPlatform, $dbKey, $updates, $apiKey, $project, $user, $team) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); if (! empty($apiKey->getProjectId())) { diff --git a/app/init/registers.php b/app/init/registers.php index e4723fc02d..d620bef1ea 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -238,7 +238,6 @@ $register->set('pools', function () { 'dsns' => System::getEnv('_APP_CONNECTIONS_LOCKS', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], - 'size' => max(1, (int) System::getEnv('_APP_POOL_SIZE_LOCK', 10)), ], ]; @@ -252,6 +251,12 @@ $register->set('pools', function () { // job may hold a connection, so pools must cover the coroutine count. $poolSize = max($poolSize, (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1)); + // Scale the lock pool per-worker like every other pool. _APP_POOL_SIZE_LOCK + // overrides the per-worker size; without it the lock pool tracks $poolSize + // so a many-worker host doesn't open size * workerCount lock connections. + $connections['lock']['size'] = max(1, (int) System::getEnv('_APP_POOL_SIZE_LOCK', $poolSize)); + + $lockPool = null; foreach ($connections as $key => $connection) { $type = $connection['type']; $multiple = $connection['multiple']; @@ -389,11 +394,8 @@ $register->set('pools', function () { } }); - if ($type === 'cache' || $type === 'lock') { - $pool - ->setRetryAttempts(1) - ->setReconnectAttempts(1) - ->setSynchronizationTimeout(1); + if ($type === 'lock') { + $lockPool = $pool; } $group->add($pool); @@ -408,6 +410,11 @@ $register->set('pools', function () { $group->setReconnectAttempts($reconnectAttempts); $group->setReconnectSleep($reconnectSleep); + // Apply lock-only fail-fast tuning after the group-wide defaults, which + // iterate every pool and would otherwise overwrite reconnectAttempts. Kept + // off the shared cache pool so its blip tolerance is unchanged. + $lockPool?->setRetryAttempts(1)->setReconnectAttempts(1)->setSynchronizationTimeout(1); + return $group; }); diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php index 8a2061425b..a7422337a0 100644 --- a/src/Appwrite/Locking/Lock.php +++ b/src/Appwrite/Locking/Lock.php @@ -151,6 +151,19 @@ final class Lock $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); + return $fn(); + } catch (\Exception $e) { + // Pool::pop() throws a bare \Exception (not \RedisException) when it + // can't hand out a connection within the retry/sync-timeout budget. + // That happens before the callback runs, so fail open. Match the + // literal class only: business exceptions (Appwrite\Extend\Exception, + // etc.) also extend \Exception and must keep propagating. + if (get_class($e) !== \Exception::class) { + throw $e; + } + $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); + $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); + return $fn(); } } diff --git a/src/Appwrite/Locking/PlatformDBLock.php b/src/Appwrite/Locking/PlatformDBLock.php index 643bd2c7f7..a43cbec842 100644 --- a/src/Appwrite/Locking/PlatformDBLock.php +++ b/src/Appwrite/Locking/PlatformDBLock.php @@ -2,6 +2,7 @@ namespace Appwrite\Locking; +use Closure; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; @@ -15,6 +16,20 @@ final class PlatformDBLock ) { } + public function tryRun( + string $collection, + string $id, + Closure $callback, + ?string $attribute = null, + ?Document $project = null, + ): mixed { + $key = $project === null + ? $this->lock->key($collection, $id, $attribute) + : $this->lock->keyForProject($project, $collection, $id, $attribute); + + return $this->lock->tryWithKey($key, $callback, target: $collection); + } + public function tryUpdateAttribute( string $collection, string $id, @@ -22,19 +37,41 @@ final class PlatformDBLock mixed $value, ?Document $project = null, ): ?Document { - $key = $project === null - ? $this->lock->key($collection, $id, $attribute) - : $this->lock->keyForProject($project, $collection, $id, $attribute); - /** @var Document|null */ - return $this->lock->tryWithKey( - $key, + return $this->tryRun( + $collection, + $id, fn () => $this->authorization->skip(fn () => $this->dbForPlatform->updateDocument( $collection, $id, new Document([$attribute => $value]) )), - target: $collection + $attribute, + $project + ); + } + + /** + * Document-level counterpart to tryUpdateAttribute() for sparse updates + * that change several fields at once. Locks at the document key (no + * attribute suffix) so the whole update is mutually exclusive. + */ + public function tryUpdateDocument( + string $collection, + string $id, + Document $updates, + ?Document $project = null, + ): ?Document { + /** @var Document|null */ + return $this->tryRun( + $collection, + $id, + fn () => $this->authorization->skip(fn () => $this->dbForPlatform->updateDocument( + $collection, + $id, + $updates + )), + project: $project ); } } diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php index 968ee0d722..673ec1519d 100644 --- a/tests/unit/Locking/LockTest.php +++ b/tests/unit/Locking/LockTest.php @@ -383,6 +383,106 @@ final class LockTest extends TestCase $this->assertArrayNotHasKey(self::KEY_PREFIX.'projects:p1:accessedAt', $this->heldLocks); } + public function test_platform_db_lock_try_run_uses_collection_key(): void + { + $dbForPlatform = $this->createMock(Database::class); + $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); + + $called = false; + $platformDBLock->tryRun('keys', 'k1', function () use (&$called): void { + $called = true; + }); + + $this->assertTrue($called); + $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); + } + + public function test_platform_db_lock_try_run_skips_on_contention(): void + { + $this->heldLocks[self::KEY_PREFIX.'keys:k1'] = true; + + $dbForPlatform = $this->createMock(Database::class); + $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); + + $called = false; + $platformDBLock->tryRun('keys', 'k1', function () use (&$called): void { + $called = true; + }); + + $this->assertFalse($called); + } + + public function test_platform_db_lock_updates_document_under_document_key(): void + { + $updates = new Document(['accessedAt' => 'now', 'sdks' => ['php']]); + + $dbForPlatform = $this->createMock(Database::class); + $dbForPlatform + ->expects($this->once()) + ->method('updateDocument') + ->with('keys', 'k1', $updates) + ->willReturn(new Document(['$id' => 'k1'])); + + $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); + + $this->assertSame( + 'k1', + $platformDBLock->tryUpdateDocument('keys', 'k1', $updates)?->getId() + ); + // Document-level lock has no attribute suffix and is released after the write. + $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); + } + + public function test_platform_db_lock_skips_document_update_on_contention(): void + { + $this->heldLocks[self::KEY_PREFIX.'keys:k1'] = true; + + $dbForPlatform = $this->createMock(Database::class); + $dbForPlatform + ->expects($this->never()) + ->method('updateDocument'); + + $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); + + $this->assertNull($platformDBLock->tryUpdateDocument('keys', 'k1', new Document(['accessedAt' => 'now']))); + } + + public function test_pool_checkout_exception_runs_callback_unlocked(): void + { + // Pool::pop() throws a bare \Exception on exhaustion, before the lock is + // acquired. The wrapper must fail open and run the callback unlocked. + $lock = new Lock( + fn (string $key, int $ttl, \Closure $callback): mixed => throw new \Exception('Pool \'lock\' is empty'), + new NoTelemetry(), + $this->log, + null, + $this->project, + ); + + $called = false; + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertTrue($called); + } + + public function test_non_pool_exception_is_not_swallowed(): void + { + // Only the literal \Exception (pool exhaustion) is treated as fail-open; + // any subclass thrown while entering the lock must keep propagating. + $lock = new Lock( + fn (string $key, int $ttl, \Closure $callback): mixed => throw new \RuntimeException('unexpected'), + new NoTelemetry(), + $this->log, + null, + $this->project, + ); + + $this->expectException(\RuntimeException::class); + $lock->run('keys', 'k1', fn () => null); + } + public function test_platform_db_lock_skips_routed_project_update_on_contention(): void { $lock = new Lock( From 2171789f7d8038547669b58f6a6fb3dabcb7bfb8 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 1 Jul 2026 14:57:39 +0100 Subject: [PATCH 10/24] Refine API key metadata locking --- app/config/variables.php | 4 +- app/controllers/shared/api.php | 79 ++++++++++++++++++++++------------ 2 files changed, 53 insertions(+), 30 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index fe5315d391..8a6223dc5e 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -471,9 +471,9 @@ return [ ], [ 'name' => '_APP_POOL_SIZE_LOCK', - 'description' => 'Maximum number of connections in the distributed lock connection pool. By default, set to 10.', + 'description' => 'Maximum number of connections in the distributed lock connection pool. By default, this follows the per-worker pool size calculated from _APP_CONNECTIONS_MAX, _APP_POOL_CLIENTS, worker count, and _APP_WORKER_MAX_COROUTINES.', 'introduction' => '1.9.3', - 'default' => '10', + 'default' => '', 'required' => false, 'question' => '', 'filter' => '' diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index aef29b542b..8741353e4d 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -163,22 +163,31 @@ Http::init() // For standard keys, update last accessed time if (\in_array($apiKey->getType(), [API_KEY_STANDARD, API_KEY_ORGANIZATION, API_KEY_ACCOUNT])) { $dbKey = null; + $ownerCollection = ''; + $ownerId = ''; + $keySecret = $request->getHeaderLine('x-appwrite-key', ''); if (! empty($apiKey->getProjectId())) { + $ownerCollection = 'projects'; + $ownerId = $project->getId(); $dbKey = $project->find( key: 'secret', - find: $request->getHeaderLine('x-appwrite-key', ''), + find: $keySecret, subject: 'keys' ); } elseif (! empty($apiKey->getUserId())) { + $ownerCollection = 'users'; + $ownerId = $user->getId(); $dbKey = $user->find( key: 'secret', - find: $request->getHeaderLine('x-appwrite-key', ''), + find: $keySecret, subject: 'keys' ); } elseif (! empty($apiKey->getTeamId())) { + $ownerCollection = 'teams'; + $ownerId = $team->getId(); $dbKey = $team->find( key: 'secret', - find: $request->getHeaderLine('x-appwrite-key', ''), + find: $keySecret, subject: 'keys' ); } @@ -187,39 +196,53 @@ Http::init() throw new Exception(Exception::USER_UNAUTHORIZED); } - $updates = new Document(); - - $accessedAt = $dbKey->getAttribute('accessedAt', 0); - - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $accessedAt) { - $updates->setAttribute('accessedAt', DateTime::now()); - } - - $sdkValidator = new WhiteList($servers, true); $sdk = $request->getHeaderLine('x-sdk-name', 'UNKNOWN'); + $sdkValidator = new WhiteList($servers, true); + $buildKeyUpdates = function (Document $key) use ($sdk, $sdkValidator): Document { + $updates = new Document(); - if ($sdk !== 'UNKNOWN' && $sdkValidator->isValid($sdk)) { - $sdks = $dbKey->getAttribute('sdks', []); - - if (! in_array($sdk, $sdks)) { - $sdks[] = $sdk; - - $updates->setAttribute('sdks', $sdks); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $key->getAttribute('accessedAt', 0)) { $updates->setAttribute('accessedAt', DateTime::now()); } - } + + if ($sdk !== 'UNKNOWN' && $sdkValidator->isValid($sdk)) { + $sdks = $key->getAttribute('sdks', []); + + if (! in_array($sdk, $sdks)) { + $sdks[] = $sdk; + + $updates->setAttribute('sdks', $sdks); + $updates->setAttribute('accessedAt', DateTime::now()); + } + } + + return $updates; + }; + + $updates = $buildKeyUpdates($dbKey); if (! $updates->isEmpty()) { - $platformDBLock->tryRun('keys', $dbKey->getId(), function () use ($dbForPlatform, $dbKey, $updates, $apiKey, $project, $user, $team) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); + $platformDBLock->tryRun('keys', $dbKey->getId(), function () use ($dbForPlatform, $ownerCollection, $ownerId, $keySecret, $buildKeyUpdates) { + $currentOwner = $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->getDocument($ownerCollection, $ownerId)); + $currentDbKey = $currentOwner->find( + key: 'secret', + find: $keySecret, + subject: 'keys' + ); - if (! empty($apiKey->getProjectId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); - } elseif (! empty($apiKey->getUserId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); - } elseif (! empty($apiKey->getTeamId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); + if (! $currentDbKey) { + return; } + + $updates = $buildKeyUpdates($currentDbKey); + + if ($updates->isEmpty()) { + return; + } + + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $currentDbKey->getId(), $updates)); + + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument($ownerCollection, $ownerId)); }); } From d4d77c5790ff20b00a348d00b36b614a8b2f8943 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 1 Jul 2026 15:23:39 +0100 Subject: [PATCH 11/24] Keep lock pool config in cloud --- app/config/variables.php | 18 ------------------ app/init/registers.php | 22 +++------------------- docker-compose.yml | 2 -- 3 files changed, 3 insertions(+), 39 deletions(-) diff --git a/app/config/variables.php b/app/config/variables.php index 8a6223dc5e..efa500953d 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -460,24 +460,6 @@ return [ 'question' => '', 'filter' => '' ], - [ - 'name' => '_APP_CONNECTIONS_LOCKS', - 'description' => 'Connection DSN for distributed locks. By default, locks use the Redis server configured by _APP_REDIS_*.', - 'introduction' => '1.9.3', - 'default' => '', - 'required' => false, - 'question' => '', - 'filter' => '' - ], - [ - 'name' => '_APP_POOL_SIZE_LOCK', - 'description' => 'Maximum number of connections in the distributed lock connection pool. By default, this follows the per-worker pool size calculated from _APP_CONNECTIONS_MAX, _APP_POOL_CLIENTS, worker count, and _APP_WORKER_MAX_COROUTINES.', - 'introduction' => '1.9.3', - 'default' => '', - 'required' => false, - 'question' => '', - 'filter' => '' - ], ], ], [ diff --git a/app/init/registers.php b/app/init/registers.php index d620bef1ea..43ae64bc53 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -235,7 +235,7 @@ $register->set('pools', function () { ], 'lock' => [ 'type' => 'lock', - 'dsns' => System::getEnv('_APP_CONNECTIONS_LOCKS', $fallbackForRedis), + 'dsns' => $fallbackForRedis, 'multiple' => false, 'schemes' => ['redis'], ], @@ -251,17 +251,10 @@ $register->set('pools', function () { // job may hold a connection, so pools must cover the coroutine count. $poolSize = max($poolSize, (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1)); - // Scale the lock pool per-worker like every other pool. _APP_POOL_SIZE_LOCK - // overrides the per-worker size; without it the lock pool tracks $poolSize - // so a many-worker host doesn't open size * workerCount lock connections. - $connections['lock']['size'] = max(1, (int) System::getEnv('_APP_POOL_SIZE_LOCK', $poolSize)); - - $lockPool = null; foreach ($connections as $key => $connection) { $type = $connection['type']; $multiple = $connection['multiple']; $schemes = $connection['schemes']; - $poolSizeForConnection = $connection['size'] ?? $poolSize; $config = []; $dsns = explode(',', $connection['dsns']); foreach ($dsns as &$dsn) { @@ -343,8 +336,8 @@ $register->set('pools', function () { // PubSub workers hold one long-lived subscribed connection and also need // spare capacity for publishes from the same process. $connectionPoolSize = match ($type) { - 'pubsub' => max(2, $poolSizeForConnection), - default => $poolSizeForConnection, + 'pubsub' => max(2, $poolSize), + default => $poolSize, }; $pool = new Pool($poolAdapter, $name, $connectionPoolSize, function () use ($type, $resource, $dsn) { @@ -394,10 +387,6 @@ $register->set('pools', function () { } }); - if ($type === 'lock') { - $lockPool = $pool; - } - $group->add($pool); } @@ -410,11 +399,6 @@ $register->set('pools', function () { $group->setReconnectAttempts($reconnectAttempts); $group->setReconnectSleep($reconnectSleep); - // Apply lock-only fail-fast tuning after the group-wide defaults, which - // iterate every pool and would otherwise overwrite reconnectAttempts. Kept - // off the shared cache pool so its blip tolerance is unchanged. - $lockPool?->setRetryAttempts(1)->setReconnectAttempts(1)->setSynchronizationTimeout(1); - return $group; }); diff --git a/docker-compose.yml b/docker-compose.yml index a7032b6414..9d8194dbcd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,8 +112,6 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_CONNECTIONS_LOCKS - - _APP_POOL_SIZE_LOCK - _APP_DB_ADAPTER - _APP_DB_HOST - _APP_DB_PORT From 75912b2130dc32402c69257e416786b057d74656 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 1 Jul 2026 15:36:46 +0100 Subject: [PATCH 12/24] Keep API key metadata update simple --- app/controllers/shared/api.php | 83 ++++++++++++---------------------- 1 file changed, 29 insertions(+), 54 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 8741353e4d..3fa889cb2a 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -163,31 +163,22 @@ Http::init() // For standard keys, update last accessed time if (\in_array($apiKey->getType(), [API_KEY_STANDARD, API_KEY_ORGANIZATION, API_KEY_ACCOUNT])) { $dbKey = null; - $ownerCollection = ''; - $ownerId = ''; - $keySecret = $request->getHeaderLine('x-appwrite-key', ''); if (! empty($apiKey->getProjectId())) { - $ownerCollection = 'projects'; - $ownerId = $project->getId(); $dbKey = $project->find( key: 'secret', - find: $keySecret, + find: $request->getHeaderLine('x-appwrite-key', ''), subject: 'keys' ); } elseif (! empty($apiKey->getUserId())) { - $ownerCollection = 'users'; - $ownerId = $user->getId(); $dbKey = $user->find( key: 'secret', - find: $keySecret, + find: $request->getHeaderLine('x-appwrite-key', ''), subject: 'keys' ); } elseif (! empty($apiKey->getTeamId())) { - $ownerCollection = 'teams'; - $ownerId = $team->getId(); $dbKey = $team->find( key: 'secret', - find: $keySecret, + find: $request->getHeaderLine('x-appwrite-key', ''), subject: 'keys' ); } @@ -196,54 +187,38 @@ Http::init() throw new Exception(Exception::USER_UNAUTHORIZED); } - $sdk = $request->getHeaderLine('x-sdk-name', 'UNKNOWN'); - $sdkValidator = new WhiteList($servers, true); - $buildKeyUpdates = function (Document $key) use ($sdk, $sdkValidator): Document { - $updates = new Document(); + $updates = new Document(); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $key->getAttribute('accessedAt', 0)) { + $accessedAt = $dbKey->getAttribute('accessedAt', 0); + + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $accessedAt) { + $updates->setAttribute('accessedAt', DateTime::now()); + } + + $sdkValidator = new WhiteList($servers, true); + $sdk = $request->getHeaderLine('x-sdk-name', 'UNKNOWN'); + + if ($sdk !== 'UNKNOWN' && $sdkValidator->isValid($sdk)) { + $sdks = $dbKey->getAttribute('sdks', []); + + if (! in_array($sdk, $sdks)) { + $sdks[] = $sdk; + + $updates->setAttribute('sdks', $sdks); $updates->setAttribute('accessedAt', DateTime::now()); } - - if ($sdk !== 'UNKNOWN' && $sdkValidator->isValid($sdk)) { - $sdks = $key->getAttribute('sdks', []); - - if (! in_array($sdk, $sdks)) { - $sdks[] = $sdk; - - $updates->setAttribute('sdks', $sdks); - $updates->setAttribute('accessedAt', DateTime::now()); - } - } - - return $updates; - }; - - $updates = $buildKeyUpdates($dbKey); + } if (! $updates->isEmpty()) { - $platformDBLock->tryRun('keys', $dbKey->getId(), function () use ($dbForPlatform, $ownerCollection, $ownerId, $keySecret, $buildKeyUpdates) { - $currentOwner = $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->getDocument($ownerCollection, $ownerId)); - $currentDbKey = $currentOwner->find( - key: 'secret', - find: $keySecret, - subject: 'keys' - ); + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); - if (! $currentDbKey) { - return; - } - - $updates = $buildKeyUpdates($currentDbKey); - - if ($updates->isEmpty()) { - return; - } - - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $currentDbKey->getId(), $updates)); - - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument($ownerCollection, $ownerId)); - }); + if (! empty($apiKey->getProjectId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); + } elseif (! empty($apiKey->getUserId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); + } elseif (! empty($apiKey->getTeamId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); + } } $userClone = clone $user; From 8e5d5589fb0362a24258482cf083c61a5dea5f0c Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 1 Jul 2026 16:09:29 +0100 Subject: [PATCH 13/24] Add temporary lock test logs --- app/controllers/general.php | 27 ++++++++++++--------------- src/Appwrite/Locking/Lock.php | 26 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 93692b72a2..1a7cfa70ae 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -12,7 +12,6 @@ use Appwrite\Event\Message\Delete as DeleteMessage; use Appwrite\Event\Publisher\Certificate; use Appwrite\Event\Publisher\Delete as DeletePublisher; use Appwrite\Extend\Exception as AppwriteException; -use Appwrite\Locking\PlatformDBLock; use Appwrite\Network\Cors; use Appwrite\Platform\Appwrite; use Appwrite\SDK\AuthType; @@ -77,7 +76,7 @@ use Utopia\Validator\Text; Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) +function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { $host = $request->getHostname(); if (!empty($previewHostname)) { @@ -146,7 +145,9 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S if (!$project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $platformDBLock->tryUpdateAttribute('projects', $project->getId(), 'accessedAt', DateTime::now(), $project); + $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ + 'accessedAt' => DateTime::now() + ]))); } /** @@ -840,9 +841,8 @@ Http::init() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('platformDBLock') ->inject('params') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock, array $params) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, array $params) { /* * Appwrite Router */ @@ -850,7 +850,7 @@ Http::init() $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!\in_array($hostname, $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { $utopia->match($request)?->route->label('router', true); } } @@ -1153,15 +1153,14 @@ Http::options() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('platformDBLock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { /* * Appwrite Router */ $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { $utopia->match($request)?->route->label('router', true); } } @@ -1556,14 +1555,13 @@ Http::get('/robots.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('platformDBLock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { $utopia->match($request)?->route->label('router', true); } } @@ -1591,14 +1589,13 @@ Http::get('/humans.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('platformDBLock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { $utopia->match($request)?->route->label('router', true); } } diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php index a7422337a0..0b703abbf3 100644 --- a/src/Appwrite/Locking/Lock.php +++ b/src/Appwrite/Locking/Lock.php @@ -35,6 +35,8 @@ final class Lock private const OUTCOME_RELEASE_ERROR = 'release_error'; + private const OUTCOME_DISABLED = 'disabled'; + private readonly bool $enabled; private readonly mixed $attempts; @@ -136,6 +138,8 @@ final class Lock bool $skipOnContention = false, ): mixed { if (! $this->enabled) { + $this->logTest(self::OUTCOME_DISABLED, $key, $target); + return $fn(); } @@ -149,6 +153,7 @@ final class Lock throw $e->getRedisException(); } catch (\RedisException $e) { $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); + $this->logTest(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); return $fn(); @@ -162,6 +167,7 @@ final class Lock throw $e; } $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); + $this->logTest(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); return $fn(); @@ -184,6 +190,7 @@ final class Lock $acquired = $lock->acquire($waitTimeout); } catch (\RedisException $e) { $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); + $this->logTest(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); return $fn(); @@ -192,15 +199,18 @@ final class Lock if (! $acquired) { if ($skipOnContention) { $this->attempts->add(1, ['outcome' => self::OUTCOME_SKIPPED, ...$labels]); + $this->logTest(self::OUTCOME_SKIPPED, $key, $target); return null; } $this->attempts->add(1, ['outcome' => self::OUTCOME_CONTENDED, ...$labels]); + $this->logTest(self::OUTCOME_CONTENDED, $key, $target); // No custom message: the lock key embeds collection and document ID. throw new Exception(Exception::GENERAL_RESOURCE_LOCKED); } $this->attempts->add(1, ['outcome' => self::OUTCOME_ACQUIRED, ...$labels]); + $this->logTest(self::OUTCOME_ACQUIRED, $key, $target); try { try { return $fn(); @@ -212,6 +222,7 @@ final class Lock $lock->release(); } catch (Throwable $e) { $this->attempts->add(1, ['outcome' => self::OUTCOME_RELEASE_ERROR, ...$labels]); + $this->logTest(self::OUTCOME_RELEASE_ERROR, $key, $target, $e); $this->reportError(self::OUTCOME_RELEASE_ERROR, $key, $target, $e); } } @@ -257,6 +268,21 @@ final class Lock return $attribute === null ? $key : "{$key}:{$attribute}"; } + private function logTest(string $outcome, string $key, string $target, ?Throwable $e = null): void + { + $message = '[dist-lock-test] lock ' + .$outcome + .' target='.$target + .' project='.$this->projectInternalId + .' keyPattern='.preg_replace('/:[^:]+$/', ':*', $key); + + if ($e !== null) { + $message .= ' exception='.get_class($e).' message='.$e->getMessage(); + } + + Console::info($message); + } + /** * Rate-limited to one push per REPORT_RATE_LIMIT_SECONDS per (action, target) * so a sustained backend outage doesn't flood Sentry across the pod fleet. From 098fd34475fd2a7790172fdeeaebaa3675f496e4 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 1 Jul 2026 16:29:57 +0100 Subject: [PATCH 14/24] Restore platform metadata locks --- app/controllers/general.php | 27 +++++++++++++++------------ app/controllers/shared/api.php | 22 +++++++++++++--------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 1a7cfa70ae..93692b72a2 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -12,6 +12,7 @@ use Appwrite\Event\Message\Delete as DeleteMessage; use Appwrite\Event\Publisher\Certificate; use Appwrite\Event\Publisher\Delete as DeletePublisher; use Appwrite\Extend\Exception as AppwriteException; +use Appwrite\Locking\PlatformDBLock; use Appwrite\Network\Cors; use Appwrite\Platform\Appwrite; use Appwrite\SDK\AuthType; @@ -76,7 +77,7 @@ use Utopia\Validator\Text; Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) +function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { $host = $request->getHostname(); if (!empty($previewHostname)) { @@ -145,9 +146,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S if (!$project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ - 'accessedAt' => DateTime::now() - ]))); + $platformDBLock->tryUpdateAttribute('projects', $project->getId(), 'accessedAt', DateTime::now(), $project); } /** @@ -841,8 +840,9 @@ Http::init() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') + ->inject('platformDBLock') ->inject('params') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, array $params) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock, array $params) { /* * Appwrite Router */ @@ -850,7 +850,7 @@ Http::init() $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!\in_array($hostname, $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1153,14 +1153,15 @@ Http::options() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { + ->inject('platformDBLock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { /* * Appwrite Router */ $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1555,13 +1556,14 @@ Http::get('/robots.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { + ->inject('platformDBLock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1589,13 +1591,14 @@ Http::get('/humans.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { + ->inject('platformDBLock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { $utopia->match($request)?->route->label('router', true); } } diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 3fa889cb2a..0c0215f0c0 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -16,6 +16,7 @@ use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\EventProcessor; +use Appwrite\Locking\Lock; use Appwrite\Locking\PlatformDBLock; use Appwrite\Platform\Modules\Storage\Config\CacheControl; use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl; @@ -61,10 +62,11 @@ Http::init() ->inject('team') ->inject('apiKey') ->inject('authorization') + ->inject('lock') ->inject('platformDBLock') ->inject('impersonatorUser') ->inject('targetUser') - ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, PlatformDBLock $platformDBLock, Document $impersonatorUser, User $targetUser) { + ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Lock $lock, PlatformDBLock $platformDBLock, Document $impersonatorUser, User $targetUser) { /** * Handle user authentication and session validation. @@ -210,15 +212,17 @@ Http::init() } if (! $updates->isEmpty()) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); + $lock->run('keys', $dbKey->getId(), function () use ($dbForPlatform, $dbKey, $updates, $apiKey, $project, $user, $team) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); - if (! empty($apiKey->getProjectId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); - } elseif (! empty($apiKey->getUserId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); - } elseif (! empty($apiKey->getTeamId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); - } + if (! empty($apiKey->getProjectId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); + } elseif (! empty($apiKey->getUserId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); + } elseif (! empty($apiKey->getTeamId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); + } + }); } $userClone = clone $user; From 9579a8fbc1758ad8c47aba0a178ae00874d4f913 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 1 Jul 2026 17:54:46 +0100 Subject: [PATCH 15/24] Clean up lock metadata updates --- app/controllers/shared/api.php | 24 +++++++++--------------- src/Appwrite/Locking/Lock.php | 26 -------------------------- tests/unit/Locking/LockTest.php | 29 ++++++++++++++++++----------- 3 files changed, 27 insertions(+), 52 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 0c0215f0c0..232d9a645c 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -16,7 +16,6 @@ use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\EventProcessor; -use Appwrite\Locking\Lock; use Appwrite\Locking\PlatformDBLock; use Appwrite\Platform\Modules\Storage\Config\CacheControl; use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl; @@ -62,11 +61,10 @@ Http::init() ->inject('team') ->inject('apiKey') ->inject('authorization') - ->inject('lock') ->inject('platformDBLock') ->inject('impersonatorUser') ->inject('targetUser') - ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Lock $lock, PlatformDBLock $platformDBLock, Document $impersonatorUser, User $targetUser) { + ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, PlatformDBLock $platformDBLock, Document $impersonatorUser, User $targetUser) { /** * Handle user authentication and session validation. @@ -211,18 +209,14 @@ Http::init() } } - if (! $updates->isEmpty()) { - $lock->run('keys', $dbKey->getId(), function () use ($dbForPlatform, $dbKey, $updates, $apiKey, $project, $user, $team) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); - - if (! empty($apiKey->getProjectId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); - } elseif (! empty($apiKey->getUserId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); - } elseif (! empty($apiKey->getTeamId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); - } - }); + if (! $updates->isEmpty() && $platformDBLock->tryUpdateDocument('keys', $dbKey->getId(), $updates) instanceof Document) { + if (! empty($apiKey->getProjectId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); + } elseif (! empty($apiKey->getUserId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); + } elseif (! empty($apiKey->getTeamId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); + } } $userClone = clone $user; diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php index 0b703abbf3..a7422337a0 100644 --- a/src/Appwrite/Locking/Lock.php +++ b/src/Appwrite/Locking/Lock.php @@ -35,8 +35,6 @@ final class Lock private const OUTCOME_RELEASE_ERROR = 'release_error'; - private const OUTCOME_DISABLED = 'disabled'; - private readonly bool $enabled; private readonly mixed $attempts; @@ -138,8 +136,6 @@ final class Lock bool $skipOnContention = false, ): mixed { if (! $this->enabled) { - $this->logTest(self::OUTCOME_DISABLED, $key, $target); - return $fn(); } @@ -153,7 +149,6 @@ final class Lock throw $e->getRedisException(); } catch (\RedisException $e) { $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); - $this->logTest(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); return $fn(); @@ -167,7 +162,6 @@ final class Lock throw $e; } $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); - $this->logTest(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); return $fn(); @@ -190,7 +184,6 @@ final class Lock $acquired = $lock->acquire($waitTimeout); } catch (\RedisException $e) { $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); - $this->logTest(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); return $fn(); @@ -199,18 +192,15 @@ final class Lock if (! $acquired) { if ($skipOnContention) { $this->attempts->add(1, ['outcome' => self::OUTCOME_SKIPPED, ...$labels]); - $this->logTest(self::OUTCOME_SKIPPED, $key, $target); return null; } $this->attempts->add(1, ['outcome' => self::OUTCOME_CONTENDED, ...$labels]); - $this->logTest(self::OUTCOME_CONTENDED, $key, $target); // No custom message: the lock key embeds collection and document ID. throw new Exception(Exception::GENERAL_RESOURCE_LOCKED); } $this->attempts->add(1, ['outcome' => self::OUTCOME_ACQUIRED, ...$labels]); - $this->logTest(self::OUTCOME_ACQUIRED, $key, $target); try { try { return $fn(); @@ -222,7 +212,6 @@ final class Lock $lock->release(); } catch (Throwable $e) { $this->attempts->add(1, ['outcome' => self::OUTCOME_RELEASE_ERROR, ...$labels]); - $this->logTest(self::OUTCOME_RELEASE_ERROR, $key, $target, $e); $this->reportError(self::OUTCOME_RELEASE_ERROR, $key, $target, $e); } } @@ -268,21 +257,6 @@ final class Lock return $attribute === null ? $key : "{$key}:{$attribute}"; } - private function logTest(string $outcome, string $key, string $target, ?Throwable $e = null): void - { - $message = '[dist-lock-test] lock ' - .$outcome - .' target='.$target - .' project='.$this->projectInternalId - .' keyPattern='.preg_replace('/:[^:]+$/', ':*', $key); - - if ($e !== null) { - $message .= ' exception='.get_class($e).' message='.$e->getMessage(); - } - - Console::info($message); - } - /** * Rate-limited to one push per REPORT_RATE_LIMIT_SECONDS per (action, target) * so a sustained backend outage doesn't flood Sentry across the pod fleet. diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php index 673ec1519d..20d003830d 100644 --- a/tests/unit/Locking/LockTest.php +++ b/tests/unit/Locking/LockTest.php @@ -85,7 +85,9 @@ final class LockTest extends TestCase function (string $key, int $lockTtl, \Closure $callback) use (&$ttl, &$timeout): mixed { $ttl = $lockTtl; - return $callback(new InspectingLock(new MemoryLock($key, $this->heldLocks), $timeout)); + return $callback(new InspectingLock(new MemoryLock($key, $this->heldLocks), function (float $acquireTimeout) use (&$timeout): void { + $timeout = $acquireTimeout; + })); }, new NoTelemetry(), $this->log, @@ -96,7 +98,7 @@ final class LockTest extends TestCase $lock->run('keys', 'k1', fn () => null); $this->assertSame(5, $ttl); - $this->assertSame(0.0, $timeout); + $this->assertEqualsWithDelta(0.0, $timeout, PHP_FLOAT_EPSILON); } public function test_run_skips_on_contention(): void @@ -124,7 +126,9 @@ final class LockTest extends TestCase function (string $key, int $lockTtl, \Closure $callback) use (&$ttl, &$timeout): mixed { $ttl = $lockTtl; - return $callback(new InspectingLock(new MemoryLock($key, $this->heldLocks), $timeout)); + return $callback(new InspectingLock(new MemoryLock($key, $this->heldLocks), function (float $acquireTimeout) use (&$timeout): void { + $timeout = $acquireTimeout; + })); }, new NoTelemetry(), $this->log, @@ -135,7 +139,7 @@ final class LockTest extends TestCase $lock->runOrFail('keys', 'k1', fn () => null); $this->assertSame(10, $ttl); - $this->assertSame(3.0, $timeout); + $this->assertEqualsWithDelta(3.0, $timeout, PHP_FLOAT_EPSILON); } public function test_best_effort_backend_error_runs_callback_unlocked(): void @@ -370,7 +374,10 @@ final class LockTest extends TestCase ->with( 'projects', 'p1', - $this->callback(fn (Document $document): bool => $document->getAttribute('accessedAt') === 'now') + $this->callback(function (Document $document): bool { + $this->assertSame('now', $document->getAttribute('accessedAt')); + return true; + }) ) ->willReturn(new Document(['$id' => 'p1'])); @@ -385,7 +392,7 @@ final class LockTest extends TestCase public function test_platform_db_lock_try_run_uses_collection_key(): void { - $dbForPlatform = $this->createMock(Database::class); + $dbForPlatform = $this->createStub(Database::class); $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); $called = false; @@ -401,7 +408,7 @@ final class LockTest extends TestCase { $this->heldLocks[self::KEY_PREFIX.'keys:k1'] = true; - $dbForPlatform = $this->createMock(Database::class); + $dbForPlatform = $this->createStub(Database::class); $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); $called = false; @@ -444,7 +451,7 @@ final class LockTest extends TestCase $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); - $this->assertNull($platformDBLock->tryUpdateDocument('keys', 'k1', new Document(['accessedAt' => 'now']))); + $this->assertNotInstanceOf(\Utopia\Database\Document::class, $platformDBLock->tryUpdateDocument('keys', 'k1', new Document(['accessedAt' => 'now']))); } public function test_pool_checkout_exception_runs_callback_unlocked(): void @@ -507,7 +514,7 @@ final class LockTest extends TestCase $platformDBLock = new PlatformDBLock($lock, $dbForPlatform, new Authorization()); - $this->assertNull($platformDBLock->tryUpdateAttribute( + $this->assertNotInstanceOf(\Utopia\Database\Document::class, $platformDBLock->tryUpdateAttribute( 'projects', 'routed-project', 'accessedAt', @@ -575,13 +582,13 @@ final class InspectingLock implements UtopiaLock { public function __construct( private readonly UtopiaLock $lock, - private ?float &$timeout, + private readonly \Closure $inspectTimeout, ) { } public function acquire(float $timeout = 0.0): bool { - $this->timeout = $timeout; + ($this->inspectTimeout)($timeout); return $this->lock->acquire($timeout); } From 30055464645ed7c97b0f11f99723b0da4ce3327a Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 1 Jul 2026 17:59:16 +0100 Subject: [PATCH 16/24] Clean up API key update lock check --- app/controllers/shared/api.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 232d9a645c..1cee4ace7b 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -209,7 +209,11 @@ Http::init() } } - if (! $updates->isEmpty() && $platformDBLock->tryUpdateDocument('keys', $dbKey->getId(), $updates) instanceof Document) { + $updatedKey = $updates->isEmpty() + ? null + : $platformDBLock->tryUpdateDocument('keys', $dbKey->getId(), $updates); + + if ($updatedKey instanceof Document) { if (! empty($apiKey->getProjectId())) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); } elseif (! empty($apiKey->getUserId())) { From bb5928799d8511134c1fbfd9cb3b87859f30a613 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 1 Jul 2026 18:17:37 +0100 Subject: [PATCH 17/24] Clean up locking unit tests --- tests/unit/Locking/LockTest.php | 190 +++------------ tests/unit/Locking/PlatformDBLockTest.php | 220 ++++++++++++++++++ .../unit/Locking/RequestLockResourceTest.php | 77 ------ 3 files changed, 247 insertions(+), 240 deletions(-) create mode 100644 tests/unit/Locking/PlatformDBLockTest.php delete mode 100644 tests/unit/Locking/RequestLockResourceTest.php diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php index 20d003830d..3cb7567330 100644 --- a/tests/unit/Locking/LockTest.php +++ b/tests/unit/Locking/LockTest.php @@ -6,17 +6,18 @@ namespace Tests\Unit\Locking; use Appwrite\Extend\Exception; use Appwrite\Locking\Lock; -use Appwrite\Locking\PlatformDBLock; use PHPUnit\Framework\TestCase; use Utopia\Config\Config; -use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Validator\Authorization; use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Lock\Lock as UtopiaLock; use Utopia\Logger\Log; use Utopia\Telemetry\Adapter\None as NoTelemetry; +if (! \class_exists(\RedisException::class)) { + \class_alias(LockingRedisException::class, 'RedisException'); +} + final class LockTest extends TestCase { /** @@ -64,7 +65,7 @@ final class LockTest extends TestCase return fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new MemoryLock($key, $this->heldLocks)); } - public function test_run_uses_per_document_key_and_invokes_callback(): void + public function testRunUsesPerDocumentKeyAndInvokesCallback(): void { $called = false; $lock = $this->makeLock(); @@ -76,7 +77,7 @@ final class LockTest extends TestCase $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); } - public function test_run_uses_short_try_once_lock_window(): void + public function testRunUsesShortTryOnceLockWindow(): void { $ttl = null; $timeout = null; @@ -101,7 +102,7 @@ final class LockTest extends TestCase $this->assertEqualsWithDelta(0.0, $timeout, PHP_FLOAT_EPSILON); } - public function test_run_skips_on_contention(): void + public function testRunSkipsOnContention(): void { $key = self::KEY_PREFIX.'keys:k1'; $this->heldLocks[$key] = true; @@ -117,7 +118,7 @@ final class LockTest extends TestCase $this->assertArrayHasKey($key, $this->heldLocks); } - public function test_run_or_fail_uses_short_http_lock_ttl_and_acquire_timeout(): void + public function testRunOrFailUsesShortHttpLockTtlAndAcquireTimeout(): void { $ttl = null; $timeout = null; @@ -142,12 +143,8 @@ final class LockTest extends TestCase $this->assertEqualsWithDelta(3.0, $timeout, PHP_FLOAT_EPSILON); } - public function test_best_effort_backend_error_runs_callback_unlocked(): void + public function testBestEffortBackendErrorRunsCallbackUnlocked(): void { - if (! \class_exists(\RedisException::class)) { - $this->markTestSkipped('Redis extension is required to simulate RedisException.'); - } - $lock = new Lock( fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingAcquireLock(new \RedisException('redis unavailable'))), new NoTelemetry(), @@ -164,12 +161,8 @@ final class LockTest extends TestCase $this->assertTrue($called); } - public function test_run_or_fail_backend_error_runs_callback_unlocked(): void + public function testRunOrFailBackendErrorRunsCallbackUnlocked(): void { - if (! \class_exists(\RedisException::class)) { - $this->markTestSkipped('Redis extension is required to simulate RedisException.'); - } - $lock = new Lock( fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingAcquireLock(new \RedisException('redis unavailable'))), new NoTelemetry(), @@ -186,12 +179,8 @@ final class LockTest extends TestCase $this->assertTrue($called); } - public function test_best_effort_pool_checkout_error_runs_callback_unlocked(): void + public function testBestEffortPoolCheckoutErrorRunsCallbackUnlocked(): void { - if (! \class_exists(\RedisException::class)) { - $this->markTestSkipped('Redis extension is required to simulate RedisException.'); - } - $lock = new Lock( fn (string $key, int $ttl, \Closure $callback): mixed => throw new \RedisException('pool unavailable'), new NoTelemetry(), @@ -208,24 +197,16 @@ final class LockTest extends TestCase $this->assertTrue($called); } - public function test_callback_redis_exception_is_not_swallowed(): void + public function testCallbackRedisExceptionIsNotSwallowed(): void { - if (! \class_exists(\RedisException::class)) { - $this->markTestSkipped('Redis extension is required to simulate RedisException.'); - } - $lock = $this->makeLock(); $this->expectException(\RedisException::class); $lock->run('keys', 'k1', fn () => throw new \RedisException('callback failed')); } - public function test_release_error_after_callback_is_logged_but_not_thrown(): void + public function testReleaseErrorAfterCallbackIsLoggedButNotThrown(): void { - if (! \class_exists(\RedisException::class)) { - $this->markTestSkipped('Redis extension is required to simulate RedisException.'); - } - $lock = new Lock( fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingReleaseLock(new MemoryLock($key, $this->heldLocks), new \RedisException('release failed'))), new NoTelemetry(), @@ -237,7 +218,7 @@ final class LockTest extends TestCase $this->assertSame('ok', $lock->runOrFail('keys', 'k1', fn () => 'ok')); } - public function test_run_or_fail_throws_on_contention(): void + public function testRunOrFailThrowsOnContention(): void { $key = self::KEY_PREFIX.'projects:p1'; $this->heldLocks[$key] = true; @@ -253,7 +234,7 @@ final class LockTest extends TestCase } } - public function test_run_or_fail_returns_callback_value_when_uncontended(): void + public function testRunOrFailReturnsCallbackValueWhenUncontended(): void { $lock = $this->makeLock(); $result = $lock->runOrFail('projects', 'p1', fn () => 'ok'); @@ -262,7 +243,7 @@ final class LockTest extends TestCase $this->assertArrayNotHasKey(self::KEY_PREFIX.'projects:p1', $this->heldLocks); } - public function test_with_key_uses_raw_key(): void + public function testWithKeyUsesRawKey(): void { $custom = 'lock:test:custom-key'; $called = false; @@ -275,7 +256,7 @@ final class LockTest extends TestCase $this->assertArrayNotHasKey($custom, $this->heldLocks); } - public function test_try_with_key_skips_on_contention(): void + public function testTryWithKeySkipsOnContention(): void { $custom = 'lock:test:contended'; $this->heldLocks[$custom] = true; @@ -289,7 +270,7 @@ final class LockTest extends TestCase $this->assertFalse($called); } - public function test_with_key_throws_on_contention(): void + public function testWithKeyThrowsOnContention(): void { $custom = 'lock:test:contended'; $this->heldLocks[$custom] = true; @@ -299,7 +280,7 @@ final class LockTest extends TestCase $lock->withKey($custom, fn () => null, ttl: 5, waitTimeout: 0.1); } - public function test_disabled_mode_runs_callback_unlocked(): void + public function testDisabledModeRunsCallbackUnlocked(): void { $previous = \getenv('_APP_LOCKING_ENABLED'); \putenv('_APP_LOCKING_ENABLED=disabled'); @@ -320,7 +301,7 @@ final class LockTest extends TestCase } } - public function test_project_without_sequence_falls_back_to_unknown(): void + public function testProjectWithoutSequenceFallsBackToUnknown(): void { $emptyProject = new Document(); $lock = new Lock( @@ -344,7 +325,7 @@ final class LockTest extends TestCase unset($this->heldLocks[$key]); } - public function test_key_for_project_uses_given_project_sequence(): void + public function testKeyForProjectUsesGivenProjectSequence(): void { $lock = new Lock( $this->withLock(), @@ -365,96 +346,7 @@ final class LockTest extends TestCase ); } - public function test_platform_db_lock_updates_attribute_under_attribute_key(): void - { - $dbForPlatform = $this->createMock(Database::class); - $dbForPlatform - ->expects($this->once()) - ->method('updateDocument') - ->with( - 'projects', - 'p1', - $this->callback(function (Document $document): bool { - $this->assertSame('now', $document->getAttribute('accessedAt')); - return true; - }) - ) - ->willReturn(new Document(['$id' => 'p1'])); - - $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); - - $this->assertSame( - 'p1', - $platformDBLock->tryUpdateAttribute('projects', 'p1', 'accessedAt', 'now')?->getId() - ); - $this->assertArrayNotHasKey(self::KEY_PREFIX.'projects:p1:accessedAt', $this->heldLocks); - } - - public function test_platform_db_lock_try_run_uses_collection_key(): void - { - $dbForPlatform = $this->createStub(Database::class); - $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); - - $called = false; - $platformDBLock->tryRun('keys', 'k1', function () use (&$called): void { - $called = true; - }); - - $this->assertTrue($called); - $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); - } - - public function test_platform_db_lock_try_run_skips_on_contention(): void - { - $this->heldLocks[self::KEY_PREFIX.'keys:k1'] = true; - - $dbForPlatform = $this->createStub(Database::class); - $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); - - $called = false; - $platformDBLock->tryRun('keys', 'k1', function () use (&$called): void { - $called = true; - }); - - $this->assertFalse($called); - } - - public function test_platform_db_lock_updates_document_under_document_key(): void - { - $updates = new Document(['accessedAt' => 'now', 'sdks' => ['php']]); - - $dbForPlatform = $this->createMock(Database::class); - $dbForPlatform - ->expects($this->once()) - ->method('updateDocument') - ->with('keys', 'k1', $updates) - ->willReturn(new Document(['$id' => 'k1'])); - - $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); - - $this->assertSame( - 'k1', - $platformDBLock->tryUpdateDocument('keys', 'k1', $updates)?->getId() - ); - // Document-level lock has no attribute suffix and is released after the write. - $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); - } - - public function test_platform_db_lock_skips_document_update_on_contention(): void - { - $this->heldLocks[self::KEY_PREFIX.'keys:k1'] = true; - - $dbForPlatform = $this->createMock(Database::class); - $dbForPlatform - ->expects($this->never()) - ->method('updateDocument'); - - $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); - - $this->assertNotInstanceOf(\Utopia\Database\Document::class, $platformDBLock->tryUpdateDocument('keys', 'k1', new Document(['accessedAt' => 'now']))); - } - - public function test_pool_checkout_exception_runs_callback_unlocked(): void + public function testPoolCheckoutExceptionRunsCallbackUnlocked(): void { // Pool::pop() throws a bare \Exception on exhaustion, before the lock is // acquired. The wrapper must fail open and run the callback unlocked. @@ -474,7 +366,7 @@ final class LockTest extends TestCase $this->assertTrue($called); } - public function test_non_pool_exception_is_not_swallowed(): void + public function testNonPoolExceptionIsNotSwallowed(): void { // Only the literal \Exception (pool exhaustion) is treated as fail-open; // any subclass thrown while entering the lock must keep propagating. @@ -490,38 +382,6 @@ final class LockTest extends TestCase $lock->run('keys', 'k1', fn () => null); } - public function test_platform_db_lock_skips_routed_project_update_on_contention(): void - { - $lock = new Lock( - $this->withLock(), - new NoTelemetry(), - $this->log, - null, - new Document(), - ); - - $project = new Document([ - '$id' => 'routed-project', - '$sequence' => '84', - ]); - - $this->heldLocks['lock:platform:84:projects:routed-project:accessedAt'] = true; - - $dbForPlatform = $this->createMock(Database::class); - $dbForPlatform - ->expects($this->never()) - ->method('updateDocument'); - - $platformDBLock = new PlatformDBLock($lock, $dbForPlatform, new Authorization()); - - $this->assertNotInstanceOf(\Utopia\Database\Document::class, $platformDBLock->tryUpdateAttribute( - 'projects', - 'routed-project', - 'accessedAt', - 'now', - $project - )); - } } final class MemoryLock implements UtopiaLock @@ -681,3 +541,7 @@ final class ThrowingReleaseLock implements UtopiaLock } } } + +final class LockingRedisException extends \Exception +{ +} diff --git a/tests/unit/Locking/PlatformDBLockTest.php b/tests/unit/Locking/PlatformDBLockTest.php new file mode 100644 index 0000000000..7e310afe4f --- /dev/null +++ b/tests/unit/Locking/PlatformDBLockTest.php @@ -0,0 +1,220 @@ + + */ + private array $heldLocks = []; + + private Log $log; + + private const PROJECT_SEQUENCE = '42'; + + private const KEY_PREFIX = 'lock:platform:'.self::PROJECT_SEQUENCE.':'; + + protected function setUp(): void + { + Config::setParam('errors', require __DIR__.'/../../../app/config/errors.php'); + + $this->heldLocks = []; + $this->log = new Log(); + } + + private function makeLock(?Document $project = null): Lock + { + return new Lock( + fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new PlatformDBLockMemoryLock($key, $this->heldLocks)), + new NoTelemetry(), + $this->log, + null, + $project ?? new Document([ + '$id' => 'test-project', + '$sequence' => self::PROJECT_SEQUENCE, + ]), + ); + } + + public function testTryUpdateAttributeUsesAttributeKey(): void + { + $dbForPlatform = $this->createMock(Database::class); + $dbForPlatform + ->expects($this->once()) + ->method('updateDocument') + ->with( + 'projects', + 'p1', + $this->callback(function (Document $document): bool { + $this->assertSame('now', $document->getAttribute('accessedAt')); + return true; + }) + ) + ->willReturn(new Document(['$id' => 'p1'])); + + $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); + + $this->assertSame( + 'p1', + $platformDBLock->tryUpdateAttribute('projects', 'p1', 'accessedAt', 'now')?->getId() + ); + $this->assertArrayNotHasKey(self::KEY_PREFIX.'projects:p1:accessedAt', $this->heldLocks); + } + + public function testTryRunUsesDocumentKey(): void + { + $dbForPlatform = $this->createStub(Database::class); + $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); + + $called = false; + $platformDBLock->tryRun('keys', 'k1', function () use (&$called): void { + $called = true; + }); + + $this->assertTrue($called); + $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); + } + + public function testTryRunSkipsOnContention(): void + { + $this->heldLocks[self::KEY_PREFIX.'keys:k1'] = true; + + $dbForPlatform = $this->createStub(Database::class); + $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); + + $called = false; + $platformDBLock->tryRun('keys', 'k1', function () use (&$called): void { + $called = true; + }); + + $this->assertFalse($called); + } + + public function testTryUpdateDocumentUsesDocumentKey(): void + { + $updates = new Document(['accessedAt' => 'now', 'sdks' => ['php']]); + + $dbForPlatform = $this->createMock(Database::class); + $dbForPlatform + ->expects($this->once()) + ->method('updateDocument') + ->with('keys', 'k1', $updates) + ->willReturn(new Document(['$id' => 'k1'])); + + $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); + + $this->assertSame( + 'k1', + $platformDBLock->tryUpdateDocument('keys', 'k1', $updates)?->getId() + ); + $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); + } + + public function testTryUpdateDocumentSkipsOnContention(): void + { + $this->heldLocks[self::KEY_PREFIX.'keys:k1'] = true; + + $dbForPlatform = $this->createMock(Database::class); + $dbForPlatform + ->expects($this->never()) + ->method('updateDocument'); + + $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); + + $this->assertNotInstanceOf(Document::class, $platformDBLock->tryUpdateDocument('keys', 'k1', new Document(['accessedAt' => 'now']))); + } + + public function testTryUpdateAttributeCanUseExplicitProjectScope(): void + { + $project = new Document([ + '$id' => 'routed-project', + '$sequence' => '84', + ]); + + $this->heldLocks['lock:platform:84:projects:routed-project:accessedAt'] = true; + + $dbForPlatform = $this->createMock(Database::class); + $dbForPlatform + ->expects($this->never()) + ->method('updateDocument'); + + $platformDBLock = new PlatformDBLock($this->makeLock(new Document()), $dbForPlatform, new Authorization()); + + $this->assertNotInstanceOf(Document::class, $platformDBLock->tryUpdateAttribute( + 'projects', + 'routed-project', + 'accessedAt', + 'now', + $project + )); + } +} + +final class PlatformDBLockMemoryLock implements UtopiaLock +{ + private bool $acquired = false; + + /** + * @param array $heldLocks + */ + public function __construct( + private readonly string $key, + private array &$heldLocks, + ) { + } + + public function acquire(float $timeout = 0.0): bool + { + return $this->tryAcquire(); + } + + public function tryAcquire(): bool + { + if (isset($this->heldLocks[$this->key])) { + return false; + } + + $this->heldLocks[$this->key] = true; + $this->acquired = true; + + return true; + } + + public function release(): void + { + if (! $this->acquired) { + return; + } + + unset($this->heldLocks[$this->key]); + $this->acquired = false; + } + + public function withLock(callable $callback, float $timeout = 0.0): mixed + { + if (! $this->acquire($timeout)) { + throw new LockContention("Failed to acquire distributed lock: {$this->key}"); + } + + try { + return $callback(); + } finally { + $this->release(); + } + } +} diff --git a/tests/unit/Locking/RequestLockResourceTest.php b/tests/unit/Locking/RequestLockResourceTest.php deleted file mode 100644 index 56f8d7e29c..0000000000 --- a/tests/unit/Locking/RequestLockResourceTest.php +++ /dev/null @@ -1,77 +0,0 @@ -markTestSkipped('Redis extension is required to exercise the registered lock resource.'); - } - - $redis = $this->createMock(\Redis::class); - $redis->method('set')->willReturn(true); - $redis->method('eval')->willReturn(1); - - $pool = new Pool(new Stack(), 'lock', 1, fn () => $redis); - $pools = new RecordingPoolGroup(); - $pools->add($pool); - - $context = new Container(); - $registrar = require __DIR__.'/../../../app/init/resources/request.php'; - $registrar($context); - - $context - ->set('pools', fn () => $pools) - ->set('telemetry', fn () => new NoTelemetry()) - ->set('logger', fn () => null) - ->set('project', fn () => new Document([ - '$id' => 'test-project', - '$sequence' => '42', - ])) - ->set('dbForPlatform', fn () => null); - - $lock = $context->get('lock'); - - $this->assertInstanceOf(Lock::class, $lock); - - $ran = false; - $lock->withKey( - $lock->key('projects', 'p1', 'accessedAt'), - function () use (&$ran): void { - $ran = true; - }, - target: 'projects' - ); - - $this->assertTrue($ran); - $this->assertSame(['lock'], $pools->names); - } -} - -final class RecordingPoolGroup extends Group -{ - /** - * @var list - */ - public array $names = []; - - public function get(string $name): Pool - { - $this->names[] = $name; - - return parent::get($name); - } -} From 3d85a75aa01cc10d0f77c816ee381f317313042e Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 1 Jul 2026 19:17:17 +0100 Subject: [PATCH 18/24] Clean up lock error reporting --- app/init/resources/request.php | 5 +- src/Appwrite/Locking/Lock.php | 77 +++++--------------- src/Appwrite/Locking/PlatformDBLock.php | 5 -- tests/unit/Locking/LockTest.php | 89 +++++++++++++++++------ tests/unit/Locking/PlatformDBLockTest.php | 5 -- 5 files changed, 88 insertions(+), 93 deletions(-) diff --git a/app/init/resources/request.php b/app/init/resources/request.php index dcb99e3e0d..3c942b006e 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -69,17 +69,16 @@ return function (Container $context): void { $context->set('logger', fn ($register) => $register->get('logger'), ['register']); - $context->set('lock', function (Group $pools, Telemetry $telemetry, Log $log, ?Logger $logger, Document $project): Lock { + $context->set('lock', function (Group $pools, Telemetry $telemetry, ?Logger $logger, Document $project): Lock { return new Lock( fn (string $key, int $ttl, Closure $callback): mixed => $pools->get('lock')->use( fn (\Redis $redis): mixed => $callback(new DistributedLock($redis, $key, $ttl)) ), $telemetry, - $log, $logger, $project ); - }, ['pools', 'telemetry', 'log', 'logger', 'project']); + }, ['pools', 'telemetry', 'logger', 'project']); $context->set('platformDBLock', fn (Lock $lock, Database $dbForPlatform, Authorization $authorization): PlatformDBLock => new PlatformDBLock( $lock, diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php index a7422337a0..3d7c839b09 100644 --- a/src/Appwrite/Locking/Lock.php +++ b/src/Appwrite/Locking/Lock.php @@ -52,7 +52,6 @@ final class Lock public function __construct( Closure $useLock, Telemetry $telemetry, - private readonly Log $log, private readonly ?Logger $logger, Document $project, ) { @@ -63,33 +62,16 @@ final class Lock $this->projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; } - /** - * Try-once lock around an arbitrary callback for a platform document. - * Idempotent metadata writes should not make HTTP requests wait behind - * another pod doing the same update. - */ public function run(string $collection, string $id, Closure $fn): void { $this->tryWithKey($this->key($collection, $id), $fn, target: $collection); } - /** - * Distributed lock around a platform document that returns the callback result. - */ public function runOrFail(string $collection, string $id, Closure $fn): mixed { return $this->execute($this->key($collection, $id), $collection, $fn); } - /** - * Generic lock primitive with full control over key, TTL, contention - * behavior, and wait timeout. Escape hatch for non-platform keys (cache, - * queue, edge) and for unusual TTL/timeout requirements. - * - * Caller may pass `target` for telemetry; otherwise it's extracted by - * position from the key (best-effort for keys following the standard - * `lock:platform:{project}:{target}:...` shape). - */ public function withKey( string $key, Closure $fn, @@ -107,10 +89,6 @@ final class Lock ); } - /** - * Try-once lock around a callback. On contention, skip the callback and - * return null; this is for best-effort timestamp/access metadata writes. - */ public function tryWithKey( string $key, Closure $fn, @@ -153,11 +131,8 @@ final class Lock return $fn(); } catch (\Exception $e) { - // Pool::pop() throws a bare \Exception (not \RedisException) when it - // can't hand out a connection within the retry/sync-timeout budget. - // That happens before the callback runs, so fail open. Match the - // literal class only: business exceptions (Appwrite\Extend\Exception, - // etc.) also extend \Exception and must keep propagating. + // Pool exhaustion throws a bare \Exception before lock acquisition. + // Only that exact class is fail-open; subclasses must propagate. if (get_class($e) !== \Exception::class) { throw $e; } @@ -196,7 +171,6 @@ final class Lock return null; } $this->attempts->add(1, ['outcome' => self::OUTCOME_CONTENDED, ...$labels]); - // No custom message: the lock key embeds collection and document ID. throw new Exception(Exception::GENERAL_RESOURCE_LOCKED); } @@ -217,11 +191,6 @@ final class Lock } } - /** - * Best-effort target extraction for telemetry. Assumes the standard - * `lock:platform:{project}:{target}:...` shape. For non-platform keys - * passed via withKey(), callers should pass `target` explicitly. - */ private static function inferTargetFromKey(string $key): string { $parts = explode(':', $key, 5); @@ -229,19 +198,11 @@ final class Lock return $parts[3] ?? 'unknown'; } - /** - * Shared platform lock key builder. Exposed so higher-level decorators can - * keep the platform key shape centralized while adding narrower scopes. - */ public function key(string $collection, string $id, ?string $attribute = null): string { return self::buildKey($this->projectInternalId, $collection, $id, $attribute); } - /** - * Build a key for a project document that was resolved after this request - * lock was constructed, such as router/custom-domain resolution. - */ public function keyForProject(Document $project, string $collection, string $id, ?string $attribute = null): string { $sequence = $project->getSequence(); @@ -258,8 +219,7 @@ final class Lock } /** - * Rate-limited to one push per REPORT_RATE_LIMIT_SECONDS per (action, target) - * so a sustained backend outage doesn't flood Sentry across the pod fleet. + * Rate-limit backend/release reports so outages don't flood Sentry. */ private function reportError(string $action, string $key, string $target, Throwable $e): void { @@ -276,26 +236,27 @@ final class Lock } self::$lastReportAt[$bucket] = $now; - $this->log->setNamespace('http'); - $this->log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); - $this->log->setVersion(APP_VERSION_STABLE); - $this->log->setType(Log::TYPE_WARNING); - $this->log->setMessage('Distributed lock '.$action.': '.$e->getMessage()); - $this->log->setAction("lock.{$action}"); - $this->log->setEnvironment(System::getEnv('_APP_ENV', 'development') === 'production' + $log = new Log(); + $log->setNamespace('http'); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); + $log->setVersion(APP_VERSION_STABLE); + $log->setType(Log::TYPE_WARNING); + $log->setMessage('Distributed lock '.$action.': '.$e->getMessage()); + $log->setAction("lock.{$action}"); + $log->setEnvironment(System::getEnv('_APP_ENV', 'development') === 'production' ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - $this->log->addTag('lock.target', $target); - $this->log->addTag('lock.project', $this->projectInternalId); + $log->addTag('lock.target', $target); + $log->addTag('lock.project', $this->projectInternalId); // Strip trailing document ID to keep aggregator cardinality bounded. - $this->log->addTag('lock.key_pattern', preg_replace('/:[^:]+$/', ':*', $key)); - $this->log->addTag('code', $e->getCode()); - $this->log->addExtra('file', $e->getFile()); - $this->log->addExtra('line', $e->getLine()); - $this->log->addExtra('trace', $e->getTraceAsString()); + $log->addTag('lock.key_pattern', preg_replace('/:[^:]+$/', ':*', $key)); + $log->addTag('code', $e->getCode()); + $log->addExtra('file', $e->getFile()); + $log->addExtra('line', $e->getLine()); + $log->addExtra('trace', $e->getTraceAsString()); try { - $this->logger->addLog($this->log); + $this->logger->addLog($log); } catch (Throwable) { } } diff --git a/src/Appwrite/Locking/PlatformDBLock.php b/src/Appwrite/Locking/PlatformDBLock.php index a43cbec842..a66b5a93cb 100644 --- a/src/Appwrite/Locking/PlatformDBLock.php +++ b/src/Appwrite/Locking/PlatformDBLock.php @@ -51,11 +51,6 @@ final class PlatformDBLock ); } - /** - * Document-level counterpart to tryUpdateAttribute() for sparse updates - * that change several fields at once. Locks at the document key (no - * attribute suffix) so the whole update is mutually exclusive. - */ public function tryUpdateDocument( string $collection, string $id, diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php index 3cb7567330..cba3281061 100644 --- a/tests/unit/Locking/LockTest.php +++ b/tests/unit/Locking/LockTest.php @@ -11,13 +11,19 @@ use Utopia\Config\Config; use Utopia\Database\Document; use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Lock\Lock as UtopiaLock; +use Utopia\Logger\Adapter as LoggerAdapter; use Utopia\Logger\Log; +use Utopia\Logger\Logger; use Utopia\Telemetry\Adapter\None as NoTelemetry; if (! \class_exists(\RedisException::class)) { \class_alias(LockingRedisException::class, 'RedisException'); } +if (! \defined('APP_VERSION_STABLE')) { + \define('APP_VERSION_STABLE', 'test'); +} + final class LockTest extends TestCase { /** @@ -27,12 +33,6 @@ final class LockTest extends TestCase private Document $project; - private Log $log; - - /** - * Project sequence used for every test; lock keys are scoped under it - * so cleanup is bounded. - */ private const PROJECT_SEQUENCE = '42'; private const KEY_PREFIX = 'lock:platform:'.self::PROJECT_SEQUENCE.':'; @@ -46,7 +46,6 @@ final class LockTest extends TestCase '$id' => 'test-project', '$sequence' => self::PROJECT_SEQUENCE, ]); - $this->log = new Log(); } private function makeLock(): Lock @@ -54,7 +53,6 @@ final class LockTest extends TestCase return new Lock( $this->withLock(), new NoTelemetry(), - $this->log, null, $this->project, ); @@ -91,7 +89,6 @@ final class LockTest extends TestCase })); }, new NoTelemetry(), - $this->log, null, $this->project, ); @@ -132,7 +129,6 @@ final class LockTest extends TestCase })); }, new NoTelemetry(), - $this->log, null, $this->project, ); @@ -148,7 +144,6 @@ final class LockTest extends TestCase $lock = new Lock( fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingAcquireLock(new \RedisException('redis unavailable'))), new NoTelemetry(), - $this->log, null, $this->project, ); @@ -161,12 +156,32 @@ final class LockTest extends TestCase $this->assertTrue($called); } + public function testBackendErrorReportDoesNotMutateRequestLog(): void + { + $requestLog = new Log(); + $adapter = new RecordingLoggerAdapter(); + $logger = new Logger($adapter); + + $lock = new Lock( + fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingAcquireLock(new \RedisException('redis unavailable'))), + new NoTelemetry(), + $logger, + $this->project, + ); + + $lock->run('keys', 'k1', fn () => null); + + $this->assertCount(1, $adapter->logs); + $this->assertNotSame($requestLog, $adapter->logs[0]); + $this->assertSame([], $requestLog->getTags()); + $this->assertSame([], $requestLog->getExtra()); + } + public function testRunOrFailBackendErrorRunsCallbackUnlocked(): void { $lock = new Lock( fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingAcquireLock(new \RedisException('redis unavailable'))), new NoTelemetry(), - $this->log, null, $this->project, ); @@ -184,7 +199,6 @@ final class LockTest extends TestCase $lock = new Lock( fn (string $key, int $ttl, \Closure $callback): mixed => throw new \RedisException('pool unavailable'), new NoTelemetry(), - $this->log, null, $this->project, ); @@ -210,7 +224,6 @@ final class LockTest extends TestCase $lock = new Lock( fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingReleaseLock(new MemoryLock($key, $this->heldLocks), new \RedisException('release failed'))), new NoTelemetry(), - $this->log, null, $this->project, ); @@ -307,7 +320,6 @@ final class LockTest extends TestCase $lock = new Lock( $this->withLock(), new NoTelemetry(), - $this->log, null, $emptyProject, ); @@ -330,7 +342,6 @@ final class LockTest extends TestCase $lock = new Lock( $this->withLock(), new NoTelemetry(), - $this->log, null, new Document(), ); @@ -348,12 +359,9 @@ final class LockTest extends TestCase public function testPoolCheckoutExceptionRunsCallbackUnlocked(): void { - // Pool::pop() throws a bare \Exception on exhaustion, before the lock is - // acquired. The wrapper must fail open and run the callback unlocked. $lock = new Lock( fn (string $key, int $ttl, \Closure $callback): mixed => throw new \Exception('Pool \'lock\' is empty'), new NoTelemetry(), - $this->log, null, $this->project, ); @@ -368,12 +376,9 @@ final class LockTest extends TestCase public function testNonPoolExceptionIsNotSwallowed(): void { - // Only the literal \Exception (pool exhaustion) is treated as fail-open; - // any subclass thrown while entering the lock must keep propagating. $lock = new Lock( fn (string $key, int $ttl, \Closure $callback): mixed => throw new \RuntimeException('unexpected'), new NoTelemetry(), - $this->log, null, $this->project, ); @@ -545,3 +550,43 @@ final class ThrowingReleaseLock implements UtopiaLock final class LockingRedisException extends \Exception { } + +final class RecordingLoggerAdapter extends LoggerAdapter +{ + /** + * @var list + */ + public array $logs = []; + + public static function getName(): string + { + return 'recording'; + } + + public function push(Log $log): int + { + $this->logs[] = $log; + + return 200; + } + + public function getSupportedTypes(): array + { + return [ + Log::TYPE_WARNING, + ]; + } + + public function getSupportedEnvironments(): array + { + return [ + Log::ENVIRONMENT_PRODUCTION, + Log::ENVIRONMENT_STAGING, + ]; + } + + public function getSupportedBreadcrumbTypes(): array + { + return []; + } +} diff --git a/tests/unit/Locking/PlatformDBLockTest.php b/tests/unit/Locking/PlatformDBLockTest.php index 7e310afe4f..df54ee5fda 100644 --- a/tests/unit/Locking/PlatformDBLockTest.php +++ b/tests/unit/Locking/PlatformDBLockTest.php @@ -13,7 +13,6 @@ use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; use Utopia\Lock\Exception\Contention as LockContention; use Utopia\Lock\Lock as UtopiaLock; -use Utopia\Logger\Log; use Utopia\Telemetry\Adapter\None as NoTelemetry; final class PlatformDBLockTest extends TestCase @@ -23,8 +22,6 @@ final class PlatformDBLockTest extends TestCase */ private array $heldLocks = []; - private Log $log; - private const PROJECT_SEQUENCE = '42'; private const KEY_PREFIX = 'lock:platform:'.self::PROJECT_SEQUENCE.':'; @@ -34,7 +31,6 @@ final class PlatformDBLockTest extends TestCase Config::setParam('errors', require __DIR__.'/../../../app/config/errors.php'); $this->heldLocks = []; - $this->log = new Log(); } private function makeLock(?Document $project = null): Lock @@ -42,7 +38,6 @@ final class PlatformDBLockTest extends TestCase return new Lock( fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new PlatformDBLockMemoryLock($key, $this->heldLocks)), new NoTelemetry(), - $this->log, null, $project ?? new Document([ '$id' => 'test-project', From 016f47c45672d1f0268d9c6ddca9a14709184dbf Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Fri, 3 Jul 2026 08:56:16 +0100 Subject: [PATCH 19/24] Remove platform DB lock wrapper --- app/controllers/general.php | 38 ++-- app/controllers/shared/api.php | 32 +++- app/init/resources/request.php | 7 - src/Appwrite/Locking/PlatformDBLock.php | 72 -------- tests/unit/Locking/LockTest.php | 10 + tests/unit/Locking/PlatformDBLockTest.php | 215 ---------------------- 6 files changed, 59 insertions(+), 315 deletions(-) delete mode 100644 src/Appwrite/Locking/PlatformDBLock.php delete mode 100644 tests/unit/Locking/PlatformDBLockTest.php diff --git a/app/controllers/general.php b/app/controllers/general.php index 93692b72a2..335a1ad568 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -12,7 +12,7 @@ use Appwrite\Event\Message\Delete as DeleteMessage; use Appwrite\Event\Publisher\Certificate; use Appwrite\Event\Publisher\Delete as DeletePublisher; use Appwrite\Extend\Exception as AppwriteException; -use Appwrite\Locking\PlatformDBLock; +use Appwrite\Locking\Lock; use Appwrite\Network\Cors; use Appwrite\Platform\Appwrite; use Appwrite\SDK\AuthType; @@ -77,7 +77,7 @@ use Utopia\Validator\Text; Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) +function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { $host = $request->getHostname(); if (!empty($previewHostname)) { @@ -146,7 +146,15 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S if (!$project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $platformDBLock->tryUpdateAttribute('projects', $project->getId(), 'accessedAt', DateTime::now(), $project); + $lock->tryWithKey( + $lock->keyForProject($project, 'projects', $project->getId(), 'accessedAt'), + fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( + 'projects', + $project->getId(), + new Document(['accessedAt' => DateTime::now()]) + )), + target: 'projects' + ); } /** @@ -840,9 +848,9 @@ Http::init() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('platformDBLock') + ->inject('lock') ->inject('params') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock, array $params) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock, array $params) { /* * Appwrite Router */ @@ -850,7 +858,7 @@ Http::init() $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!\in_array($hostname, $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1153,15 +1161,15 @@ Http::options() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('platformDBLock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { + ->inject('lock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { /* * Appwrite Router */ $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1556,14 +1564,14 @@ Http::get('/robots.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('platformDBLock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { + ->inject('lock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1591,14 +1599,14 @@ Http::get('/humans.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->inject('platformDBLock') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, PlatformDBLock $platformDBLock) { + ->inject('lock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $platformDBLock)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { $utopia->match($request)?->route->label('router', true); } } diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 1cee4ace7b..61a8fb3a9e 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -16,7 +16,7 @@ use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\EventProcessor; -use Appwrite\Locking\PlatformDBLock; +use Appwrite\Locking\Lock; use Appwrite\Platform\Modules\Storage\Config\CacheControl; use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl; use Appwrite\Reference\Renderer; @@ -61,10 +61,10 @@ Http::init() ->inject('team') ->inject('apiKey') ->inject('authorization') - ->inject('platformDBLock') + ->inject('lock') ->inject('impersonatorUser') ->inject('targetUser') - ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, PlatformDBLock $platformDBLock, Document $impersonatorUser, User $targetUser) { + ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Lock $lock, Document $impersonatorUser, User $targetUser) { /** * Handle user authentication and session validation. @@ -211,7 +211,11 @@ Http::init() $updatedKey = $updates->isEmpty() ? null - : $platformDBLock->tryUpdateDocument('keys', $dbKey->getId(), $updates); + : $lock->tryWithKey( + $lock->key('keys', $dbKey->getId()), + fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)), + target: 'keys' + ); if ($updatedKey instanceof Document) { if (! empty($apiKey->getProjectId())) { @@ -376,7 +380,15 @@ Http::init() if ($project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $platformDBLock->tryUpdateAttribute('projects', $project->getId(), 'accessedAt', DateTime::now()); + $lock->tryWithKey( + $lock->key('projects', $project->getId(), 'accessedAt'), + fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( + 'projects', + $project->getId(), + new Document(['accessedAt' => DateTime::now()]) + )), + target: 'projects' + ); } } @@ -392,7 +404,15 @@ Http::init() 'accessedAt' => $user->getAttribute('accessedAt') ])); } else { - $platformDBLock->tryUpdateAttribute('users', $user->getId(), 'accessedAt', $user->getAttribute('accessedAt')); + $lock->tryWithKey( + $lock->key('users', $user->getId(), 'accessedAt'), + fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( + 'users', + $user->getId(), + new Document(['accessedAt' => $user->getAttribute('accessedAt')]) + )), + target: 'users' + ); } } } diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 3c942b006e..461c6311b1 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -15,7 +15,6 @@ use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\GraphQL\Schema; use Appwrite\Locking\Lock; -use Appwrite\Locking\PlatformDBLock; use Appwrite\Network\Cors; use Appwrite\Network\Platform; use Appwrite\Network\Validator\Origin; @@ -80,12 +79,6 @@ return function (Container $context): void { ); }, ['pools', 'telemetry', 'logger', 'project']); - $context->set('platformDBLock', fn (Lock $lock, Database $dbForPlatform, Authorization $authorization): PlatformDBLock => new PlatformDBLock( - $lock, - $dbForPlatform, - $authorization - ), ['lock', 'dbForPlatform', 'authorization']); - $context->set('authorization', fn () => new Authorization(), []); $context->set('store', fn (): Store => new Store(), []); diff --git a/src/Appwrite/Locking/PlatformDBLock.php b/src/Appwrite/Locking/PlatformDBLock.php deleted file mode 100644 index a66b5a93cb..0000000000 --- a/src/Appwrite/Locking/PlatformDBLock.php +++ /dev/null @@ -1,72 +0,0 @@ -lock->key($collection, $id, $attribute) - : $this->lock->keyForProject($project, $collection, $id, $attribute); - - return $this->lock->tryWithKey($key, $callback, target: $collection); - } - - public function tryUpdateAttribute( - string $collection, - string $id, - string $attribute, - mixed $value, - ?Document $project = null, - ): ?Document { - /** @var Document|null */ - return $this->tryRun( - $collection, - $id, - fn () => $this->authorization->skip(fn () => $this->dbForPlatform->updateDocument( - $collection, - $id, - new Document([$attribute => $value]) - )), - $attribute, - $project - ); - } - - public function tryUpdateDocument( - string $collection, - string $id, - Document $updates, - ?Document $project = null, - ): ?Document { - /** @var Document|null */ - return $this->tryRun( - $collection, - $id, - fn () => $this->authorization->skip(fn () => $this->dbForPlatform->updateDocument( - $collection, - $id, - $updates - )), - project: $project - ); - } -} diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php index cba3281061..66b59839e2 100644 --- a/tests/unit/Locking/LockTest.php +++ b/tests/unit/Locking/LockTest.php @@ -357,6 +357,16 @@ final class LockTest extends TestCase ); } + public function testKeyUsesAttributeSuffix(): void + { + $lock = $this->makeLock(); + + $this->assertSame( + self::KEY_PREFIX.'projects:p1:accessedAt', + $lock->key('projects', 'p1', 'accessedAt') + ); + } + public function testPoolCheckoutExceptionRunsCallbackUnlocked(): void { $lock = new Lock( diff --git a/tests/unit/Locking/PlatformDBLockTest.php b/tests/unit/Locking/PlatformDBLockTest.php deleted file mode 100644 index df54ee5fda..0000000000 --- a/tests/unit/Locking/PlatformDBLockTest.php +++ /dev/null @@ -1,215 +0,0 @@ - - */ - private array $heldLocks = []; - - private const PROJECT_SEQUENCE = '42'; - - private const KEY_PREFIX = 'lock:platform:'.self::PROJECT_SEQUENCE.':'; - - protected function setUp(): void - { - Config::setParam('errors', require __DIR__.'/../../../app/config/errors.php'); - - $this->heldLocks = []; - } - - private function makeLock(?Document $project = null): Lock - { - return new Lock( - fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new PlatformDBLockMemoryLock($key, $this->heldLocks)), - new NoTelemetry(), - null, - $project ?? new Document([ - '$id' => 'test-project', - '$sequence' => self::PROJECT_SEQUENCE, - ]), - ); - } - - public function testTryUpdateAttributeUsesAttributeKey(): void - { - $dbForPlatform = $this->createMock(Database::class); - $dbForPlatform - ->expects($this->once()) - ->method('updateDocument') - ->with( - 'projects', - 'p1', - $this->callback(function (Document $document): bool { - $this->assertSame('now', $document->getAttribute('accessedAt')); - return true; - }) - ) - ->willReturn(new Document(['$id' => 'p1'])); - - $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); - - $this->assertSame( - 'p1', - $platformDBLock->tryUpdateAttribute('projects', 'p1', 'accessedAt', 'now')?->getId() - ); - $this->assertArrayNotHasKey(self::KEY_PREFIX.'projects:p1:accessedAt', $this->heldLocks); - } - - public function testTryRunUsesDocumentKey(): void - { - $dbForPlatform = $this->createStub(Database::class); - $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); - - $called = false; - $platformDBLock->tryRun('keys', 'k1', function () use (&$called): void { - $called = true; - }); - - $this->assertTrue($called); - $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); - } - - public function testTryRunSkipsOnContention(): void - { - $this->heldLocks[self::KEY_PREFIX.'keys:k1'] = true; - - $dbForPlatform = $this->createStub(Database::class); - $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); - - $called = false; - $platformDBLock->tryRun('keys', 'k1', function () use (&$called): void { - $called = true; - }); - - $this->assertFalse($called); - } - - public function testTryUpdateDocumentUsesDocumentKey(): void - { - $updates = new Document(['accessedAt' => 'now', 'sdks' => ['php']]); - - $dbForPlatform = $this->createMock(Database::class); - $dbForPlatform - ->expects($this->once()) - ->method('updateDocument') - ->with('keys', 'k1', $updates) - ->willReturn(new Document(['$id' => 'k1'])); - - $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); - - $this->assertSame( - 'k1', - $platformDBLock->tryUpdateDocument('keys', 'k1', $updates)?->getId() - ); - $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); - } - - public function testTryUpdateDocumentSkipsOnContention(): void - { - $this->heldLocks[self::KEY_PREFIX.'keys:k1'] = true; - - $dbForPlatform = $this->createMock(Database::class); - $dbForPlatform - ->expects($this->never()) - ->method('updateDocument'); - - $platformDBLock = new PlatformDBLock($this->makeLock(), $dbForPlatform, new Authorization()); - - $this->assertNotInstanceOf(Document::class, $platformDBLock->tryUpdateDocument('keys', 'k1', new Document(['accessedAt' => 'now']))); - } - - public function testTryUpdateAttributeCanUseExplicitProjectScope(): void - { - $project = new Document([ - '$id' => 'routed-project', - '$sequence' => '84', - ]); - - $this->heldLocks['lock:platform:84:projects:routed-project:accessedAt'] = true; - - $dbForPlatform = $this->createMock(Database::class); - $dbForPlatform - ->expects($this->never()) - ->method('updateDocument'); - - $platformDBLock = new PlatformDBLock($this->makeLock(new Document()), $dbForPlatform, new Authorization()); - - $this->assertNotInstanceOf(Document::class, $platformDBLock->tryUpdateAttribute( - 'projects', - 'routed-project', - 'accessedAt', - 'now', - $project - )); - } -} - -final class PlatformDBLockMemoryLock implements UtopiaLock -{ - private bool $acquired = false; - - /** - * @param array $heldLocks - */ - public function __construct( - private readonly string $key, - private array &$heldLocks, - ) { - } - - public function acquire(float $timeout = 0.0): bool - { - return $this->tryAcquire(); - } - - public function tryAcquire(): bool - { - if (isset($this->heldLocks[$this->key])) { - return false; - } - - $this->heldLocks[$this->key] = true; - $this->acquired = true; - - return true; - } - - public function release(): void - { - if (! $this->acquired) { - return; - } - - unset($this->heldLocks[$this->key]); - $this->acquired = false; - } - - public function withLock(callable $callback, float $timeout = 0.0): mixed - { - if (! $this->acquire($timeout)) { - throw new LockContention("Failed to acquire distributed lock: {$this->key}"); - } - - try { - return $callback(); - } finally { - $this->release(); - } - } -} From f93e2626641fc8f3cd608e41bf0a3b0e892e1643 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Fri, 3 Jul 2026 13:41:47 +0100 Subject: [PATCH 20/24] Simplify distributed lock keys --- app/controllers/general.php | 4 +- app/controllers/shared/api.php | 17 ++++- src/Appwrite/Locking/Lock.php | 45 ++---------- tests/unit/Locking/LockTest.php | 121 +++++++++----------------------- 4 files changed, 55 insertions(+), 132 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 335a1ad568..cd7010e413 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -146,8 +146,10 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S if (!$project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $sequence = $project->getSequence(); + $projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; $lock->tryWithKey( - $lock->keyForProject($project, 'projects', $project->getId(), 'accessedAt'), + 'lock:platform:'.$projectInternalId.':projects:'.$project->getId().':accessedAt', fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( 'projects', $project->getId(), diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 61a8fb3a9e..ccc8e73753 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -163,24 +163,31 @@ Http::init() // For standard keys, update last accessed time if (\in_array($apiKey->getType(), [API_KEY_STANDARD, API_KEY_ORGANIZATION, API_KEY_ACCOUNT])) { $dbKey = null; + $keyOwnerInternalId = 'unknown'; if (! empty($apiKey->getProjectId())) { $dbKey = $project->find( key: 'secret', find: $request->getHeaderLine('x-appwrite-key', ''), subject: 'keys' ); + $sequence = $project->getSequence(); + $keyOwnerInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; } elseif (! empty($apiKey->getUserId())) { $dbKey = $user->find( key: 'secret', find: $request->getHeaderLine('x-appwrite-key', ''), subject: 'keys' ); + $sequence = $user->getSequence(); + $keyOwnerInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; } elseif (! empty($apiKey->getTeamId())) { $dbKey = $team->find( key: 'secret', find: $request->getHeaderLine('x-appwrite-key', ''), subject: 'keys' ); + $sequence = $team->getSequence(); + $keyOwnerInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; } if (!$dbKey) { @@ -212,7 +219,7 @@ Http::init() $updatedKey = $updates->isEmpty() ? null : $lock->tryWithKey( - $lock->key('keys', $dbKey->getId()), + 'lock:platform:'.$keyOwnerInternalId.':keys:'.$dbKey->getId(), fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)), target: 'keys' ); @@ -380,8 +387,10 @@ Http::init() if ($project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $sequence = $project->getSequence(); + $projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; $lock->tryWithKey( - $lock->key('projects', $project->getId(), 'accessedAt'), + 'lock:platform:'.$projectInternalId.':projects:'.$project->getId().':accessedAt', fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( 'projects', $project->getId(), @@ -404,8 +413,10 @@ Http::init() 'accessedAt' => $user->getAttribute('accessedAt') ])); } else { + $sequence = $user->getSequence(); + $userInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; $lock->tryWithKey( - $lock->key('users', $user->getId(), 'accessedAt'), + 'lock:platform:'.$userInternalId.':users:'.$user->getId().':accessedAt', fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( 'users', $user->getId(), diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php index 3d7c839b09..938f5262a7 100644 --- a/src/Appwrite/Locking/Lock.php +++ b/src/Appwrite/Locking/Lock.php @@ -62,26 +62,16 @@ final class Lock $this->projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; } - public function run(string $collection, string $id, Closure $fn): void - { - $this->tryWithKey($this->key($collection, $id), $fn, target: $collection); - } - - public function runOrFail(string $collection, string $id, Closure $fn): mixed - { - return $this->execute($this->key($collection, $id), $collection, $fn); - } - public function withKey( string $key, Closure $fn, + string $target, int $ttl = self::FAIL_TTL_SECONDS, float $waitTimeout = self::FAIL_WAIT_SECONDS, - ?string $target = null, ): mixed { return $this->execute( $key, - $target ?? self::inferTargetFromKey($key), + $target, $fn, ttl: $ttl, waitTimeout: $waitTimeout, @@ -92,12 +82,12 @@ final class Lock public function tryWithKey( string $key, Closure $fn, + string $target, int $ttl = self::SKIP_TTL_SECONDS, - ?string $target = null, ): mixed { return $this->execute( $key, - $target ?? self::inferTargetFromKey($key), + $target, $fn, ttl: $ttl, waitTimeout: self::SKIP_WAIT_SECONDS, @@ -191,33 +181,6 @@ final class Lock } } - private static function inferTargetFromKey(string $key): string - { - $parts = explode(':', $key, 5); - - return $parts[3] ?? 'unknown'; - } - - public function key(string $collection, string $id, ?string $attribute = null): string - { - return self::buildKey($this->projectInternalId, $collection, $id, $attribute); - } - - public function keyForProject(Document $project, string $collection, string $id, ?string $attribute = null): string - { - $sequence = $project->getSequence(); - $projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; - - return self::buildKey($projectInternalId, $collection, $id, $attribute); - } - - private static function buildKey(string $projectInternalId, string $collection, string $id, ?string $attribute = null): string - { - $key = "lock:platform:{$projectInternalId}:{$collection}:{$id}"; - - return $attribute === null ? $key : "{$key}:{$attribute}"; - } - /** * Rate-limit backend/release reports so outages don't flood Sentry. */ diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php index 66b59839e2..8409f4eb56 100644 --- a/tests/unit/Locking/LockTest.php +++ b/tests/unit/Locking/LockTest.php @@ -63,19 +63,19 @@ final class LockTest extends TestCase return fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new MemoryLock($key, $this->heldLocks)); } - public function testRunUsesPerDocumentKeyAndInvokesCallback(): void + public function testTryWithKeyUsesGivenKeyAndInvokesCallback(): void { $called = false; $lock = $this->makeLock(); - $lock->run('keys', 'k1', function () use (&$called) { + $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', function () use (&$called) { $called = true; - }); + }, target: 'keys'); $this->assertTrue($called); $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks); } - public function testRunUsesShortTryOnceLockWindow(): void + public function testTryWithKeyUsesShortTryOnceLockWindow(): void { $ttl = null; $timeout = null; @@ -93,13 +93,13 @@ final class LockTest extends TestCase $this->project, ); - $lock->run('keys', 'k1', fn () => null); + $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', fn () => null, target: 'keys'); $this->assertSame(5, $ttl); $this->assertEqualsWithDelta(0.0, $timeout, PHP_FLOAT_EPSILON); } - public function testRunSkipsOnContention(): void + public function testTryWithKeySkipsOnContention(): void { $key = self::KEY_PREFIX.'keys:k1'; $this->heldLocks[$key] = true; @@ -107,15 +107,15 @@ final class LockTest extends TestCase $called = false; $lock = $this->makeLock(); - $lock->run('keys', 'k1', function () use (&$called) { + $lock->tryWithKey($key, function () use (&$called) { $called = true; - }); + }, target: 'keys'); $this->assertFalse($called); $this->assertArrayHasKey($key, $this->heldLocks); } - public function testRunOrFailUsesShortHttpLockTtlAndAcquireTimeout(): void + public function testWithKeyUsesShortHttpLockTtlAndAcquireTimeout(): void { $ttl = null; $timeout = null; @@ -133,7 +133,7 @@ final class LockTest extends TestCase $this->project, ); - $lock->runOrFail('keys', 'k1', fn () => null); + $lock->withKey(self::KEY_PREFIX.'keys:k1', fn () => null, target: 'keys'); $this->assertSame(10, $ttl); $this->assertEqualsWithDelta(3.0, $timeout, PHP_FLOAT_EPSILON); @@ -149,9 +149,9 @@ final class LockTest extends TestCase ); $called = false; - $lock->run('keys', 'k1', function () use (&$called) { + $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', function () use (&$called) { $called = true; - }); + }, target: 'keys'); $this->assertTrue($called); } @@ -169,7 +169,7 @@ final class LockTest extends TestCase $this->project, ); - $lock->run('keys', 'k1', fn () => null); + $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', fn () => null, target: 'keys'); $this->assertCount(1, $adapter->logs); $this->assertNotSame($requestLog, $adapter->logs[0]); @@ -177,7 +177,7 @@ final class LockTest extends TestCase $this->assertSame([], $requestLog->getExtra()); } - public function testRunOrFailBackendErrorRunsCallbackUnlocked(): void + public function testWithKeyBackendErrorRunsCallbackUnlocked(): void { $lock = new Lock( fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingAcquireLock(new \RedisException('redis unavailable'))), @@ -187,9 +187,9 @@ final class LockTest extends TestCase ); $called = false; - $lock->runOrFail('keys', 'k1', function () use (&$called) { + $lock->withKey(self::KEY_PREFIX.'keys:k1', function () use (&$called) { $called = true; - }); + }, target: 'keys'); $this->assertTrue($called); } @@ -204,9 +204,9 @@ final class LockTest extends TestCase ); $called = false; - $lock->run('keys', 'k1', function () use (&$called) { + $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', function () use (&$called) { $called = true; - }); + }, target: 'keys'); $this->assertTrue($called); } @@ -216,7 +216,7 @@ final class LockTest extends TestCase $lock = $this->makeLock(); $this->expectException(\RedisException::class); - $lock->run('keys', 'k1', fn () => throw new \RedisException('callback failed')); + $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', fn () => throw new \RedisException('callback failed'), target: 'keys'); } public function testReleaseErrorAfterCallbackIsLoggedButNotThrown(): void @@ -228,10 +228,10 @@ final class LockTest extends TestCase $this->project, ); - $this->assertSame('ok', $lock->runOrFail('keys', 'k1', fn () => 'ok')); + $this->assertSame('ok', $lock->withKey(self::KEY_PREFIX.'keys:k1', fn () => 'ok', target: 'keys')); } - public function testRunOrFailThrowsOnContention(): void + public function testWithKeyThrowsOnContention(): void { $key = self::KEY_PREFIX.'projects:p1'; $this->heldLocks[$key] = true; @@ -240,17 +240,17 @@ final class LockTest extends TestCase $this->expectException(Exception::class); try { - $lock->runOrFail('projects', 'p1', fn () => 'never-runs'); + $lock->withKey($key, fn () => 'never-runs', target: 'projects'); } catch (Exception $e) { $this->assertSame(Exception::GENERAL_RESOURCE_LOCKED, $e->getType()); throw $e; } } - public function testRunOrFailReturnsCallbackValueWhenUncontended(): void + public function testWithKeyReturnsCallbackValueWhenUncontended(): void { $lock = $this->makeLock(); - $result = $lock->runOrFail('projects', 'p1', fn () => 'ok'); + $result = $lock->withKey(self::KEY_PREFIX.'projects:p1', fn () => 'ok', target: 'projects'); $this->assertSame('ok', $result); $this->assertArrayNotHasKey(self::KEY_PREFIX.'projects:p1', $this->heldLocks); @@ -263,13 +263,13 @@ final class LockTest extends TestCase $lock = $this->makeLock(); $lock->withKey($custom, function () use (&$called) { $called = true; - }); + }, target: 'custom'); $this->assertTrue($called); $this->assertArrayNotHasKey($custom, $this->heldLocks); } - public function testTryWithKeySkipsOnContention(): void + public function testTryWithKeySkipsGivenKeyOnContention(): void { $custom = 'lock:test:contended'; $this->heldLocks[$custom] = true; @@ -278,19 +278,19 @@ final class LockTest extends TestCase $called = false; $lock->tryWithKey($custom, function () use (&$called): void { $called = true; - }, ttl: 5); + }, target: 'custom', ttl: 5); $this->assertFalse($called); } - public function testWithKeyThrowsOnContention(): void + public function testWithKeyThrowsGivenKeyOnContention(): void { $custom = 'lock:test:contended'; $this->heldLocks[$custom] = true; $lock = $this->makeLock(); $this->expectException(Exception::class); - $lock->withKey($custom, fn () => null, ttl: 5, waitTimeout: 0.1); + $lock->withKey($custom, fn () => null, target: 'custom', ttl: 5, waitTimeout: 0.1); } public function testDisabledModeRunsCallbackUnlocked(): void @@ -303,9 +303,9 @@ final class LockTest extends TestCase $called = false; $lock = $this->makeLock(); - $lock->run('keys', 'k1', function () use (&$called) { + $lock->tryWithKey($key, function () use (&$called) { $called = true; - }); + }, target: 'keys'); $this->assertTrue($called); $this->assertArrayHasKey($key, $this->heldLocks); @@ -314,59 +314,6 @@ final class LockTest extends TestCase } } - public function testProjectWithoutSequenceFallsBackToUnknown(): void - { - $emptyProject = new Document(); - $lock = new Lock( - $this->withLock(), - new NoTelemetry(), - null, - $emptyProject, - ); - - $key = 'lock:platform:unknown:keys:k1'; - $this->heldLocks[$key] = true; - - $called = false; - $lock->run('keys', 'k1', function () use (&$called) { - $called = true; - }); - - $this->assertFalse($called, 'Lock without project sequence should hash to the unknown bucket'); - - unset($this->heldLocks[$key]); - } - - public function testKeyForProjectUsesGivenProjectSequence(): void - { - $lock = new Lock( - $this->withLock(), - new NoTelemetry(), - null, - new Document(), - ); - - $project = new Document([ - '$id' => 'routed-project', - '$sequence' => '84', - ]); - - $this->assertSame( - 'lock:platform:84:projects:routed-project:accessedAt', - $lock->keyForProject($project, 'projects', 'routed-project', 'accessedAt') - ); - } - - public function testKeyUsesAttributeSuffix(): void - { - $lock = $this->makeLock(); - - $this->assertSame( - self::KEY_PREFIX.'projects:p1:accessedAt', - $lock->key('projects', 'p1', 'accessedAt') - ); - } - public function testPoolCheckoutExceptionRunsCallbackUnlocked(): void { $lock = new Lock( @@ -377,9 +324,9 @@ final class LockTest extends TestCase ); $called = false; - $lock->run('keys', 'k1', function () use (&$called) { + $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', function () use (&$called) { $called = true; - }); + }, target: 'keys'); $this->assertTrue($called); } @@ -394,7 +341,7 @@ final class LockTest extends TestCase ); $this->expectException(\RuntimeException::class); - $lock->run('keys', 'k1', fn () => null); + $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', fn () => null, target: 'keys'); } } From c372e29c83027f111622ad4413ac5d5db992e594 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Fri, 3 Jul 2026 14:01:10 +0100 Subject: [PATCH 21/24] Use document ids for lock fallback keys --- app/controllers/general.php | 2 +- app/controllers/shared/api.php | 12 ++++++------ src/Appwrite/Locking/Lock.php | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index cd7010e413..72ee159a3f 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -147,7 +147,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { $sequence = $project->getSequence(); - $projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; + $projectInternalId = (string) ($sequence ?: $project->getId()); $lock->tryWithKey( 'lock:platform:'.$projectInternalId.':projects:'.$project->getId().':accessedAt', fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index ccc8e73753..b4d6989f11 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -163,7 +163,7 @@ Http::init() // For standard keys, update last accessed time if (\in_array($apiKey->getType(), [API_KEY_STANDARD, API_KEY_ORGANIZATION, API_KEY_ACCOUNT])) { $dbKey = null; - $keyOwnerInternalId = 'unknown'; + $keyOwnerInternalId = ''; if (! empty($apiKey->getProjectId())) { $dbKey = $project->find( key: 'secret', @@ -171,7 +171,7 @@ Http::init() subject: 'keys' ); $sequence = $project->getSequence(); - $keyOwnerInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; + $keyOwnerInternalId = (string) ($sequence ?: $project->getId()); } elseif (! empty($apiKey->getUserId())) { $dbKey = $user->find( key: 'secret', @@ -179,7 +179,7 @@ Http::init() subject: 'keys' ); $sequence = $user->getSequence(); - $keyOwnerInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; + $keyOwnerInternalId = (string) ($sequence ?: $user->getId()); } elseif (! empty($apiKey->getTeamId())) { $dbKey = $team->find( key: 'secret', @@ -187,7 +187,7 @@ Http::init() subject: 'keys' ); $sequence = $team->getSequence(); - $keyOwnerInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; + $keyOwnerInternalId = (string) ($sequence ?: $team->getId()); } if (!$dbKey) { @@ -388,7 +388,7 @@ Http::init() $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { $sequence = $project->getSequence(); - $projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; + $projectInternalId = (string) ($sequence ?: $project->getId()); $lock->tryWithKey( 'lock:platform:'.$projectInternalId.':projects:'.$project->getId().':accessedAt', fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( @@ -414,7 +414,7 @@ Http::init() ])); } else { $sequence = $user->getSequence(); - $userInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; + $userInternalId = (string) ($sequence ?: $user->getId()); $lock->tryWithKey( 'lock:platform:'.$userInternalId.':users:'.$user->getId().':accessedAt', fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php index 938f5262a7..b0820b43c4 100644 --- a/src/Appwrite/Locking/Lock.php +++ b/src/Appwrite/Locking/Lock.php @@ -59,7 +59,7 @@ final class Lock $this->enabled = System::getEnv('_APP_LOCKING_ENABLED', 'enabled') !== 'disabled'; $this->attempts = $telemetry->createCounter('lock.attempts', null, 'Distributed lock acquire outcomes'); $sequence = $project->getSequence(); - $this->projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; + $this->projectInternalId = (string) ($sequence ?: $project->getId()); } public function withKey( From 9333afb8a596238b0f01ef15f76e03fcb09895c2 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Fri, 3 Jul 2026 14:08:02 +0100 Subject: [PATCH 22/24] Simplify lock fallback id assignments --- app/controllers/general.php | 3 +-- app/controllers/shared/api.php | 15 +++++---------- src/Appwrite/Locking/Lock.php | 3 +-- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 72ee159a3f..5d4684591a 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -146,8 +146,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S if (!$project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $sequence = $project->getSequence(); - $projectInternalId = (string) ($sequence ?: $project->getId()); + $projectInternalId = (string) ($project->getSequence() ?: $project->getId()); $lock->tryWithKey( 'lock:platform:'.$projectInternalId.':projects:'.$project->getId().':accessedAt', fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index b4d6989f11..52261b9378 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -170,24 +170,21 @@ Http::init() find: $request->getHeaderLine('x-appwrite-key', ''), subject: 'keys' ); - $sequence = $project->getSequence(); - $keyOwnerInternalId = (string) ($sequence ?: $project->getId()); + $keyOwnerInternalId = (string) ($project->getSequence() ?: $project->getId()); } elseif (! empty($apiKey->getUserId())) { $dbKey = $user->find( key: 'secret', find: $request->getHeaderLine('x-appwrite-key', ''), subject: 'keys' ); - $sequence = $user->getSequence(); - $keyOwnerInternalId = (string) ($sequence ?: $user->getId()); + $keyOwnerInternalId = (string) ($user->getSequence() ?: $user->getId()); } elseif (! empty($apiKey->getTeamId())) { $dbKey = $team->find( key: 'secret', find: $request->getHeaderLine('x-appwrite-key', ''), subject: 'keys' ); - $sequence = $team->getSequence(); - $keyOwnerInternalId = (string) ($sequence ?: $team->getId()); + $keyOwnerInternalId = (string) ($team->getSequence() ?: $team->getId()); } if (!$dbKey) { @@ -387,8 +384,7 @@ Http::init() if ($project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $sequence = $project->getSequence(); - $projectInternalId = (string) ($sequence ?: $project->getId()); + $projectInternalId = (string) ($project->getSequence() ?: $project->getId()); $lock->tryWithKey( 'lock:platform:'.$projectInternalId.':projects:'.$project->getId().':accessedAt', fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( @@ -413,8 +409,7 @@ Http::init() 'accessedAt' => $user->getAttribute('accessedAt') ])); } else { - $sequence = $user->getSequence(); - $userInternalId = (string) ($sequence ?: $user->getId()); + $userInternalId = (string) ($user->getSequence() ?: $user->getId()); $lock->tryWithKey( 'lock:platform:'.$userInternalId.':users:'.$user->getId().':accessedAt', fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument( diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php index b0820b43c4..01447640b9 100644 --- a/src/Appwrite/Locking/Lock.php +++ b/src/Appwrite/Locking/Lock.php @@ -58,8 +58,7 @@ final class Lock $this->useLock = $useLock; $this->enabled = System::getEnv('_APP_LOCKING_ENABLED', 'enabled') !== 'disabled'; $this->attempts = $telemetry->createCounter('lock.attempts', null, 'Distributed lock acquire outcomes'); - $sequence = $project->getSequence(); - $this->projectInternalId = (string) ($sequence ?: $project->getId()); + $this->projectInternalId = (string) ($project->getSequence() ?: $project->getId()); } public function withKey( From 498dbf0f1006a358e922cb765cc50c1414c3f4af Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 6 Jul 2026 11:04:40 +0530 Subject: [PATCH 23/24] Add RunsOn spot interruption retry policy --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80c855f389..71264d7efe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,8 +189,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const runner4 = `runs-on=${context.runId}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache`; - const runner8 = `runs-on=${context.runId}/runner=8cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache`; + const runner4 = `runs-on=${context.runId}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache`; + const runner8 = `runs-on=${context.runId}/runner=8cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache`; const services = [ { name: 'Account', runner: runner4, functional: true, processes: null, timeout: 8 }, @@ -257,7 +257,7 @@ jobs: build: name: Build - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+s3-cache+ecr-cache + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+s3-cache+ecr-cache timeout-minutes: 30 steps: - name: Checkout repository @@ -291,7 +291,7 @@ jobs: unit: name: Tests / Unit - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache timeout-minutes: 15 needs: build permissions: @@ -356,7 +356,7 @@ jobs: e2e_general: name: Tests / E2E / General - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache timeout-minutes: 15 needs: build permissions: @@ -554,7 +554,7 @@ jobs: e2e_abuse: name: Tests / E2E / Abuse (${{ matrix.mode }}) - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache timeout-minutes: 15 needs: [build, matrix] permissions: @@ -633,7 +633,7 @@ jobs: e2e_screenshots: name: Tests / E2E / Screenshots (${{ matrix.mode }}) - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache timeout-minutes: 15 needs: [build, matrix] permissions: @@ -718,7 +718,7 @@ jobs: benchmark: name: Benchmark if: github.event_name == 'pull_request' || github.event_name == 'push' - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache timeout-minutes: 20 needs: build permissions: From d296993b398a353045fe7960c49b53b748ce8f11 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Mon, 6 Jul 2026 11:01:12 +0100 Subject: [PATCH 24/24] Remove lock callback exception wrapper --- src/Appwrite/Locking/Lock.php | 39 ++++++++++++++------------------- tests/unit/Locking/LockTest.php | 2 +- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php index 01447640b9..f808ce404c 100644 --- a/src/Appwrite/Locking/Lock.php +++ b/src/Appwrite/Locking/Lock.php @@ -108,18 +108,26 @@ final class Lock $labels = ['target' => $target, 'project' => $this->projectInternalId]; + $callbackException = false; + try { - return ($this->useLock)($key, $ttl, function (UtopiaLock $lock) use ($fn, $key, $labels, $skipOnContention, $target, $waitTimeout): mixed { - return $this->executeWithLock($lock, $key, $target, $labels, $fn, $waitTimeout, $skipOnContention); + return ($this->useLock)($key, $ttl, function (UtopiaLock $lock) use ($fn, $key, $labels, $skipOnContention, $target, $waitTimeout, &$callbackException): mixed { + return $this->executeWithLock($lock, $key, $target, $labels, $fn, $waitTimeout, $skipOnContention, $callbackException); }); - } catch (CallbackRedisException $e) { - throw $e->getRedisException(); } catch (\RedisException $e) { + if ($callbackException) { + throw $e; + } + $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); return $fn(); } catch (\Exception $e) { + if ($callbackException) { + throw $e; + } + // Pool exhaustion throws a bare \Exception before lock acquisition. // Only that exact class is fail-open; subclasses must propagate. if (get_class($e) !== \Exception::class) { @@ -143,6 +151,7 @@ final class Lock Closure $fn, float $waitTimeout, bool $skipOnContention, + bool &$callbackException, ): mixed { try { $acquired = $lock->acquire($waitTimeout); @@ -150,6 +159,7 @@ final class Lock $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); + $callbackException = true; return $fn(); } @@ -164,12 +174,9 @@ final class Lock } $this->attempts->add(1, ['outcome' => self::OUTCOME_ACQUIRED, ...$labels]); + $callbackException = true; try { - try { - return $fn(); - } catch (\RedisException $e) { - throw new CallbackRedisException($e); - } + return $fn(); } finally { try { $lock->release(); @@ -223,17 +230,3 @@ final class Lock } } } - -final class CallbackRedisException extends \RuntimeException -{ - public function __construct(\RedisException $previous) - { - parent::__construct($previous->getMessage(), $previous->getCode(), $previous); - } - - public function getRedisException(): \RedisException - { - /** @var \RedisException */ - return parent::getPrevious(); - } -} diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php index 8409f4eb56..d4e9a9b512 100644 --- a/tests/unit/Locking/LockTest.php +++ b/tests/unit/Locking/LockTest.php @@ -211,7 +211,7 @@ final class LockTest extends TestCase $this->assertTrue($called); } - public function testCallbackRedisExceptionIsNotSwallowed(): void + public function testRedisExceptionThrownByCallbackIsNotTreatedAsBackendError(): void { $lock = $this->makeLock();