diff --git a/app/config/errors.php b/app/config/errors.php index 4a6f08d432..0e5f6608b9 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -54,6 +54,11 @@ return [ 'description' => 'Rate limit for the current endpoint has been exceeded. Please try again after some time.', 'code' => 429, ], + Exception::GENERAL_RESOURCE_LOCKED => [ + 'name' => Exception::GENERAL_RESOURCE_LOCKED, + 'description' => 'The requested resource is currently being modified by another request. Please retry after a brief delay.', + 'code' => 409, + ], Exception::GENERAL_SMTP_DISABLED => [ 'name' => Exception::GENERAL_SMTP_DISABLED, 'description' => 'SMTP is disabled on your Appwrite instance. You can learn more about setting up SMTP in our docs.', diff --git a/app/config/variables.php b/app/config/variables.php index 90df9b4518..23d4eaf315 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -34,6 +34,15 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_LOCKING_ENABLED', + 'description' => 'Enable distributed locking for platform writes. Locks coordinate concurrent updates across API pods so that read-modify-write operations on shared documents do not lose updates. By default, set to \'enabled\'. Set to \'disabled\' as an emergency kill switch — locks become no-ops (fail-open) and concurrent writes will race.', + 'introduction' => '1.9.3', + 'default' => 'enabled', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_OPTIONS_FORCE_HTTPS', 'description' => 'Allows you to force HTTPS connection to your API. This feature redirects any HTTP call to HTTPS and adds the \'Strict-Transport-Security\' header to all HTTP responses. By default, set to \'enabled\'. To disable, set to \'disabled\'. This feature will work only when your ports are set to default 80 and 443, and you have set up wildcard certificates with DNS challenge.', diff --git a/app/controllers/general.php b/app/controllers/general.php index 94caf4760d..9e26af3198 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -12,6 +12,7 @@ use Appwrite\Event\Message\Delete as DeleteMessage; use Appwrite\Event\Publisher\Certificate; use Appwrite\Event\Publisher\Delete as DeletePublisher; use Appwrite\Extend\Exception as AppwriteException; +use Appwrite\Locking\Lock; use Appwrite\Network\Cors; use Appwrite\Platform\Appwrite; use Appwrite\SDK\Method; @@ -75,7 +76,7 @@ use Utopia\Validator\Text; Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE); -function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) +function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { $host = $request->getHostname(); if (!empty($previewHostname)) { @@ -144,9 +145,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S if (!$project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ - 'accessedAt' => DateTime::now() - ]))); + $lock->set('projects', $project->getId(), 'accessedAt', DateTime::now()); } /** @@ -840,8 +839,9 @@ Http::init() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') + ->inject('lock') ->inject('params') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, array $params) { + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, Reader $geodb, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock, array $params) { /* * Appwrite Router */ @@ -849,7 +849,7 @@ Http::init() $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!\in_array($hostname, $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1152,14 +1152,15 @@ Http::options() ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { + ->inject('lock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { /* * Appwrite Router */ $platformHostnames = $platform['hostnames'] ?? []; // Only run Router when external domain if (!in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1554,13 +1555,14 @@ Http::get('/robots.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { + ->inject('lock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/robots.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { $utopia->match($request)?->route->label('router', true); } } @@ -1588,13 +1590,14 @@ Http::get('/humans.txt') ->inject('authorization') ->inject('publisherForDeletes') ->inject('executionsRetentionCount') - ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) { + ->inject('lock') + ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, Reader $geodb, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) { $platformHostnames = $platform['hostnames'] ?? []; if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) { $template = new View(__DIR__ . '/../views/general/humans.phtml'); $response->text($template->render(false)); } else { - if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) { + if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geodb, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) { $utopia->match($request)?->route->label('router', true); } } diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 4b5f7170ff..e11ba433ee 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -16,6 +16,7 @@ use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\EventProcessor; +use Appwrite\Locking\Lock; use Appwrite\Platform\Modules\Storage\Config\CacheControl; use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl; use Appwrite\SDK\Method; @@ -99,7 +100,8 @@ Http::init() ->inject('team') ->inject('apiKey') ->inject('authorization') - ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization) { + ->inject('lock') + ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Lock $lock) { /** * Handle user authentication and session validation. @@ -240,20 +242,22 @@ Http::init() $sdks[] = $sdk; $updates->setAttribute('sdks', $sdks); - $updates->setAttribute('accessedAt', Datetime::now()); + $updates->setAttribute('accessedAt', DateTime::now()); } } if (! $updates->isEmpty()) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); + $lock->run('keys', $dbKey->getId(), function () use ($dbForPlatform, $dbKey, $updates, $apiKey, $project, $user, $team) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); - if (! empty($apiKey->getProjectId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); - } elseif (! empty($apiKey->getUserId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); - } elseif (! empty($apiKey->getTeamId())) { - $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); - } + if (! empty($apiKey->getProjectId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); + } elseif (! empty($apiKey->getUserId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); + } elseif (! empty($apiKey->getTeamId())) { + $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); + } + }); } $userClone = clone $user; @@ -384,9 +388,7 @@ Http::init() if ($project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ - 'accessedAt' => DateTime::now() - ]))); + $lock->set('projects', $project->getId(), 'accessedAt', DateTime::now()); } } @@ -403,9 +405,7 @@ Http::init() 'accessedAt' => $user->getAttribute('accessedAt') ])); } else { - $authorization->skip(fn () => $dbForPlatform->updateDocument('users', $user->getId(), new Document([ - 'accessedAt' => $user->getAttribute('accessedAt') - ]))); + $lock->set('users', $user->getId(), 'accessedAt', $user->getAttribute('accessedAt')); } } } diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 982cd13088..fa480034f0 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -14,6 +14,7 @@ use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\GraphQL\Schema; +use Appwrite\Locking\Lock; use Appwrite\Network\Cors; use Appwrite\Network\Platform; use Appwrite\Network\Validator\Origin; @@ -44,6 +45,7 @@ use Utopia\Domains\Domain; use Utopia\Http\Http; use Utopia\Locale\Locale; use Utopia\Logger\Log; +use Utopia\Logger\Logger; use Utopia\Pools\Group; use Utopia\Queue\Publisher; use Utopia\Queue\Queue; @@ -65,6 +67,10 @@ return function (Container $context): void { $context->set('logger', fn ($register) => $register->get('logger'), ['register']); + $context->set('lock', function (\Redis $redis, Telemetry $telemetry, Database $dbForPlatform, Authorization $authorization, Log $log, ?Logger $logger, Document $project): Lock { + return new Lock($redis, $telemetry, $dbForPlatform, $authorization, $log, $logger, $project); + }, ['redis', 'telemetry', 'dbForPlatform', 'authorization', 'log', 'logger', 'project']); + $context->set('authorization', fn () => new Authorization(), []); $context->set('store', fn (): Store => new Store(), []); diff --git a/composer.json b/composer.json index e1ab3c62ab..3d8f3b4bf1 100644 --- a/composer.json +++ b/composer.json @@ -100,6 +100,12 @@ "league/csv": "9.14.*", "enshrined/svg-sanitize": "0.22.*" }, + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/utopia-php/lock" + } + ], "require-dev": { "ext-fileinfo": "*", "appwrite/sdk-generator": "*", diff --git a/composer.lock b/composer.lock index 7014501161..80ff9265c9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e922d0ba4d0c7d12cc24cff52b552803", + "content-hash": "1cc46b6b8e657c29ee41fd975a2033f8", "packages": [ { "name": "adhocore/jwt", diff --git a/docker-compose.yml b/docker-compose.yml index 212b0f1f47..3a954cebd1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -212,6 +212,7 @@ services: - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - _APP_LOGGING_CONFIG + - _APP_LOCKING_ENABLED - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 6506127c59..afe6f9fe21 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -45,6 +45,7 @@ class Exception extends \Exception public const string GENERAL_SERVICE_DISABLED = 'general_service_disabled'; public const string GENERAL_UNAUTHORIZED_SCOPE = 'general_unauthorized_scope'; public const string GENERAL_RATE_LIMIT_EXCEEDED = 'general_rate_limit_exceeded'; + public const string GENERAL_RESOURCE_LOCKED = 'general_resource_locked'; public const string GENERAL_SMTP_DISABLED = 'general_smtp_disabled'; public const string GENERAL_PHONE_DISABLED = 'general_phone_disabled'; public const string GENERAL_ARGUMENT_INVALID = 'general_argument_invalid'; diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php new file mode 100644 index 0000000000..9ef6d70c03 --- /dev/null +++ b/src/Appwrite/Locking/Lock.php @@ -0,0 +1,235 @@ + */ + private static array $lastReportAt = []; + + public function __construct( + private readonly \Redis $redis, + Telemetry $telemetry, + private readonly Database $dbForPlatform, + private readonly Authorization $authorization, + private readonly Log $log, + private readonly ?Logger $logger, + Document $project, + ) { + $this->enabled = System::getEnv('_APP_LOCKING_ENABLED', 'enabled') !== 'disabled'; + $this->attempts = $telemetry->createCounter('lock.attempts', null, 'Distributed lock acquire outcomes'); + $sequence = $project->getSequence(); + $this->projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown'; + } + + /** + * Throttled single-attribute write under a per-attribute skip-on-contention + * lock with authorization bypass. For idempotent timestamp-style updates + * (accessedAt, mcpAccessedAt) where regional pods writing the same value + * would thrash the platform DB. + */ + public function set( + string $collection, + string $id, + string $attribute, + string $value, + ): void { + $key = "lock:platform:{$this->projectInternalId}:{$collection}:{$id}:{$attribute}"; + $this->execute($key, $collection, function () use ($collection, $id, $attribute, $value) { + $this->authorization->skip(fn () => $this->dbForPlatform->updateDocument( + $collection, + $id, + new Document([$attribute => $value]) + )); + }); + } + + /** + * Skip-on-contention lock around an arbitrary callback for a platform + * document. For idempotent multi-statement writes that don't fit `set`. + */ + public function run(string $collection, string $id, Closure $fn): void + { + $key = "lock:platform:{$this->projectInternalId}:{$collection}:{$id}"; + $this->execute($key, $collection, $fn); + } + + /** + * Block-then-409 lock around an arbitrary callback for a platform document. + * For read-modify-write endpoints where silently dropping a concurrent + * request would lose user data. + */ + public function runOrFail(string $collection, string $id, Closure $fn): mixed + { + $key = "lock:platform:{$this->projectInternalId}:{$collection}:{$id}"; + + return $this->execute($key, $collection, $fn, ttl: self::FAIL_TTL_SECONDS, orFail: true); + } + + /** + * Generic lock primitive with full control over key, TTL, contention + * behavior, and wait timeout. Escape hatch for non-platform keys + * (cache, queue, edge) and for unusual TTL/timeout requirements. + * + * Caller may pass `target` for telemetry; otherwise it's extracted by + * position from the key (best-effort for keys following the standard + * `lock:platform:{project}:{target}:...` shape). + */ + public function withKey( + string $key, + Closure $fn, + int $ttl = self::SKIP_TTL_SECONDS, + bool $orFail = false, + float $waitTimeout = self::FAIL_WAIT_SECONDS, + ?string $target = null, + ): mixed { + return $this->execute( + $key, + $target ?? self::inferTargetFromKey($key), + $fn, + ttl: $ttl, + orFail: $orFail, + waitTimeout: $waitTimeout, + ); + } + + private function execute( + string $key, + string $target, + Closure $fn, + int $ttl = self::SKIP_TTL_SECONDS, + bool $orFail = false, + float $waitTimeout = self::FAIL_WAIT_SECONDS, + ): mixed { + if (! $this->enabled) { + return $fn(); + } + + $lock = new DistributedLock($this->redis, $key, $ttl); + $labels = ['target' => $target, 'project' => $this->projectInternalId]; + + try { + $acquired = $orFail ? $lock->acquire($waitTimeout) : $lock->tryAcquire(); + } catch (\RedisException $e) { + $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]); + $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e); + + return $fn(); + } + + if (! $acquired) { + if ($orFail) { + $this->attempts->add(1, ['outcome' => self::OUTCOME_CONTENDED, ...$labels]); + // No custom message — the lock key embeds collection + document id. + throw new Exception(Exception::GENERAL_RESOURCE_LOCKED); + } + $this->attempts->add(1, ['outcome' => self::OUTCOME_SKIPPED, ...$labels]); + + return null; + } + + $this->attempts->add(1, ['outcome' => self::OUTCOME_ACQUIRED, ...$labels]); + try { + return $fn(); + } finally { + try { + $lock->release(); + } catch (Throwable $e) { + $this->attempts->add(1, ['outcome' => self::OUTCOME_RELEASE_ERROR, ...$labels]); + $this->reportError(self::OUTCOME_RELEASE_ERROR, $key, $target, $e); + } + } + } + + /** + * Best-effort target extraction for telemetry. Assumes the standard + * `lock:platform:{project}:{target}:...` shape. For non-platform keys + * passed via withKey(), callers should pass `target` explicitly. + */ + private static function inferTargetFromKey(string $key): string + { + $parts = explode(':', $key, 5); + + return $parts[3] ?? 'unknown'; + } + + /** + * Rate-limited to one push per REPORT_RATE_LIMIT_SECONDS per (action, target) + * so a sustained backend outage doesn't flood Sentry across the pod fleet. + */ + private function reportError(string $action, string $key, string $target, Throwable $e): void + { + Console::warning("Lock {$action} for {$key}: {$e->getMessage()}"); + + if ($this->logger === null) { + return; + } + + $bucket = $action.':'.$target; + $now = time(); + if ((self::$lastReportAt[$bucket] ?? 0) + self::REPORT_RATE_LIMIT_SECONDS > $now) { + return; + } + self::$lastReportAt[$bucket] = $now; + + $this->log->setNamespace('http'); + $this->log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); + $this->log->setVersion(APP_VERSION_STABLE); + $this->log->setType(Log::TYPE_WARNING); + $this->log->setMessage('Distributed lock '.$action.': '.$e->getMessage()); + $this->log->setAction("lock.{$action}"); + $this->log->setEnvironment(System::getEnv('_APP_ENV', 'development') === 'production' + ? Log::ENVIRONMENT_PRODUCTION + : Log::ENVIRONMENT_STAGING); + $this->log->addTag('lock.target', $target); + $this->log->addTag('lock.project', $this->projectInternalId); + // Strip trailing document ID to keep aggregator cardinality bounded. + $this->log->addTag('lock.key_pattern', preg_replace('/:[^:]+$/', ':*', $key)); + $this->log->addTag('code', $e->getCode()); + $this->log->addExtra('file', $e->getFile()); + $this->log->addExtra('line', $e->getLine()); + $this->log->addExtra('trace', $e->getTraceAsString()); + + try { + $this->logger->addLog($this->log); + } catch (Throwable) { + } + } +} diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php new file mode 100644 index 0000000000..b6fcfba4d2 --- /dev/null +++ b/tests/unit/Locking/LockTest.php @@ -0,0 +1,259 @@ +redis = new Redis(); + $this->redis->connect($host, $port, 1.0); + + $this->project = new Document([ + '$id' => 'test-project', + '$sequence' => self::PROJECT_SEQUENCE, + ]); + $this->authorization = new Authorization(); + $this->log = new Log(); + + $this->cleanupKeys(); + } + + protected function tearDown(): void + { + if (isset($this->redis) && $this->redis->isConnected()) { + $this->cleanupKeys(); + } + } + + private function cleanupKeys(): void + { + foreach ($this->redis->keys(self::KEY_PREFIX.'*') as $key) { + $this->redis->del($key); + } + // also clean keys produced by withKey() tests + foreach ($this->redis->keys('lock:test:*') as $key) { + $this->redis->del($key); + } + } + + private function makeLock(?Database $db = null, ?Authorization $auth = null): Lock + { + return new Lock( + $this->redis, + new NoTelemetry(), + $db ?? $this->createStub(Database::class), + $auth ?? $this->authorization, + $this->log, + null, + $this->project, + ); + } + + public function test_set_uses_per_attribute_key_and_auth_skipped_update(): void + { + $captured = null; + $db = $this->createMock(Database::class); + $db->expects($this->once()) + ->method('updateDocument') + ->with('projects', 'p1', $this->callback(function (Document $doc) use (&$captured) { + $captured = $doc->getArrayCopy(); + + return true; + })) + ->willReturnArgument(2); + + $lock = $this->makeLock($db); + $lock->set('projects', 'p1', 'accessedAt', '2024-06-01 12:00:00'); + + $this->assertSame(['accessedAt' => '2024-06-01 12:00:00'], $captured); + $this->assertSame(0, $this->redis->exists(self::KEY_PREFIX.'projects:p1:accessedAt')); + } + + public function test_set_skips_on_contention(): void + { + $key = self::KEY_PREFIX.'projects:p1:accessedAt'; + $this->redis->set($key, 'other-owner', ['NX', 'EX' => 30]); + + $db = $this->createMock(Database::class); + $db->expects($this->never())->method('updateDocument'); + + $lock = $this->makeLock($db); + $lock->set('projects', 'p1', 'accessedAt', '2024-06-01 12:00:00'); + + $this->assertSame('other-owner', $this->redis->get($key)); + } + + public function test_set_different_attributes_do_not_compete(): void + { + // Hold accessedAt + $heldKey = self::KEY_PREFIX.'projects:p1:accessedAt'; + $this->redis->set($heldKey, 'other-owner', ['NX', 'EX' => 30]); + + // mcpAccessedAt should still be acquirable + $db = $this->createMock(Database::class); + $db->expects($this->once()) + ->method('updateDocument') + ->with('projects', 'p1', $this->isInstanceOf(Document::class)) + ->willReturnArgument(2); + + $lock = $this->makeLock($db); + $lock->set('projects', 'p1', 'mcpAccessedAt', '2024-06-01 12:00:00'); + + $this->assertSame('other-owner', $this->redis->get($heldKey)); + } + + public function test_run_uses_per_document_key_and_invokes_callback(): void + { + $called = false; + $lock = $this->makeLock(); + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertTrue($called); + $this->assertSame(0, $this->redis->exists(self::KEY_PREFIX.'keys:k1')); + } + + public function test_run_skips_on_contention(): void + { + $key = self::KEY_PREFIX.'keys:k1'; + $this->redis->set($key, 'other-owner', ['NX', 'EX' => 30]); + + $called = false; + $lock = $this->makeLock(); + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertFalse($called); + $this->assertSame('other-owner', $this->redis->get($key)); + } + + public function test_run_or_fail_throws_on_contention(): void + { + $key = self::KEY_PREFIX.'projects:p1'; + $this->redis->set($key, 'other-owner', ['NX', 'EX' => 30]); + + $lock = $this->makeLock(); + + $this->expectException(Exception::class); + try { + $lock->runOrFail('projects', 'p1', fn () => 'never-runs'); + } catch (Exception $e) { + $this->assertSame(Exception::GENERAL_RESOURCE_LOCKED, $e->getType()); + throw $e; + } + } + + public function test_run_or_fail_returns_callback_value_when_uncontended(): void + { + $lock = $this->makeLock(); + $result = $lock->runOrFail('projects', 'p1', fn () => 'ok'); + + $this->assertSame('ok', $result); + $this->assertSame(0, $this->redis->exists(self::KEY_PREFIX.'projects:p1')); + } + + public function test_with_key_uses_raw_key(): void + { + $custom = 'lock:test:custom-key'; + $called = false; + $lock = $this->makeLock(); + $lock->withKey($custom, function () use (&$called) { + $called = true; + }); + + $this->assertTrue($called); + $this->assertSame(0, $this->redis->exists($custom)); + } + + public function test_with_key_or_fail_flag_throws_on_contention(): void + { + $custom = 'lock:test:contended'; + $this->redis->set($custom, 'other', ['NX', 'EX' => 30]); + + $lock = $this->makeLock(); + $this->expectException(Exception::class); + $lock->withKey($custom, fn () => null, ttl: 5, orFail: true, waitTimeout: 0.1); + } + + public function test_disabled_mode_runs_callback_unlocked(): void + { + $previous = \getenv('_APP_LOCKING_ENABLED'); + \putenv('_APP_LOCKING_ENABLED=disabled'); + try { + // Even when the key is already held, the callback must still run. + $key = self::KEY_PREFIX.'keys:k1'; + $this->redis->set($key, 'other-owner', ['NX', 'EX' => 30]); + + $called = false; + $lock = $this->makeLock(); + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + + $this->assertTrue($called); + $this->assertSame('other-owner', $this->redis->get($key)); + } finally { + $previous === false ? \putenv('_APP_LOCKING_ENABLED') : \putenv('_APP_LOCKING_ENABLED='.$previous); + } + } + + public function test_project_without_sequence_falls_back_to_unknown(): void + { + $emptyProject = new Document(); + $lock = new Lock( + $this->redis, + new NoTelemetry(), + $this->createStub(Database::class), + $this->authorization, + $this->log, + null, + $emptyProject, + ); + + // Pre-acquire the lock at the 'unknown' projectInternalId path. + $key = 'lock:platform:unknown:keys:k1'; + $this->redis->set($key, 'held', ['NX', 'EX' => 30]); + + $called = false; + $lock->run('keys', 'k1', function () use (&$called) { + $called = true; + }); + $this->assertFalse($called, 'Lock without project sequence should hash to the unknown bucket'); + + $this->redis->del($key); + } +}