diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 80c855f389..71264d7efe 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -189,8 +189,8 @@ jobs:
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
- const runner4 = `runs-on=${context.runId}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache`;
- const runner8 = `runs-on=${context.runId}/runner=8cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache`;
+ const runner4 = `runs-on=${context.runId}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache`;
+ const runner8 = `runs-on=${context.runId}/runner=8cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache`;
const services = [
{ name: 'Account', runner: runner4, functional: true, processes: null, timeout: 8 },
@@ -257,7 +257,7 @@ jobs:
build:
name: Build
- runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+s3-cache+ecr-cache
+ runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+s3-cache+ecr-cache
timeout-minutes: 30
steps:
- name: Checkout repository
@@ -291,7 +291,7 @@ jobs:
unit:
name: Tests / Unit
- runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache
+ runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache
timeout-minutes: 15
needs: build
permissions:
@@ -356,7 +356,7 @@ jobs:
e2e_general:
name: Tests / E2E / General
- runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache
+ runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache
timeout-minutes: 15
needs: build
permissions:
@@ -554,7 +554,7 @@ jobs:
e2e_abuse:
name: Tests / E2E / Abuse (${{ matrix.mode }})
- runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache
+ runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache
timeout-minutes: 15
needs: [build, matrix]
permissions:
@@ -633,7 +633,7 @@ jobs:
e2e_screenshots:
name: Tests / E2E / Screenshots (${{ matrix.mode }})
- runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache
+ runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache
timeout-minutes: 15
needs: [build, matrix]
permissions:
@@ -718,7 +718,7 @@ jobs:
benchmark:
name: Benchmark
if: github.event_name == 'pull_request' || github.event_name == 'push'
- runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache
+ runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache
timeout-minutes: 20
needs: build
permissions:
diff --git a/app/config/errors.php b/app/config/errors.php
index 76022eb21f..cff9a6367c 100644
--- a/app/config/errors.php
+++ b/app/config/errors.php
@@ -54,6 +54,11 @@ return [
'description' => 'Rate limit for the current endpoint has been exceeded. Please try again after some time.',
'code' => 429,
],
+ Exception::GENERAL_RESOURCE_LOCKED => [
+ 'name' => Exception::GENERAL_RESOURCE_LOCKED,
+ 'description' => 'The requested resource is currently being modified by another request. Please retry after a brief delay.',
+ 'code' => 409,
+ ],
Exception::GENERAL_SMTP_DISABLED => [
'name' => Exception::GENERAL_SMTP_DISABLED,
'description' => 'SMTP is disabled on your Appwrite instance. You can learn more about setting up SMTP in our docs.',
diff --git a/app/config/variables.php b/app/config/variables.php
index d73a6b3b64..6c23af5c5e 100644
--- a/app/config/variables.php
+++ b/app/config/variables.php
@@ -43,6 +43,15 @@ return [
'question' => '',
'filter' => ''
],
+ [
+ 'name' => '_APP_LOCKING_ENABLED',
+ 'description' => 'Enable distributed locking for platform writes. Locks coordinate concurrent updates across API pods so read-modify-write operations on shared documents do not lose updates. By default, set to \'enabled\'. Set to \'disabled\' as an emergency kill switch; locks become no-ops and concurrent writes will race.',
+ 'introduction' => '1.9.3',
+ 'default' => 'enabled',
+ 'required' => false,
+ 'question' => '',
+ 'filter' => ''
+ ],
[
'name' => '_APP_OPTIONS_FORCE_HTTPS',
'description' => 'Allows you to force HTTPS connection to your API. This feature redirects any HTTP call to HTTPS and adds the \'Strict-Transport-Security\' header to all HTTP responses. By default, set to \'enabled\'. To disable, set to \'disabled\'. This feature will work only when your ports are set to default 80 and 443, and you have set up wildcard certificates with DNS challenge.',
diff --git a/app/controllers/general.php b/app/controllers/general.php
index 55118d4186..acf8d3e23f 100644
--- a/app/controllers/general.php
+++ b/app/controllers/general.php
@@ -13,6 +13,7 @@ use Appwrite\Event\Publisher\Certificate;
use Appwrite\Event\Publisher\Delete as DeletePublisher;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Locale\GeoRecord;
+use Appwrite\Locking\Lock;
use Appwrite\Network\Cors;
use Appwrite\Platform\Appwrite;
use Appwrite\SDK\AuthType;
@@ -76,7 +77,7 @@ use Utopia\Validator\Text;
Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE);
-function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, GeoRecord $geoRecord, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount)
+function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Event $queueForEvents, Bus $bus, Executor $executor, GeoRecord $geoRecord, callable $isResourceBlocked, array $platform, string $previewHostname, Authorization $authorization, ?Key $apiKey, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock)
{
$host = $request->getHostname();
if (!empty($previewHostname)) {
@@ -145,9 +146,16 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S
if (!$project->isEmpty() && $project->getId() !== 'console') {
$accessedAt = $project->getAttribute('accessedAt', 0);
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) {
- $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([
- 'accessedAt' => DateTime::now()
- ])));
+ $projectInternalId = (string) ($project->getSequence() ?: $project->getId());
+ $lock->tryWithKey(
+ 'lock:platform:'.$projectInternalId.':projects:'.$project->getId().':accessedAt',
+ fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument(
+ 'projects',
+ $project->getId(),
+ new Document(['accessedAt' => DateTime::now()])
+ )),
+ target: 'projects'
+ );
}
/**
@@ -835,8 +843,9 @@ Http::init()
->inject('authorization')
->inject('publisherForDeletes')
->inject('executionsRetentionCount')
+ ->inject('lock')
->inject('params')
- ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, GeoRecord $geoRecord, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, array $params) {
+ ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, GeoRecord $geoRecord, Event $queueForEvents, Bus $bus, Executor $executor, array $platform, callable $isResourceBlocked, string $previewHostname, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock, array $params) {
/*
* Appwrite Router
*/
@@ -844,7 +853,7 @@ Http::init()
$platformHostnames = $platform['hostnames'] ?? [];
// Only run Router when external domain
if (!\in_array($hostname, $platformHostnames) || !empty($previewHostname)) {
- if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geoRecord, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) {
+ if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geoRecord, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) {
$utopia->match($request)?->route->label('router', true);
}
}
@@ -1147,14 +1156,15 @@ Http::options()
->inject('authorization')
->inject('publisherForDeletes')
->inject('executionsRetentionCount')
- ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, GeoRecord $geoRecord, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) {
+ ->inject('lock')
+ ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, GeoRecord $geoRecord, callable $isResourceBlocked, array $platform, string $previewHostname, Document $project, Document $devKey, ?Key $apiKey, Cors $cors, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) {
/*
* Appwrite Router
*/
$platformHostnames = $platform['hostnames'] ?? [];
// Only run Router when external domain
if (!in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) {
- if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geoRecord, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) {
+ if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geoRecord, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) {
$utopia->match($request)?->route->label('router', true);
}
}
@@ -1549,13 +1559,14 @@ Http::get('/robots.txt')
->inject('authorization')
->inject('publisherForDeletes')
->inject('executionsRetentionCount')
- ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, GeoRecord $geoRecord, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) {
+ ->inject('lock')
+ ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, GeoRecord $geoRecord, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) {
$platformHostnames = $platform['hostnames'] ?? [];
if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) {
$template = new View(__DIR__ . '/../views/general/robots.phtml');
$response->text($template->render(false));
} else {
- if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geoRecord, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) {
+ if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geoRecord, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) {
$utopia->match($request)?->route->label('router', true);
}
}
@@ -1583,13 +1594,14 @@ Http::get('/humans.txt')
->inject('authorization')
->inject('publisherForDeletes')
->inject('executionsRetentionCount')
- ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, GeoRecord $geoRecord, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount) {
+ ->inject('lock')
+ ->action(function (Http $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Log $log, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Bus $bus, Executor $executor, GeoRecord $geoRecord, callable $isResourceBlocked, array $platform, string $previewHostname, ?Key $apiKey, Authorization $authorization, DeletePublisher $publisherForDeletes, int $executionsRetentionCount, Lock $lock) {
$platformHostnames = $platform['hostnames'] ?? [];
if (in_array($request->getHostname(), $platformHostnames) || !empty($previewHostname)) {
$template = new View(__DIR__ . '/../views/general/humans.phtml');
$response->text($template->render(false));
} else {
- if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geoRecord, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount)) {
+ if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $log, $queueForEvents, $bus, $executor, $geoRecord, $isResourceBlocked, $platform, $previewHostname, $authorization, $apiKey, $publisherForDeletes, $executionsRetentionCount, $lock)) {
$utopia->match($request)?->route->label('router', true);
}
}
diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php
index b3fb5f4d89..359c5e0a5b 100644
--- a/app/controllers/shared/api.php
+++ b/app/controllers/shared/api.php
@@ -16,6 +16,7 @@ use Appwrite\Event\Webhook;
use Appwrite\Extend\Exception;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Functions\EventProcessor;
+use Appwrite\Locking\Lock;
use Appwrite\Platform\Modules\Storage\Config\CacheControl;
use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl;
use Appwrite\Reference\Renderer;
@@ -59,9 +60,10 @@ Http::init()
->inject('team')
->inject('apiKey')
->inject('authorization')
+ ->inject('lock')
->inject('impersonatorUser')
->inject('targetUser')
- ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Document $impersonatorUser, User $targetUser) {
+ ->action(function (Route $route, Request $request, Database $dbForPlatform, Database $dbForProject, AuditContext $auditContext, Document $project, User $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey, Authorization $authorization, Lock $lock, Document $impersonatorUser, User $targetUser) {
/**
* Handle user authentication and session validation.
@@ -160,24 +162,28 @@ Http::init()
// For standard keys, update last accessed time
if (\in_array($apiKey->getType(), [API_KEY_STANDARD, API_KEY_ORGANIZATION, API_KEY_ACCOUNT])) {
$dbKey = null;
+ $keyOwnerInternalId = '';
if (! empty($apiKey->getProjectId())) {
$dbKey = $project->find(
key: 'secret',
find: $request->getHeaderLine('x-appwrite-key', ''),
subject: 'keys'
);
+ $keyOwnerInternalId = (string) ($project->getSequence() ?: $project->getId());
} elseif (! empty($apiKey->getUserId())) {
$dbKey = $user->find(
key: 'secret',
find: $request->getHeaderLine('x-appwrite-key', ''),
subject: 'keys'
);
+ $keyOwnerInternalId = (string) ($user->getSequence() ?: $user->getId());
} elseif (! empty($apiKey->getTeamId())) {
$dbKey = $team->find(
key: 'secret',
find: $request->getHeaderLine('x-appwrite-key', ''),
subject: 'keys'
);
+ $keyOwnerInternalId = (string) ($team->getSequence() ?: $team->getId());
}
if (!$dbKey) {
@@ -202,13 +208,19 @@ Http::init()
$sdks[] = $sdk;
$updates->setAttribute('sdks', $sdks);
- $updates->setAttribute('accessedAt', Datetime::now());
+ $updates->setAttribute('accessedAt', DateTime::now());
}
}
- if (! $updates->isEmpty()) {
- $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates));
+ $updatedKey = $updates->isEmpty()
+ ? null
+ : $lock->tryWithKey(
+ 'lock:platform:'.$keyOwnerInternalId.':keys:'.$dbKey->getId(),
+ fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)),
+ target: 'keys'
+ );
+ if ($updatedKey instanceof Document) {
if (! empty($apiKey->getProjectId())) {
$dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId()));
} elseif (! empty($apiKey->getUserId())) {
@@ -371,9 +383,16 @@ Http::init()
if ($project->getId() !== 'console') {
$accessedAt = $project->getAttribute('accessedAt', 0);
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) {
- $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([
- 'accessedAt' => DateTime::now()
- ])));
+ $projectInternalId = (string) ($project->getSequence() ?: $project->getId());
+ $lock->tryWithKey(
+ 'lock:platform:'.$projectInternalId.':projects:'.$project->getId().':accessedAt',
+ fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument(
+ 'projects',
+ $project->getId(),
+ new Document(['accessedAt' => DateTime::now()])
+ )),
+ target: 'projects'
+ );
}
}
@@ -389,9 +408,16 @@ Http::init()
'accessedAt' => $user->getAttribute('accessedAt')
]));
} else {
- $authorization->skip(fn () => $dbForPlatform->updateDocument('users', $user->getId(), new Document([
- 'accessedAt' => $user->getAttribute('accessedAt')
- ])));
+ $userInternalId = (string) ($user->getSequence() ?: $user->getId());
+ $lock->tryWithKey(
+ 'lock:platform:'.$userInternalId.':users:'.$user->getId().':accessedAt',
+ fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument(
+ 'users',
+ $user->getId(),
+ new Document(['accessedAt' => $user->getAttribute('accessedAt')])
+ )),
+ target: 'users'
+ );
}
}
}
diff --git a/app/init/resources/request.php b/app/init/resources/request.php
index dcf783e6d4..f1f17f3930 100644
--- a/app/init/resources/request.php
+++ b/app/init/resources/request.php
@@ -15,6 +15,7 @@ use Appwrite\Extend\Exception;
use Appwrite\Functions\EventProcessor;
use Appwrite\GraphQL\Schema;
use Appwrite\Locale\GeoRecord;
+use Appwrite\Locking\Lock;
use Appwrite\Network\Cors;
use Appwrite\Network\Platform;
use Appwrite\Network\Validator\Origin;
@@ -46,7 +47,9 @@ use Utopia\Domains\Domain;
use Utopia\Fetch\Client;
use Utopia\Http\Http;
use Utopia\Locale\Locale;
+use Utopia\Lock\Distributed as DistributedLock;
use Utopia\Logger\Log;
+use Utopia\Logger\Logger;
use Utopia\Pools\Group;
use Utopia\Queue\Publisher;
use Utopia\Queue\Queue;
@@ -68,6 +71,17 @@ return function (Container $context): void {
$context->set('logger', fn ($register) => $register->get('logger'), ['register']);
+ $context->set('lock', function (Group $pools, Telemetry $telemetry, ?Logger $logger, Document $project): Lock {
+ return new Lock(
+ fn (string $key, int $ttl, Closure $callback): mixed => $pools->get('lock')->use(
+ fn (\Redis $redis): mixed => $callback(new DistributedLock($redis, $key, $ttl))
+ ),
+ $telemetry,
+ $logger,
+ $project
+ );
+ }, ['pools', 'telemetry', 'logger', 'project']);
+
$context->set('authorization', fn () => new Authorization(), []);
$context->set('store', fn (): Store => new Store(), []);
diff --git a/docker-compose.yml b/docker-compose.yml
index 964f963323..1b5163e1e5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -171,6 +171,7 @@ services:
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
- _APP_LOGGING_CONFIG
+ - _APP_LOCKING_ENABLED
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php
index 416d628ee0..d490848037 100644
--- a/src/Appwrite/Extend/Exception.php
+++ b/src/Appwrite/Extend/Exception.php
@@ -45,6 +45,7 @@ class Exception extends \Exception
public const string GENERAL_SERVICE_DISABLED = 'general_service_disabled';
public const string GENERAL_UNAUTHORIZED_SCOPE = 'general_unauthorized_scope';
public const string GENERAL_RATE_LIMIT_EXCEEDED = 'general_rate_limit_exceeded';
+ public const string GENERAL_RESOURCE_LOCKED = 'general_resource_locked';
public const string GENERAL_SMTP_DISABLED = 'general_smtp_disabled';
public const string GENERAL_PHONE_DISABLED = 'general_phone_disabled';
public const string GENERAL_ARGUMENT_INVALID = 'general_argument_invalid';
diff --git a/src/Appwrite/Locking/Lock.php b/src/Appwrite/Locking/Lock.php
new file mode 100644
index 0000000000..f808ce404c
--- /dev/null
+++ b/src/Appwrite/Locking/Lock.php
@@ -0,0 +1,232 @@
+ */
+ private static array $lastReportAt = [];
+
+ /**
+ * @var Closure(string, int, Closure(UtopiaLock): mixed): mixed
+ */
+ private readonly Closure $useLock;
+
+ public function __construct(
+ Closure $useLock,
+ Telemetry $telemetry,
+ private readonly ?Logger $logger,
+ Document $project,
+ ) {
+ $this->useLock = $useLock;
+ $this->enabled = System::getEnv('_APP_LOCKING_ENABLED', 'enabled') !== 'disabled';
+ $this->attempts = $telemetry->createCounter('lock.attempts', null, 'Distributed lock acquire outcomes');
+ $this->projectInternalId = (string) ($project->getSequence() ?: $project->getId());
+ }
+
+ public function withKey(
+ string $key,
+ Closure $fn,
+ string $target,
+ int $ttl = self::FAIL_TTL_SECONDS,
+ float $waitTimeout = self::FAIL_WAIT_SECONDS,
+ ): mixed {
+ return $this->execute(
+ $key,
+ $target,
+ $fn,
+ ttl: $ttl,
+ waitTimeout: $waitTimeout,
+ skipOnContention: false,
+ );
+ }
+
+ public function tryWithKey(
+ string $key,
+ Closure $fn,
+ string $target,
+ int $ttl = self::SKIP_TTL_SECONDS,
+ ): mixed {
+ return $this->execute(
+ $key,
+ $target,
+ $fn,
+ ttl: $ttl,
+ waitTimeout: self::SKIP_WAIT_SECONDS,
+ skipOnContention: true,
+ );
+ }
+
+ private function execute(
+ string $key,
+ string $target,
+ Closure $fn,
+ int $ttl = self::FAIL_TTL_SECONDS,
+ float $waitTimeout = self::FAIL_WAIT_SECONDS,
+ bool $skipOnContention = false,
+ ): mixed {
+ if (! $this->enabled) {
+ return $fn();
+ }
+
+ $labels = ['target' => $target, 'project' => $this->projectInternalId];
+
+ $callbackException = false;
+
+ try {
+ return ($this->useLock)($key, $ttl, function (UtopiaLock $lock) use ($fn, $key, $labels, $skipOnContention, $target, $waitTimeout, &$callbackException): mixed {
+ return $this->executeWithLock($lock, $key, $target, $labels, $fn, $waitTimeout, $skipOnContention, $callbackException);
+ });
+ } catch (\RedisException $e) {
+ if ($callbackException) {
+ throw $e;
+ }
+
+ $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]);
+ $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e);
+
+ return $fn();
+ } catch (\Exception $e) {
+ if ($callbackException) {
+ throw $e;
+ }
+
+ // Pool exhaustion throws a bare \Exception before lock acquisition.
+ // Only that exact class is fail-open; subclasses must propagate.
+ if (get_class($e) !== \Exception::class) {
+ throw $e;
+ }
+ $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]);
+ $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e);
+
+ return $fn();
+ }
+ }
+
+ /**
+ * @param array{target: string, project: string} $labels
+ */
+ private function executeWithLock(
+ UtopiaLock $lock,
+ string $key,
+ string $target,
+ array $labels,
+ Closure $fn,
+ float $waitTimeout,
+ bool $skipOnContention,
+ bool &$callbackException,
+ ): mixed {
+ try {
+ $acquired = $lock->acquire($waitTimeout);
+ } catch (\RedisException $e) {
+ $this->attempts->add(1, ['outcome' => self::OUTCOME_BACKEND_ERROR, ...$labels]);
+ $this->reportError(self::OUTCOME_BACKEND_ERROR, $key, $target, $e);
+
+ $callbackException = true;
+ return $fn();
+ }
+
+ if (! $acquired) {
+ if ($skipOnContention) {
+ $this->attempts->add(1, ['outcome' => self::OUTCOME_SKIPPED, ...$labels]);
+
+ return null;
+ }
+ $this->attempts->add(1, ['outcome' => self::OUTCOME_CONTENDED, ...$labels]);
+ throw new Exception(Exception::GENERAL_RESOURCE_LOCKED);
+ }
+
+ $this->attempts->add(1, ['outcome' => self::OUTCOME_ACQUIRED, ...$labels]);
+ $callbackException = true;
+ try {
+ return $fn();
+ } finally {
+ try {
+ $lock->release();
+ } catch (Throwable $e) {
+ $this->attempts->add(1, ['outcome' => self::OUTCOME_RELEASE_ERROR, ...$labels]);
+ $this->reportError(self::OUTCOME_RELEASE_ERROR, $key, $target, $e);
+ }
+ }
+ }
+
+ /**
+ * Rate-limit backend/release reports so outages don't flood Sentry.
+ */
+ private function reportError(string $action, string $key, string $target, Throwable $e): void
+ {
+ Console::warning("Lock {$action} for {$key}: {$e->getMessage()}");
+
+ if ($this->logger === null) {
+ return;
+ }
+
+ $bucket = $action.':'.$target;
+ $now = time();
+ if ((self::$lastReportAt[$bucket] ?? 0) + self::REPORT_RATE_LIMIT_SECONDS > $now) {
+ return;
+ }
+ self::$lastReportAt[$bucket] = $now;
+
+ $log = new Log();
+ $log->setNamespace('http');
+ $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
+ $log->setVersion(APP_VERSION_STABLE);
+ $log->setType(Log::TYPE_WARNING);
+ $log->setMessage('Distributed lock '.$action.': '.$e->getMessage());
+ $log->setAction("lock.{$action}");
+ $log->setEnvironment(System::getEnv('_APP_ENV', 'development') === 'production'
+ ? Log::ENVIRONMENT_PRODUCTION
+ : Log::ENVIRONMENT_STAGING);
+ $log->addTag('lock.target', $target);
+ $log->addTag('lock.project', $this->projectInternalId);
+ // Strip trailing document ID to keep aggregator cardinality bounded.
+ $log->addTag('lock.key_pattern', preg_replace('/:[^:]+$/', ':*', $key));
+ $log->addTag('code', $e->getCode());
+ $log->addExtra('file', $e->getFile());
+ $log->addExtra('line', $e->getLine());
+ $log->addExtra('trace', $e->getTraceAsString());
+
+ try {
+ $this->logger->addLog($log);
+ } catch (Throwable) {
+ }
+ }
+}
diff --git a/tests/unit/Locking/LockTest.php b/tests/unit/Locking/LockTest.php
new file mode 100644
index 0000000000..d4e9a9b512
--- /dev/null
+++ b/tests/unit/Locking/LockTest.php
@@ -0,0 +1,549 @@
+
+ */
+ private array $heldLocks = [];
+
+ private Document $project;
+
+ private const PROJECT_SEQUENCE = '42';
+
+ private const KEY_PREFIX = 'lock:platform:'.self::PROJECT_SEQUENCE.':';
+
+ protected function setUp(): void
+ {
+ Config::setParam('errors', require __DIR__.'/../../../app/config/errors.php');
+
+ $this->heldLocks = [];
+ $this->project = new Document([
+ '$id' => 'test-project',
+ '$sequence' => self::PROJECT_SEQUENCE,
+ ]);
+ }
+
+ private function makeLock(): Lock
+ {
+ return new Lock(
+ $this->withLock(),
+ new NoTelemetry(),
+ null,
+ $this->project,
+ );
+ }
+
+ private function withLock(): \Closure
+ {
+ return fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new MemoryLock($key, $this->heldLocks));
+ }
+
+ public function testTryWithKeyUsesGivenKeyAndInvokesCallback(): void
+ {
+ $called = false;
+ $lock = $this->makeLock();
+ $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', function () use (&$called) {
+ $called = true;
+ }, target: 'keys');
+
+ $this->assertTrue($called);
+ $this->assertArrayNotHasKey(self::KEY_PREFIX.'keys:k1', $this->heldLocks);
+ }
+
+ public function testTryWithKeyUsesShortTryOnceLockWindow(): void
+ {
+ $ttl = null;
+ $timeout = null;
+
+ $lock = new Lock(
+ function (string $key, int $lockTtl, \Closure $callback) use (&$ttl, &$timeout): mixed {
+ $ttl = $lockTtl;
+
+ return $callback(new InspectingLock(new MemoryLock($key, $this->heldLocks), function (float $acquireTimeout) use (&$timeout): void {
+ $timeout = $acquireTimeout;
+ }));
+ },
+ new NoTelemetry(),
+ null,
+ $this->project,
+ );
+
+ $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', fn () => null, target: 'keys');
+
+ $this->assertSame(5, $ttl);
+ $this->assertEqualsWithDelta(0.0, $timeout, PHP_FLOAT_EPSILON);
+ }
+
+ public function testTryWithKeySkipsOnContention(): void
+ {
+ $key = self::KEY_PREFIX.'keys:k1';
+ $this->heldLocks[$key] = true;
+
+ $called = false;
+ $lock = $this->makeLock();
+
+ $lock->tryWithKey($key, function () use (&$called) {
+ $called = true;
+ }, target: 'keys');
+
+ $this->assertFalse($called);
+ $this->assertArrayHasKey($key, $this->heldLocks);
+ }
+
+ public function testWithKeyUsesShortHttpLockTtlAndAcquireTimeout(): void
+ {
+ $ttl = null;
+ $timeout = null;
+
+ $lock = new Lock(
+ function (string $key, int $lockTtl, \Closure $callback) use (&$ttl, &$timeout): mixed {
+ $ttl = $lockTtl;
+
+ return $callback(new InspectingLock(new MemoryLock($key, $this->heldLocks), function (float $acquireTimeout) use (&$timeout): void {
+ $timeout = $acquireTimeout;
+ }));
+ },
+ new NoTelemetry(),
+ null,
+ $this->project,
+ );
+
+ $lock->withKey(self::KEY_PREFIX.'keys:k1', fn () => null, target: 'keys');
+
+ $this->assertSame(10, $ttl);
+ $this->assertEqualsWithDelta(3.0, $timeout, PHP_FLOAT_EPSILON);
+ }
+
+ public function testBestEffortBackendErrorRunsCallbackUnlocked(): void
+ {
+ $lock = new Lock(
+ fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingAcquireLock(new \RedisException('redis unavailable'))),
+ new NoTelemetry(),
+ null,
+ $this->project,
+ );
+
+ $called = false;
+ $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', function () use (&$called) {
+ $called = true;
+ }, target: 'keys');
+
+ $this->assertTrue($called);
+ }
+
+ public function testBackendErrorReportDoesNotMutateRequestLog(): void
+ {
+ $requestLog = new Log();
+ $adapter = new RecordingLoggerAdapter();
+ $logger = new Logger($adapter);
+
+ $lock = new Lock(
+ fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingAcquireLock(new \RedisException('redis unavailable'))),
+ new NoTelemetry(),
+ $logger,
+ $this->project,
+ );
+
+ $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', fn () => null, target: 'keys');
+
+ $this->assertCount(1, $adapter->logs);
+ $this->assertNotSame($requestLog, $adapter->logs[0]);
+ $this->assertSame([], $requestLog->getTags());
+ $this->assertSame([], $requestLog->getExtra());
+ }
+
+ public function testWithKeyBackendErrorRunsCallbackUnlocked(): void
+ {
+ $lock = new Lock(
+ fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingAcquireLock(new \RedisException('redis unavailable'))),
+ new NoTelemetry(),
+ null,
+ $this->project,
+ );
+
+ $called = false;
+ $lock->withKey(self::KEY_PREFIX.'keys:k1', function () use (&$called) {
+ $called = true;
+ }, target: 'keys');
+
+ $this->assertTrue($called);
+ }
+
+ public function testBestEffortPoolCheckoutErrorRunsCallbackUnlocked(): void
+ {
+ $lock = new Lock(
+ fn (string $key, int $ttl, \Closure $callback): mixed => throw new \RedisException('pool unavailable'),
+ new NoTelemetry(),
+ null,
+ $this->project,
+ );
+
+ $called = false;
+ $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', function () use (&$called) {
+ $called = true;
+ }, target: 'keys');
+
+ $this->assertTrue($called);
+ }
+
+ public function testRedisExceptionThrownByCallbackIsNotTreatedAsBackendError(): void
+ {
+ $lock = $this->makeLock();
+
+ $this->expectException(\RedisException::class);
+ $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', fn () => throw new \RedisException('callback failed'), target: 'keys');
+ }
+
+ public function testReleaseErrorAfterCallbackIsLoggedButNotThrown(): void
+ {
+ $lock = new Lock(
+ fn (string $key, int $ttl, \Closure $callback): mixed => $callback(new ThrowingReleaseLock(new MemoryLock($key, $this->heldLocks), new \RedisException('release failed'))),
+ new NoTelemetry(),
+ null,
+ $this->project,
+ );
+
+ $this->assertSame('ok', $lock->withKey(self::KEY_PREFIX.'keys:k1', fn () => 'ok', target: 'keys'));
+ }
+
+ public function testWithKeyThrowsOnContention(): void
+ {
+ $key = self::KEY_PREFIX.'projects:p1';
+ $this->heldLocks[$key] = true;
+
+ $lock = $this->makeLock();
+
+ $this->expectException(Exception::class);
+ try {
+ $lock->withKey($key, fn () => 'never-runs', target: 'projects');
+ } catch (Exception $e) {
+ $this->assertSame(Exception::GENERAL_RESOURCE_LOCKED, $e->getType());
+ throw $e;
+ }
+ }
+
+ public function testWithKeyReturnsCallbackValueWhenUncontended(): void
+ {
+ $lock = $this->makeLock();
+ $result = $lock->withKey(self::KEY_PREFIX.'projects:p1', fn () => 'ok', target: 'projects');
+
+ $this->assertSame('ok', $result);
+ $this->assertArrayNotHasKey(self::KEY_PREFIX.'projects:p1', $this->heldLocks);
+ }
+
+ public function testWithKeyUsesRawKey(): void
+ {
+ $custom = 'lock:test:custom-key';
+ $called = false;
+ $lock = $this->makeLock();
+ $lock->withKey($custom, function () use (&$called) {
+ $called = true;
+ }, target: 'custom');
+
+ $this->assertTrue($called);
+ $this->assertArrayNotHasKey($custom, $this->heldLocks);
+ }
+
+ public function testTryWithKeySkipsGivenKeyOnContention(): void
+ {
+ $custom = 'lock:test:contended';
+ $this->heldLocks[$custom] = true;
+
+ $lock = $this->makeLock();
+ $called = false;
+ $lock->tryWithKey($custom, function () use (&$called): void {
+ $called = true;
+ }, target: 'custom', ttl: 5);
+
+ $this->assertFalse($called);
+ }
+
+ public function testWithKeyThrowsGivenKeyOnContention(): void
+ {
+ $custom = 'lock:test:contended';
+ $this->heldLocks[$custom] = true;
+
+ $lock = $this->makeLock();
+ $this->expectException(Exception::class);
+ $lock->withKey($custom, fn () => null, target: 'custom', ttl: 5, waitTimeout: 0.1);
+ }
+
+ public function testDisabledModeRunsCallbackUnlocked(): void
+ {
+ $previous = \getenv('_APP_LOCKING_ENABLED');
+ \putenv('_APP_LOCKING_ENABLED=disabled');
+ try {
+ $key = self::KEY_PREFIX.'keys:k1';
+ $this->heldLocks[$key] = true;
+
+ $called = false;
+ $lock = $this->makeLock();
+ $lock->tryWithKey($key, function () use (&$called) {
+ $called = true;
+ }, target: 'keys');
+
+ $this->assertTrue($called);
+ $this->assertArrayHasKey($key, $this->heldLocks);
+ } finally {
+ $previous === false ? \putenv('_APP_LOCKING_ENABLED') : \putenv('_APP_LOCKING_ENABLED='.$previous);
+ }
+ }
+
+ public function testPoolCheckoutExceptionRunsCallbackUnlocked(): void
+ {
+ $lock = new Lock(
+ fn (string $key, int $ttl, \Closure $callback): mixed => throw new \Exception('Pool \'lock\' is empty'),
+ new NoTelemetry(),
+ null,
+ $this->project,
+ );
+
+ $called = false;
+ $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', function () use (&$called) {
+ $called = true;
+ }, target: 'keys');
+
+ $this->assertTrue($called);
+ }
+
+ public function testNonPoolExceptionIsNotSwallowed(): void
+ {
+ $lock = new Lock(
+ fn (string $key, int $ttl, \Closure $callback): mixed => throw new \RuntimeException('unexpected'),
+ new NoTelemetry(),
+ null,
+ $this->project,
+ );
+
+ $this->expectException(\RuntimeException::class);
+ $lock->tryWithKey(self::KEY_PREFIX.'keys:k1', fn () => null, target: 'keys');
+ }
+
+}
+
+final class MemoryLock implements UtopiaLock
+{
+ private bool $acquired = false;
+
+ /**
+ * @param array $heldLocks
+ */
+ public function __construct(
+ private readonly string $key,
+ private array &$heldLocks,
+ ) {
+ }
+
+ public function acquire(float $timeout = 0.0): bool
+ {
+ return $this->tryAcquire();
+ }
+
+ public function tryAcquire(): bool
+ {
+ if (isset($this->heldLocks[$this->key])) {
+ return false;
+ }
+
+ $this->heldLocks[$this->key] = true;
+ $this->acquired = true;
+
+ return true;
+ }
+
+ public function release(): void
+ {
+ if (! $this->acquired) {
+ return;
+ }
+
+ unset($this->heldLocks[$this->key]);
+ $this->acquired = false;
+ }
+
+ public function withLock(callable $callback, float $timeout = 0.0): mixed
+ {
+ if (! $this->acquire($timeout)) {
+ throw new LockContention("Failed to acquire distributed lock: {$this->key}");
+ }
+
+ try {
+ return $callback();
+ } finally {
+ $this->release();
+ }
+ }
+}
+
+final class InspectingLock implements UtopiaLock
+{
+ public function __construct(
+ private readonly UtopiaLock $lock,
+ private readonly \Closure $inspectTimeout,
+ ) {
+ }
+
+ public function acquire(float $timeout = 0.0): bool
+ {
+ ($this->inspectTimeout)($timeout);
+
+ return $this->lock->acquire($timeout);
+ }
+
+ public function tryAcquire(): bool
+ {
+ return $this->lock->tryAcquire();
+ }
+
+ public function release(): void
+ {
+ $this->lock->release();
+ }
+
+ public function withLock(callable $callback, float $timeout = 0.0): mixed
+ {
+ if (! $this->acquire($timeout)) {
+ throw new LockContention('Failed to acquire distributed lock');
+ }
+
+ try {
+ return $callback();
+ } finally {
+ $this->release();
+ }
+ }
+}
+
+final class ThrowingAcquireLock implements UtopiaLock
+{
+ public function __construct(private readonly \Throwable $throwable)
+ {
+ }
+
+ public function acquire(float $timeout = 0.0): bool
+ {
+ throw $this->throwable;
+ }
+
+ public function tryAcquire(): bool
+ {
+ throw $this->throwable;
+ }
+
+ public function release(): void
+ {
+ }
+
+ public function withLock(callable $callback, float $timeout = 0.0): mixed
+ {
+ throw $this->throwable;
+ }
+}
+
+final class ThrowingReleaseLock implements UtopiaLock
+{
+ public function __construct(
+ private readonly UtopiaLock $lock,
+ private readonly \Throwable $throwable,
+ ) {
+ }
+
+ public function acquire(float $timeout = 0.0): bool
+ {
+ return $this->lock->acquire($timeout);
+ }
+
+ public function tryAcquire(): bool
+ {
+ return $this->lock->tryAcquire();
+ }
+
+ public function release(): void
+ {
+ $this->lock->release();
+
+ throw $this->throwable;
+ }
+
+ public function withLock(callable $callback, float $timeout = 0.0): mixed
+ {
+ if (! $this->acquire($timeout)) {
+ throw new LockContention('Failed to acquire distributed lock');
+ }
+
+ try {
+ return $callback();
+ } finally {
+ $this->release();
+ }
+ }
+}
+
+final class LockingRedisException extends \Exception
+{
+}
+
+final class RecordingLoggerAdapter extends LoggerAdapter
+{
+ /**
+ * @var list
+ */
+ public array $logs = [];
+
+ public static function getName(): string
+ {
+ return 'recording';
+ }
+
+ public function push(Log $log): int
+ {
+ $this->logs[] = $log;
+
+ return 200;
+ }
+
+ public function getSupportedTypes(): array
+ {
+ return [
+ Log::TYPE_WARNING,
+ ];
+ }
+
+ public function getSupportedEnvironments(): array
+ {
+ return [
+ Log::ENVIRONMENT_PRODUCTION,
+ Log::ENVIRONMENT_STAGING,
+ ];
+ }
+
+ public function getSupportedBreadcrumbTypes(): array
+ {
+ return [];
+ }
+}