feat(lock): include project internal id in lock key + telemetry

Per-manager request, lock keys are now prefixed with the project's
internal id (sequence) so that:
  - Locks are partitioned by project — Redis cluster slot affinity
    if/when sharded.
  - Cross-project requests can't compete on the same key for
    collection-scoped resources.
  - Telemetry (counter + Sentry tags) carries 'project' alongside
    'target', so dashboards can filter contention by project.

Key shapes:
  set:        lock:platform:{project}:{collection}:{id}:{attribute}
  run/orFail: lock:platform:{project}:{collection}:{id}
  withKey:    raw (caller-provided)

Lock now requires a project document at construction. All existing
call sites (4 in CE + 2 in cloud) run inside Http::init()-resolved
request scope where the project document is set, so no migration
needed. Workers/CLI without project context can use withKey directly.
This commit is contained in:
Prem Palanisamy
2026-04-29 11:26:18 +01:00
parent b2b9ac5b4d
commit c2a249c48b
2 changed files with 51 additions and 26 deletions
+3 -3
View File
@@ -75,9 +75,9 @@ return function (Container $container): void {
return $register->get('logger'); return $register->get('logger');
}, ['register']); }, ['register']);
$container->set('lock', function (\Redis $redis, Telemetry $telemetry, Database $dbForPlatform, Authorization $authorization, Log $log, ?Logger $logger): Lock { $container->set('lock', function (\Redis $redis, Telemetry $telemetry, Database $dbForPlatform, Authorization $authorization, Log $log, ?Logger $logger, Document $project): Lock {
return new Lock($redis, $telemetry, $dbForPlatform, $authorization, $log, $logger); return new Lock($redis, $telemetry, $dbForPlatform, $authorization, $log, $logger, $project);
}, ['redis', 'telemetry', 'dbForPlatform', 'authorization', 'log', 'logger']); }, ['redis', 'telemetry', 'dbForPlatform', 'authorization', 'log', 'logger', 'project']);
$container->set('authorization', function () { $container->set('authorization', function () {
return new Authorization(); return new Authorization();
+48 -23
View File
@@ -22,6 +22,8 @@ final class Lock
private readonly mixed $attempts; private readonly mixed $attempts;
private readonly string $projectInternalId;
/** @var array<string,int> */ /** @var array<string,int> */
private static array $lastReportAt = []; private static array $lastReportAt = [];
@@ -32,9 +34,12 @@ final class Lock
private readonly Authorization $authorization, private readonly Authorization $authorization,
private readonly Log $log, private readonly Log $log,
private readonly ?Logger $logger, private readonly ?Logger $logger,
Document $project,
) { ) {
$this->enabled = System::getEnv('_APP_LOCKING_ENABLED', 'enabled') !== 'disabled'; $this->enabled = System::getEnv('_APP_LOCKING_ENABLED', 'enabled') !== 'disabled';
$this->attempts = $telemetry->createCounter('lock.attempts', null, 'Distributed lock acquire outcomes'); $this->attempts = $telemetry->createCounter('lock.attempts', null, 'Distributed lock acquire outcomes');
$sequence = $project->getSequence();
$this->projectInternalId = ($sequence !== null && $sequence !== '') ? (string) $sequence : 'unknown';
} }
/** /**
@@ -49,16 +54,14 @@ final class Lock
string $attribute = 'accessedAt', string $attribute = 'accessedAt',
?string $value = null, ?string $value = null,
): void { ): void {
$this->withKey( $key = "lock:platform:{$this->projectInternalId}:{$collection}:{$id}:{$attribute}";
"lock:platform:{$collection}:{$id}:{$attribute}", $this->execute($key, $collection, function () use ($collection, $id, $attribute, $value) {
function () use ($collection, $id, $attribute, $value) { $this->authorization->skip(fn () => $this->dbForPlatform->updateDocument(
$this->authorization->skip(fn () => $this->dbForPlatform->updateDocument( $collection,
$collection, $id,
$id, new Document([$attribute => $value ?? DateTime::now()])
new Document([$attribute => $value ?? DateTime::now()]) ));
)); });
}
);
} }
/** /**
@@ -67,7 +70,8 @@ final class Lock
*/ */
public function run(string $collection, string $id, Closure $fn): void public function run(string $collection, string $id, Closure $fn): void
{ {
$this->withKey("lock:platform:{$collection}:{$id}", $fn); $key = "lock:platform:{$this->projectInternalId}:{$collection}:{$id}";
$this->execute($key, $collection, $fn);
} }
/** /**
@@ -77,18 +81,19 @@ final class Lock
*/ */
public function runOrFail(string $collection, string $id, Closure $fn): mixed public function runOrFail(string $collection, string $id, Closure $fn): mixed
{ {
return $this->withKey( $key = "lock:platform:{$this->projectInternalId}:{$collection}:{$id}";
"lock:platform:{$collection}:{$id}",
$fn, return $this->execute($key, $collection, $fn, ttl: 10, orFail: true);
ttl: 10,
orFail: true,
);
} }
/** /**
* Generic lock primitive with full control over key, TTL, contention * Generic lock primitive with full control over key, TTL, contention
* behavior, and wait timeout. Escape hatch for non-platform keys * behavior, and wait timeout. Escape hatch for non-platform keys
* (cache, queue, edge) and for unusual TTL/timeout requirements. * (cache, queue, edge) and for unusual TTL/timeout requirements.
*
* Caller may pass `target` for telemetry; otherwise it's extracted by
* position from the key (best-effort for keys following the standard
* `lock:<scope>:<...>:<target>:<...>` shape).
*/ */
public function withKey( public function withKey(
string $key, string $key,
@@ -96,18 +101,37 @@ final class Lock
int $ttl = 5, int $ttl = 5,
bool $orFail = false, bool $orFail = false,
float $waitTimeout = 3.0, float $waitTimeout = 3.0,
?string $target = null,
): mixed {
return $this->execute(
$key,
$target ?? self::targetOf($key),
$fn,
ttl: $ttl,
orFail: $orFail,
waitTimeout: $waitTimeout,
);
}
private function execute(
string $key,
string $target,
Closure $fn,
int $ttl = 5,
bool $orFail = false,
float $waitTimeout = 3.0,
): mixed { ): mixed {
if (! $this->enabled) { if (! $this->enabled) {
return $fn(); return $fn();
} }
$target = self::targetOf($key);
$lock = new DistributedLock($this->redis, $key, $ttl); $lock = new DistributedLock($this->redis, $key, $ttl);
$labels = ['target' => $target, 'project' => $this->projectInternalId];
try { try {
$acquired = $orFail ? $lock->acquire($waitTimeout) : $lock->tryAcquire(); $acquired = $orFail ? $lock->acquire($waitTimeout) : $lock->tryAcquire();
} catch (\RedisException $e) { } catch (\RedisException $e) {
$this->attempts->add(1, ['outcome' => 'backend_error', 'target' => $target]); $this->attempts->add(1, ['outcome' => 'backend_error', ...$labels]);
$this->reportError('backend_error', $key, $target, $e); $this->reportError('backend_error', $key, $target, $e);
return $fn(); return $fn();
@@ -115,23 +139,23 @@ final class Lock
if (! $acquired) { if (! $acquired) {
if ($orFail) { if ($orFail) {
$this->attempts->add(1, ['outcome' => 'contended', 'target' => $target]); $this->attempts->add(1, ['outcome' => 'contended', ...$labels]);
// No custom message — the lock key embeds collection + document id. // No custom message — the lock key embeds collection + document id.
throw new Exception(Exception::GENERAL_RESOURCE_LOCKED); throw new Exception(Exception::GENERAL_RESOURCE_LOCKED);
} }
$this->attempts->add(1, ['outcome' => 'skipped', 'target' => $target]); $this->attempts->add(1, ['outcome' => 'skipped', ...$labels]);
return null; return null;
} }
$this->attempts->add(1, ['outcome' => 'acquired', 'target' => $target]); $this->attempts->add(1, ['outcome' => 'acquired', ...$labels]);
try { try {
return $fn(); return $fn();
} finally { } finally {
try { try {
$lock->release(); $lock->release();
} catch (Throwable $e) { } catch (Throwable $e) {
$this->attempts->add(1, ['outcome' => 'release_error', 'target' => $target]); $this->attempts->add(1, ['outcome' => 'release_error', ...$labels]);
$this->reportError('release_error', $key, $target, $e); $this->reportError('release_error', $key, $target, $e);
} }
} }
@@ -173,6 +197,7 @@ final class Lock
? Log::ENVIRONMENT_PRODUCTION ? Log::ENVIRONMENT_PRODUCTION
: Log::ENVIRONMENT_STAGING); : Log::ENVIRONMENT_STAGING);
$this->log->addTag('lock.target', $target); $this->log->addTag('lock.target', $target);
$this->log->addTag('lock.project', $this->projectInternalId);
// Strip trailing document ID to keep aggregator cardinality bounded. // Strip trailing document ID to keep aggregator cardinality bounded.
$this->log->addTag('lock.key_pattern', preg_replace('/:[^:]+$/', ':*', $key)); $this->log->addTag('lock.key_pattern', preg_replace('/:[^:]+$/', ':*', $key));
$this->log->addTag('code', $e->getCode()); $this->log->addTag('code', $e->getCode());