diff --git a/.env b/.env index 4a6a3ac344..57deb8cc0a 100644 --- a/.env +++ b/.env @@ -55,8 +55,8 @@ _APP_DB_PORT_DOCUMENTSDB=27017 _APP_DB_ADAPTER_VECTORSDB=postgresql _APP_DB_HOST_VECTORSDB=postgresql _APP_DB_PORT_VECTORSDB=5432 -_APP_EMBEDDING_MODELS=embeddinggemma -_APP_EMBEDDING_ENDPOINT='http://ollama:11434/api/embed' +_APP_EMBEDDING_MODELS=nomic-embed-text +_APP_EMBEDDING_ENDPOINT='http://appwrite-embedding:3000/embed' _APP_EMBEDDING_TIMEOUT=30000 _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= @@ -150,3 +150,6 @@ _APP_POOL_ADAPTER=stack _APP_WORKER_SCREENSHOTS_ROUTER=http://appwrite _TESTS_OAUTH2_GITHUB_CLIENT_ID= _TESTS_OAUTH2_GITHUB_CLIENT_SECRET= + +_APP_WORKERS_NUM=1 +_APP_WORKER_MAX_COROUTINES=1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08804bd723..c3278d76af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -406,7 +406,7 @@ jobs: e2e_service: name: Tests / E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} - runs-on: ${{ matrix.runner || format('runs-on={0}/runner=4cpu-linux-x64/volume=120g/spot=false', github.run_id) }} + runs-on: ${{ matrix.runner || format('runs-on={0}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel', github.run_id) }} needs: [build, matrix] permissions: contents: read @@ -447,11 +447,11 @@ jobs: ] include: - service: Databases - runner: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/volume=120g/spot=false + runner: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/volume=120g/spot=false/extras=otel paratest_processes: 3 timeout_minutes: 30 - service: TablesDB - runner: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/volume=120g/spot=false + runner: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/volume=120g/spot=false/extras=otel paratest_processes: 3 timeout_minutes: 30 - service: Migrations @@ -761,6 +761,15 @@ jobs: run: | docker tag ${{ env.IMAGE }}:before ${{ env.IMAGE }} docker compose up -d --wait --no-build + for attempt in $(seq 1 10); do + if curl -fsS http://localhost/v1/health/version; then + break + fi + if [ "$attempt" -eq 10 ]; then + exit 1 + fi + sleep 1 + done - name: Prepare benchmark files run: rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before-samples.json benchmark-after-samples.json @@ -816,6 +825,15 @@ jobs: run: | docker tag ${{ env.IMAGE }}:after ${{ env.IMAGE }} docker compose up -d --wait --no-build + for attempt in $(seq 1 10); do + if curl -fsS http://localhost/v1/health/version; then + break + fi + if [ "$attempt" -eq 10 ]; then + exit 1 + fi + sleep 1 + done - name: Benchmark after id: benchmark_after diff --git a/app/assets/dbip/dbip-country-lite-2025-12.mmdb b/app/assets/dbip/dbip-country-lite-2025-12.mmdb deleted file mode 100644 index 4ecabbf735..0000000000 Binary files a/app/assets/dbip/dbip-country-lite-2025-12.mmdb and /dev/null differ diff --git a/app/assets/dbip/dbip-country-lite-2026-06.mmdb b/app/assets/dbip/dbip-country-lite-2026-06.mmdb new file mode 100644 index 0000000000..df09a565b3 Binary files /dev/null and b/app/assets/dbip/dbip-country-lite-2026-06.mmdb differ diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 67affddd20..789fce6d71 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -3932,6 +3932,8 @@ Http::put('/v1/account/recovery') $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]); + $sessions = $profile->getAttribute('sessions', []); + $profile = $dbForProject->updateDocument('users', $profile->getId(), new Document( [ 'password' => $newPassword, @@ -3944,6 +3946,14 @@ Http::put('/v1/account/recovery') $user->setAttributes($profile->getArrayCopy()); + $invalidate = $project->getAttribute('auths', default: [])['invalidateSessions'] ?? false; + if ($invalidate) { + foreach ($sessions as $session) { + /** @var Document $session */ + $dbForProject->deleteDocument('sessions', $session->getId()); + } + } + $recoveryDocument = $dbForProject->getDocument('tokens', $verifiedToken->getId()); /** diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 26f6bf21c4..8665fbf24c 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -236,6 +236,100 @@ Http::post('/v1/messaging/providers/sendgrid') ->dynamic($provider, Response::MODEL_PROVIDER); }); +Http::post('/v1/messaging/providers/ses') + ->desc('Create Amazon SES provider') + ->groups(['api', 'messaging']) + ->label('audits.event', 'provider.create') + ->label('audits.resource', 'provider/{response.$id}') + ->label('event', 'providers.[providerId].create') + ->label('scope', 'providers.write') + ->label('resourceType', RESOURCE_TYPE_PROVIDERS) + ->label('sdk', new Method( + namespace: 'messaging', + group: 'providers', + name: 'createSesProvider', + description: '/docs/references/messaging/create-ses-provider.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_CREATED, + model: Response::MODEL_PROVIDER, + ) + ] + )) + ->param('providerId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Provider ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) + ->param('name', '', new Text(128), 'Provider name.') + ->param('accessKey', '', new Text(0), 'AWS access key ID.', true) + ->param('secretKey', '', new Text(0), 'AWS secret access key.', true) + ->param('region', '', new Text(128), 'AWS region, for example us-east-1.', true) + ->param('fromName', '', new Text(128, 0), 'Sender Name.', true) + ->param('fromEmail', '', new Email(), 'Sender email address.', true) + ->param('replyToName', '', new Text(128, 0), 'Name set in the reply to field for the mail. Default value is sender name.', true) + ->param('replyToEmail', '', new Email(), 'Email set in the reply to field for the mail. Default value is sender email.', true) + ->param('enabled', null, new Nullable(new Boolean()), 'Set as enabled.', true) + ->inject('queueForEvents') + ->inject('dbForProject') + ->inject('response') + ->action(function (string $providerId, string $name, string $accessKey, string $secretKey, string $region, string $fromName, string $fromEmail, string $replyToName, string $replyToEmail, ?bool $enabled, Event $queueForEvents, Database $dbForProject, Response $response) { + $providerId = $providerId == 'unique()' ? ID::unique() : $providerId; + + $credentials = []; + + if (!empty($accessKey)) { + $credentials['accessKey'] = $accessKey; + } + + if (!empty($secretKey)) { + $credentials['secretKey'] = $secretKey; + } + + if (!empty($region)) { + $credentials['region'] = $region; + } + + $options = [ + 'fromName' => $fromName, + 'fromEmail' => $fromEmail, + 'replyToName' => $replyToName, + 'replyToEmail' => $replyToEmail, + ]; + + if ( + $enabled === true + && !empty($fromEmail) + && !empty($credentials['accessKey']) + && !empty($credentials['secretKey']) + && !empty($credentials['region']) + ) { + $enabled = true; + } else { + $enabled = false; + } + + $provider = new Document([ + '$id' => $providerId, + 'name' => $name, + 'provider' => 'ses', + 'type' => MESSAGE_TYPE_EMAIL, + 'enabled' => $enabled, + 'credentials' => $credentials, + 'options' => $options, + ]); + + try { + $provider = $dbForProject->createDocument('providers', $provider); + } catch (DuplicateException) { + throw new Exception(Exception::PROVIDER_ALREADY_EXISTS); + } + + $queueForEvents + ->setParam('providerId', $provider->getId()); + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($provider, Response::MODEL_PROVIDER); + }); + Http::post('/v1/messaging/providers/resend') ->desc('Create Resend provider') ->groups(['api', 'messaging']) @@ -1458,6 +1552,119 @@ Http::patch('/v1/messaging/providers/sendgrid/:providerId') ->dynamic($provider, Response::MODEL_PROVIDER); }); +Http::patch('/v1/messaging/providers/ses/:providerId') + ->desc('Update Amazon SES provider') + ->groups(['api', 'messaging']) + ->label('audits.event', 'provider.update') + ->label('audits.resource', 'provider/{response.$id}') + ->label('event', 'providers.[providerId].update') + ->label('scope', 'providers.write') + ->label('resourceType', RESOURCE_TYPE_PROVIDERS) + ->label('sdk', new Method( + namespace: 'messaging', + group: 'providers', + name: 'updateSesProvider', + description: '/docs/references/messaging/update-ses-provider.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_PROVIDER, + ) + ] + )) + ->param('providerId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Provider ID.', false, ['dbForProject']) + ->param('name', '', new Text(128), 'Provider name.', true) + ->param('enabled', null, new Nullable(new Boolean()), 'Set as enabled.', true) + ->param('accessKey', '', new Text(0), 'AWS access key ID.', true) + ->param('secretKey', '', new Text(0), 'AWS secret access key.', true) + ->param('region', '', new Text(128), 'AWS region, for example us-east-1.', true) + ->param('fromName', '', new Text(128), 'Sender Name.', true) + ->param('fromEmail', '', new Email(), 'Sender email address.', true) + ->param('replyToName', '', new Text(128), 'Name set in the Reply To field for the mail. Default value is Sender Name.', true) + ->param('replyToEmail', '', new Text(128), 'Email set in the Reply To field for the mail. Default value is Sender Email.', true) + ->inject('queueForEvents') + ->inject('dbForProject') + ->inject('response') + ->action(function (string $providerId, string $name, ?bool $enabled, string $accessKey, string $secretKey, string $region, string $fromName, string $fromEmail, string $replyToName, string $replyToEmail, Event $queueForEvents, Database $dbForProject, Response $response) { + $provider = $dbForProject->getDocument('providers', $providerId); + + if ($provider->isEmpty()) { + throw new Exception(Exception::PROVIDER_NOT_FOUND); + } + + $providerAttr = $provider->getAttribute('provider'); + + if ($providerAttr !== 'ses') { + throw new Exception(Exception::PROVIDER_INCORRECT_TYPE); + } + + if (!empty($name)) { + $provider->setAttribute('name', $name); + } + + $options = $provider->getAttribute('options'); + + if (!empty($fromName)) { + $options['fromName'] = $fromName; + } + + if (!empty($fromEmail)) { + $options['fromEmail'] = $fromEmail; + } + + if (!empty($replyToName)) { + $options['replyToName'] = $replyToName; + } + + if (!empty($replyToEmail)) { + $options['replyToEmail'] = $replyToEmail; + } + + $provider->setAttribute('options', $options); + + $credentials = $provider->getAttribute('credentials'); + + if (!empty($accessKey)) { + $credentials['accessKey'] = $accessKey; + } + + if (!empty($secretKey)) { + $credentials['secretKey'] = $secretKey; + } + + if (!empty($region)) { + $credentials['region'] = $region; + } + + $provider->setAttribute('credentials', $credentials); + + if (!\is_null($enabled)) { + if ($enabled) { + if ( + !empty($credentials['accessKey']) && + !empty($credentials['secretKey']) && + !empty($credentials['region']) && + !empty($options['fromEmail']) + ) { + $provider->setAttribute('enabled', true); + } else { + throw new Exception(Exception::PROVIDER_MISSING_CREDENTIALS); + } + } else { + $provider->setAttribute('enabled', false); + } + } + + $provider = $dbForProject->updateDocument('providers', $provider->getId(), $provider); + + $queueForEvents + ->setParam('providerId', $provider->getId()); + + $response + ->dynamic($provider, Response::MODEL_PROVIDER); + }); + Http::patch('/v1/messaging/providers/resend/:providerId') ->desc('Update Resend provider') ->groups(['api', 'messaging']) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index c8ea66ea51..19cff6aa01 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -591,7 +591,7 @@ Http::post('/v1/users/:userId/targets') ->label('audits.event', 'target.create') ->label('audits.resource', 'target/response.$id') ->label('event', 'users.[userId].targets.[targetId].create') - ->label('scope', 'targets.write') + ->label('scope', 'users.write') ->label('sdk', new Method( namespace: 'users', group: 'targets', @@ -852,7 +852,7 @@ Http::get('/v1/users/:userId/prefs') Http::get('/v1/users/:userId/targets/:targetId') ->desc('Get user target') ->groups(['api', 'users']) - ->label('scope', 'targets.read') + ->label('scope', 'users.read') ->label('sdk', new Method( namespace: 'users', group: 'targets', @@ -1078,7 +1078,7 @@ Http::get('/v1/users/:userId/logs') Http::get('/v1/users/:userId/targets') ->desc('List user targets') ->groups(['api', 'users']) - ->label('scope', 'targets.read') + ->label('scope', 'users.read') ->label('sdk', new Method( namespace: 'users', group: 'targets', @@ -1843,7 +1843,7 @@ Http::patch('/v1/users/:userId/targets/:targetId') ->label('audits.event', 'target.update') ->label('audits.resource', 'target/{response.$id}') ->label('event', 'users.[userId].targets.[targetId].update') - ->label('scope', 'targets.write') + ->label('scope', 'users.write') ->label('sdk', new Method( namespace: 'users', group: 'targets', @@ -2661,7 +2661,7 @@ Http::delete('/v1/users/:userId/targets/:targetId') ->label('audits.event', 'target.delete') ->label('audits.resource', 'target/{request.$targetId}') ->label('event', 'users.[userId].targets.[targetId].delete') - ->label('scope', 'targets.write') + ->label('scope', 'users.write') ->label('sdk', new Method( namespace: 'users', group: 'targets', diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 494bd3da28..db1bf99edb 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -786,124 +786,132 @@ Http::shutdown() Http::shutdown() ->groups(['api']) ->inject('route') - ->inject('request') ->inject('response') ->inject('project') - ->inject('user') ->inject('queueForEvents') - ->inject('auditContext') - ->inject('publisherForAudits') - ->inject('usage') - ->inject('publisherForUsage') ->inject('publisherForFunctions') ->inject('queueForWebhooks') ->inject('queueForRealtime') ->inject('dbForProject') - ->inject('authorization') - ->inject('timelimit') ->inject('eventProcessor') - ->inject('bus') - ->inject('apiKey') - ->inject('mode') - ->action(function (Route $route, Request $request, Response $response, Document $project, User $user, Event $queueForEvents, AuditContext $auditContext, Audit $publisherForAudits, Context $usage, UsagePublisher $publisherForUsage, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, Authorization $authorization, callable $timelimit, EventProcessor $eventProcessor, Bus $bus, ?Key $apiKey, string $mode) use ($parseLabel) { + ->action(function (Route $route, Response $response, Document $project, Event $queueForEvents, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, EventProcessor $eventProcessor) { + if (empty($queueForEvents->getEvent())) { + return; + } - $responsePayload = $response->getPayload(); + if (empty($queueForEvents->getPayload())) { + $queueForEvents->setPayload($response->getPayload()); + } - if (! empty($queueForEvents->getEvent())) { - if (empty($queueForEvents->getPayload())) { - $queueForEvents->setPayload($responsePayload); - } + // Get project and function/webhook events (cached) + $functionsEvents = $eventProcessor->getFunctionsEvents($project, $dbForProject); + $webhooksEvents = $eventProcessor->getWebhooksEvents($project); - // Get project and function/webhook events (cached) - $functionsEvents = $eventProcessor->getFunctionsEvents($project, $dbForProject); - $webhooksEvents = $eventProcessor->getWebhooksEvents($project); + // Generate events for this operation + $generatedEvents = Event::generateEvents( + $queueForEvents->getEvent(), + $queueForEvents->getParams() + ); - // Generate events for this operation - $generatedEvents = Event::generateEvents( - $queueForEvents->getEvent(), - $queueForEvents->getParams() - ); + $allowedOnConsole = !empty(\array_intersect($route->getGroups(), Realtime::CONSOLE_ALLOWLIST)); + if ($project->getId() !== 'console' || $allowedOnConsole) { + $queueForRealtime + ->from($queueForEvents) + ->trigger(); + } - if ($project->getId() !== 'console') { - $queueForRealtime - ->from($queueForEvents) - ->trigger(); - } - - // Only trigger functions if there are matching function events - if (! empty($functionsEvents)) { - foreach ($generatedEvents as $event) { - if (isset($functionsEvents[$event])) { - $publisherForFunctions->enqueue(FunctionMessage::fromEvent( - event: $queueForEvents->getEvent(), - params: $queueForEvents->getParams(), - project: $queueForEvents->getProject(), - user: $queueForEvents->getUser(), - userId: $queueForEvents->getUserId(), - payload: $queueForEvents->getPayload(), - platform: $queueForEvents->getPlatform(), - )); - break; - } - } - } - - // Only trigger webhooks if there are matching webhook events - if (! empty($webhooksEvents)) { - foreach ($generatedEvents as $event) { - if (isset($webhooksEvents[$event])) { - $queueForWebhooks - ->from($queueForEvents) - ->trigger(); - break; - } + // Only trigger functions if there are matching function events + if (! empty($functionsEvents)) { + foreach ($generatedEvents as $event) { + if (isset($functionsEvents[$event])) { + $publisherForFunctions->enqueue(FunctionMessage::fromEvent( + event: $queueForEvents->getEvent(), + params: $queueForEvents->getParams(), + project: $queueForEvents->getProject(), + user: $queueForEvents->getUser(), + userId: $queueForEvents->getUserId(), + payload: $queueForEvents->getPayload(), + platform: $queueForEvents->getPlatform(), + )); + break; } } } - $requestParams = $route->getParamsValues(); + // Only trigger webhooks if there are matching webhook events + if (! empty($webhooksEvents)) { + foreach ($generatedEvents as $event) { + if (isset($webhooksEvents[$event])) { + $queueForWebhooks + ->from($queueForEvents) + ->trigger(); + break; + } + } + } + }); - /** - * Abuse labels - */ +Http::shutdown() + ->groups(['api']) + ->inject('route') + ->inject('request') + ->inject('response') + ->inject('project') + ->inject('user') + ->inject('timelimit') + ->action(function (Route $route, Request $request, Response $response, Document $project, User $user, callable $timelimit) { $abuseEnabled = System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') !== 'disabled'; $abuseResetCode = $route->getLabel('abuse-reset', []); $abuseResetCode = \is_array($abuseResetCode) ? $abuseResetCode : [$abuseResetCode]; - if ($abuseEnabled && \count($abuseResetCode) > 0 && \in_array($response->getStatusCode(), $abuseResetCode)) { - $abuseKeyLabel = $route->getLabel('abuse-key', 'url:{url},ip:{ip}'); - $abuseKeyLabel = (! is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; - - foreach ($abuseKeyLabel as $abuseKey) { - $start = $request->getContentRangeStart(); - $end = $request->getContentRangeEnd(); - $timeLimit = $timelimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600)); - $timeLimit - ->setParam('{projectId}', $project->getId()) - ->setParam('{userId}', $user->getId()) - ->setParam('{userAgent}', $request->getUserAgent('')) - ->setParam('{ip}', $request->getIP()) - ->setParam('{url}', $request->getHostname() . $route->getPath()) - ->setParam('{method}', $request->getMethod()) - ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start))); - - foreach ($request->getParams() as $key => $value) { // Set request params as potential abuse keys - if (! empty($value)) { - $timeLimit->setParam('{param-' . $key . '}', (\is_array($value)) ? \json_encode($value) : $value); - } - } - - $abuse = new Abuse($timeLimit); - $abuse->reset(); - } + if (! $abuseEnabled || \count($abuseResetCode) === 0 || ! \in_array($response->getStatusCode(), $abuseResetCode)) { + return; } - /** - * Audit labels - */ + $abuseKeyLabel = $route->getLabel('abuse-key', 'url:{url},ip:{ip}'); + $abuseKeyLabel = (! is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; + + foreach ($abuseKeyLabel as $abuseKey) { + $start = $request->getContentRangeStart(); + $end = $request->getContentRangeEnd(); + $timeLimit = $timelimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600)); + $timeLimit + ->setParam('{projectId}', $project->getId()) + ->setParam('{userId}', $user->getId()) + ->setParam('{userAgent}', $request->getUserAgent('')) + ->setParam('{ip}', $request->getIP()) + ->setParam('{url}', $request->getHostname() . $route->getPath()) + ->setParam('{method}', $request->getMethod()) + ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start))); + + foreach ($request->getParams() as $key => $value) { // Set request params as potential abuse keys + if (! empty($value)) { + $timeLimit->setParam('{param-' . $key . '}', (\is_array($value)) ? \json_encode($value) : $value); + } + } + + $abuse = new Abuse($timeLimit); + $abuse->reset(); + } + }); + +Http::shutdown() + ->groups(['api']) + ->inject('route') + ->inject('params') + ->inject('request') + ->inject('response') + ->inject('project') + ->inject('user') + ->inject('auditContext') + ->inject('publisherForAudits') + ->inject('mode') + ->action(function (Route $route, array $params, Request $request, Response $response, Document $project, User $user, AuditContext $auditContext, Audit $publisherForAudits, string $mode) use ($parseLabel) { + $responsePayload = $response->getPayload(); + $pattern = $route->getLabel('audits.resource', null); if (! empty($pattern)) { - $resource = $parseLabel($pattern, $responsePayload, $requestParams, $user, $project); + $resource = $parseLabel($pattern, $responsePayload, $params, $user, $project); if (! empty($resource) && $resource !== $pattern) { $auditContext->resource = $resource; } @@ -950,97 +958,129 @@ Http::shutdown() $publisherForAudits->enqueue(AuditMessage::fromContext($auditContext)); } + }); - // Cache label - $useCache = $route->getLabel('cache', false); - if ($useCache) { - $resource = $resourceType = null; - $data = $response->getPayload(); - $statusCode = $response->getStatusCode(); - if (! empty($data['payload']) && $statusCode >= 200 && $statusCode < 300) { - $pattern = $route->getLabel('cache.resource', null); - if (! empty($pattern)) { - $resource = $parseLabel($pattern, $responsePayload, $requestParams, $user, $project); - } +Http::shutdown() + ->groups(['api']) + ->inject('route') + ->inject('params') + ->inject('request') + ->inject('response') + ->inject('project') + ->inject('user') + ->inject('dbForProject') + ->inject('authorization') + ->action(function (Route $route, array $params, Request $request, Response $response, Document $project, User $user, Database $dbForProject, Authorization $authorization) use ($parseLabel) { + if (! $route->getLabel('cache', false)) { + return; + } - $pattern = $route->getLabel('cache.resourceType', null); - if (! empty($pattern)) { - $resourceType = $parseLabel($pattern, $responsePayload, $requestParams, $user, $project); - } + $data = $response->getPayload(); + $statusCode = $response->getStatusCode(); + if (empty($data['payload']) || $statusCode < 200 || $statusCode >= 300) { + return; + } - $cache = new Cache( - new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) - ); + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) + ); + $key = $request->cacheIdentifier(); + $signature = md5($data['payload']); + $now = DateTime::now(); + $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); - $key = $request->cacheIdentifier(); - $signature = md5($data['payload']); + // First request for this resource: record it and persist the payload. + if ($cacheLog->isEmpty()) { + // Resolve resource labels lazily — only needed when creating the entry. + $requestParams = []; + foreach ($route->getParams() as $paramKey => $param) { + $requestParams[$paramKey] = $params[$paramKey] ?? $request->getParam($paramKey, $param['default']); + } + + $resourcePattern = $route->getLabel('cache.resource', null); + $resourceTypePattern = $route->getLabel('cache.resourceType', null); + + try { + $authorization->skip(fn () => $dbForProject->createDocument('cache', new Document([ + '$id' => $key, + 'resource' => empty($resourcePattern) ? null : $parseLabel($resourcePattern, $data, $requestParams, $user, $project), + 'resourceType' => empty($resourceTypePattern) ? null : $parseLabel($resourceTypePattern, $data, $requestParams, $user, $project), + 'mimeType' => $response->getContentType(), + 'accessedAt' => $now, + 'signature' => $signature, + ]))); + $cache->save($key, $data['payload']); + + return; + } catch (DuplicateException) { + // Race condition: another concurrent request already created the entry. $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); - $accessedAt = $cacheLog->getAttribute('accessedAt', 0); - $now = DateTime::now(); - if ($cacheLog->isEmpty()) { - try { - $authorization->skip(fn () => $dbForProject->createDocument('cache', new Document([ - '$id' => $key, - 'resource' => $resource, - 'resourceType' => $resourceType, - 'mimeType' => $response->getContentType(), - 'accessedAt' => $now, - 'signature' => $signature, - ]))); - } catch (DuplicateException) { - // Race condition: another concurrent request already created the cache document - $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); - } - } elseif (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_CACHE_UPDATE)) > $accessedAt) { - $cacheLog->setAttribute('accessedAt', $now); - $authorization->skip(fn () => $dbForProject->updateDocument('cache', $cacheLog->getId(), new Document([ - 'accessedAt' => $cacheLog->getAttribute('accessedAt') - ]))); - // Overwrite the file every APP_CACHE_UPDATE seconds to update the file modified time that is used in the TTL checks in cache->load() - $cache->save($key, $data['payload']); - } - - if ($signature !== $cacheLog->getAttribute('signature')) { - $cache->save($key, $data['payload']); - } } } - if ($project->getId() !== 'console') { - if (! $user->isPrivileged($authorization->getRoles())) { - $bus->dispatch(new RequestCompleted( - project: $project->getArrayCopy(), - request: $request, - response: $response, - )); - } + // Existing entry: refresh the access time once per APP_CACHE_UPDATE window + // (keeps the file mtime current for TTL checks) and re-persist on content change. + $stale = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_CACHE_UPDATE)) > $cacheLog->getAttribute('accessedAt', 0); + if ($stale) { + $authorization->skip(fn () => $dbForProject->updateDocument('cache', $cacheLog->getId(), new Document([ + 'accessedAt' => $now, + ]))); + } - // Publish usage metrics if context has data - if (! $usage->isEmpty()) { - $metrics = $usage->getMetrics(); + if ($stale || $signature !== $cacheLog->getAttribute('signature')) { + $cache->save($key, $data['payload']); + } + }); - // Filter out API key disabled metrics using suffix pattern matching - $disabledMetrics = $apiKey?->getDisabledMetrics() ?? []; - if (! empty($disabledMetrics)) { - $metrics = array_values(array_filter($metrics, function ($metric) use ($disabledMetrics) { - foreach ($disabledMetrics as $pattern) { - if (str_ends_with($metric['key'], $pattern)) { - return false; - } +Http::shutdown() + ->groups(['api']) + ->inject('request') + ->inject('response') + ->inject('project') + ->inject('user') + ->inject('usage') + ->inject('publisherForUsage') + ->inject('authorization') + ->inject('bus') + ->inject('apiKey') + ->action(function (Request $request, Response $response, Document $project, User $user, Context $usage, UsagePublisher $publisherForUsage, Authorization $authorization, Bus $bus, ?Key $apiKey) { + if ($project->getId() === 'console') { + return; + } + + if (! $user->isPrivileged($authorization->getRoles())) { + $bus->dispatch(new RequestCompleted( + project: $project->getArrayCopy(), + request: $request, + response: $response, + )); + } + + // Publish usage metrics if context has data + if (! $usage->isEmpty()) { + $metrics = $usage->getMetrics(); + + // Filter out API key disabled metrics using suffix pattern matching + $disabledMetrics = $apiKey?->getDisabledMetrics() ?? []; + if (! empty($disabledMetrics)) { + $metrics = array_values(array_filter($metrics, function ($metric) use ($disabledMetrics) { + foreach ($disabledMetrics as $pattern) { + if (str_ends_with($metric['key'], $pattern)) { + return false; } + } - return true; - })); - } - - $message = new UsageMessage( - project: $project, - metrics: $metrics, - reduce: $usage->getReduce() - ); - - $publisherForUsage->enqueue($message); + return true; + })); } + + $message = new UsageMessage( + project: $project, + metrics: $metrics, + reduce: $usage->getReduce() + ); + + $publisherForUsage->enqueue($message); } }); diff --git a/app/init/constants.php b/app/init/constants.php index 4bb607e2fb..4fe7c418a9 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -108,6 +108,9 @@ const APP_VCS_GITHUB_EMAIL = 'team@appwrite.io'; const APP_VCS_GITHUB_URL = 'https://github.com/TeamAppwrite'; const APP_BRANDED_EMAIL_BASE_TEMPLATE = 'email-base-styled'; +// Embeddings +const APP_EMBEDDING_BATCH_LIMIT = 100; + /** * JWT for Resource Tokens. */ @@ -239,6 +242,12 @@ const RULE_STATUS_VERIFIED = 'verified'; // Message types const MESSAGE_SEND_TYPE_INTERNAL = 'internal'; const MESSAGE_SEND_TYPE_EXTERNAL = 'external'; +// External message fan-out tuning +const MESSAGE_RECIPIENTS_PAGE_SIZE = 5000; // Recipients resolved from the database per page while streaming a send; aligned with APP_LIMIT_COUNT to minimise round-trips on large topics +const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job +const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message +const MESSAGE_SEND_MAX_RETRIES = 5; // Maximum number of attempts to deliver a batch when the provider throttles or fails transiently +const MESSAGE_SEND_RETRY_DELAY = 1.0; // Base seconds for exponential backoff between batch send retries; overridable in tests for an instant suite // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; diff --git a/app/init/registers.php b/app/init/registers.php index 9ea19eee24..f47cc43e60 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -254,6 +254,10 @@ $register->set('pools', function () { $workerCount = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); $poolSize = max(1, (int)($instanceConnections / $workerCount)); + // Queue workers consume jobs concurrently with coroutines; each in-flight + // job may hold a connection, so pools must cover the coroutine count. + $poolSize = max($poolSize, (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1)); + foreach ($connections as $key => $connection) { $type = $connection['type']; $multiple = $connection['multiple']; @@ -340,9 +344,13 @@ $register->set('pools', function () { // PubSub workers hold one long-lived subscribed connection and also need // spare capacity for publishes from the same process. - $connectionPoolSize = $type === 'pubsub' - ? max(2, $poolSize) - : $poolSize; + // Consumer pools lease one connection per consume coroutine for the + // lifetime of the worker, plus headroom for queue-size reads. + $connectionPoolSize = match ($type) { + 'pubsub' => max(2, $poolSize), + 'consumer' => $poolSize + 2, + default => $poolSize, + }; $pool = new Pool($poolAdapter, $name, $connectionPoolSize, function () use ($type, $resource, $dsn) { // Get Adapter @@ -451,7 +459,7 @@ $register->set('smtp', function () { ); }); $register->set('geodb', function () { - return new Reader(__DIR__ . '/../assets/dbip/dbip-country-lite-2025-12.mmdb'); + return new Reader(__DIR__ . '/../assets/dbip/dbip-country-lite-2026-06.mmdb'); }); $register->set('passwordsDictionary', function () { $content = \file_get_contents(__DIR__ . '/../assets/security/10k-common-passwords'); diff --git a/app/init/resources.php b/app/init/resources.php index 7b7e13482c..34f66d3de7 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1,5 +1,6 @@ set('register', fn () => $register); + +$container->set('logger', fn ($register) => $register->get('logger'), ['register']); + +$container->set('hooks', fn ($register) => $register->get('hooks'), ['register']); + $container->set('console', fn () => new Document(Config::getParam('console')), []); +$container->set('platform', fn () => Config::getParam('platform', []), []); + +$container->set('localeCodes', fn () => array_map(fn ($locale) => $locale['code'], Config::getParam('locale-codes', []))); + $container->set('executor', fn () => new Executor(), []); $container->set('telemetry', fn () => new NoTelemetry(), []); +$container->set('authorization', fn () => new Authorization(), []); + $container->set('publisher', fn (Group $pools) => new BrokerPool(publisher: $pools->get('publisher')), ['pools']); $container->set('publisherDatabases', fn (Publisher $publisher) => $publisher, ['publisher']); @@ -140,85 +150,36 @@ $container->set('publisherForMessaging', fn (Publisher $publisher) => new Messag new Queue(System::getEnv('_APP_MESSAGING_QUEUE_NAME', Event::MESSAGING_QUEUE_NAME)) ), ['publisher']); -$container->set('logger', function ($register) { - return $register->get('logger'); -}, ['register']); +$container->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( + $pools, + $cache, + $authorization +), ['pools', 'cache', 'authorization']); -$container->set('hooks', function ($register) { - return $register->get('hooks'); -}, ['register']); +$container->set('dbForPlatform', fn (DatabaseFactory $databaseFactory) => $databaseFactory->platform( + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES, + ['host' => \gethostname(), 'project' => 'console'] +), ['databaseFactory']); -$container->set('register', fn () => $register); - -$container->set('localeCodes', function () { - return array_map(fn ($locale) => $locale['code'], Config::getParam('locale-codes', [])); -}); - - -/** - * Platform configuration - */ -$container->set('platform', function () { - return Config::getParam('platform', []); -}, []); - -$container->set('authorization', function () { - return new Authorization(); -}, []); - -$container->set('dbForPlatform', function (Group $pools, Cache $cache, Authorization $authorization) { - - $adapter = new DatabasePool($pools->get('console')); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setNamespace('_console') - ->setMetadata('host', \gethostname()) - ->setMetadata('project', 'console') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - - $database->setDocumentType('users', User::class); - - return $database; -}, ['pools', 'cache', 'authorization']); - -$container->set('getLogsDB', function (Group $pools, Cache $cache, Authorization $authorization) { +$container->set('getLogsDB', function (DatabaseFactory $databaseFactory) { $database = null; - return function (?Document $project = null) use ($pools, $cache, $authorization, &$database) { + return function (?Document $project = null) use ($databaseFactory, &$database) { if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { $database->setTenant($project->getSequence()); return $database; } - $adapter = new DatabasePool($pools->get('logs')); - $database = new Database($adapter, $cache); - - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $logsCollections = $collections['logs'] ?? []; - $logsCollections = array_keys($logsCollections); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setSharedTables(true) - ->setGlobalCollections($logsCollections) - ->setNamespace('logsV1') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - - // set tenant - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant($project->getSequence()); - } + $database = $databaseFactory->logs( + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES + ); return $database; }; -}, ['pools', 'cache', 'authorization']); +}, ['databaseFactory']); $container->set('cache', function (Group $pools, Telemetry $telemetry) { $list = Config::getParam('pools-cache', []); @@ -234,9 +195,7 @@ $container->set('cache', function (Group $pools, Telemetry $telemetry) { return $cache; }, ['pools', 'telemetry']); -$container->set('cacheControlForStorage', fn () => function (StorageCacheControl $config): string { - return \sprintf('private, max-age=%d', $config->maxAge); -}); +$container->set('cacheControlForStorage', fn () => fn (StorageCacheControl $config): string => \sprintf('private, max-age=%d', $config->maxAge)); $container->set('redis', function () { $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); @@ -253,25 +212,14 @@ $container->set('redis', function () { return $redis; }); -$container->set('locks', function (Group $pools) { - return function (string $key, int $ttl, callable $callback, float $timeout = 0.0) use ($pools): mixed { - return $pools->get('lock')->use(function (\Redis $redis) use ($key, $ttl, $callback, $timeout) { - $lock = new Distributed($redis, $key, ttl: $ttl); +$container->set('locks', fn (Group $pools) => fn (string $key, int $ttl, callable $callback, float $timeout = 0.0): mixed => $pools->get('lock')->use( + fn (\Redis $redis) => (new Distributed($redis, $key, ttl: $ttl))->withLock($callback, timeout: $timeout) +), ['pools']); - return $lock->withLock($callback, timeout: $timeout); - }); - }; -}, ['pools']); +$container->set('timelimit', fn (\Redis $redis) => fn (string $key, int $limit, int $time) => new TimeLimitRedis($key, $limit, $time, $redis), ['redis']); -$container->set('timelimit', function (\Redis $redis) { - return function (string $key, int $limit, int $time) use ($redis) { - return new TimeLimitRedis($key, $limit, $time, $redis); - }; -}, ['redis']); +$container->set('deviceForLocal', fn (Telemetry $telemetry) => new Device\Telemetry($telemetry, new Local()), ['telemetry']); -$container->set('deviceForLocal', function (Telemetry $telemetry) { - return new Device\Telemetry($telemetry, new Local()); -}, ['telemetry']); function getDevice(string $root, string $connection = ''): Device { $connection = ! empty($connection) ? $connection : System::getEnv('_APP_CONNECTIONS_STORAGE', ''); @@ -299,14 +247,14 @@ function getDevice(string $root, string $connection = ''): Device switch ($device) { case Storage::DEVICE_S3: if (! empty($url)) { - $bucketRoot = (! empty($bucket) ? $bucket . '/' : '') . \ltrim($root, '/'); + $bucketRoot = (! empty($bucket) ? "{$bucket}/" : '') . \ltrim($root, '/'); return new S3($bucketRoot, $accessKey, $accessSecret, $url, $region, $acl); } else { return new AWS($root, $accessKey, $accessSecret, $bucket, $region, $acl); } // no break - case STORAGE::DEVICE_DO_SPACES: + case Storage::DEVICE_DO_SPACES: $device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); $device->setHttpVersion(S3::HTTP_VERSION_1_1); @@ -323,9 +271,6 @@ function getDevice(string $root, string $connection = ''): Device } } else { switch (strtolower(System::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL))) { - case Storage::DEVICE_LOCAL: - default: - return new Local($root); case Storage::DEVICE_S3: $s3AccessKey = System::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); $s3SecretKey = System::getEnv('_APP_STORAGE_S3_SECRET', ''); @@ -334,7 +279,7 @@ function getDevice(string $root, string $connection = ''): Device $s3Acl = 'private'; $s3EndpointUrl = System::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); if (! empty($s3EndpointUrl)) { - $bucketRoot = (! empty($s3Bucket) ? $s3Bucket . '/' : '') . \ltrim($root, '/'); + $bucketRoot = (! empty($s3Bucket) ? "{$s3Bucket}/" : '') . \ltrim($root, '/'); return new S3($bucketRoot, $s3AccessKey, $s3SecretKey, $s3EndpointUrl, $s3Region, $s3Acl); } else { @@ -375,46 +320,33 @@ function getDevice(string $root, string $connection = ''): Device $wasabiAcl = 'private'; return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); + case Storage::DEVICE_LOCAL: + default: + return new Local($root); } } } -$container->set('geodb', function ($register) { - /** @var Utopia\Registry\Registry $register */ - return $register->get('geodb'); -}, ['register']); +$container->set('geodb', fn ($register) => $register->get('geodb'), ['register']); -$container->set('passwordsDictionary', function ($register) { - /** @var Utopia\Registry\Registry $register */ - return $register->get('passwordsDictionary'); -}, ['register']); +$container->set('passwordsDictionary', fn ($register) => $register->get('passwordsDictionary'), ['register']); $container->set('servers', function () { $platforms = Config::getParam('sdks'); $server = $platforms[APP_SDK_PLATFORM_SERVER]; - $languages = array_map(function ($language) { - return strtolower($language['name']); - }, $server['sdks']); + $languages = array_map(fn ($language) => strtolower($language['name']), $server['sdks']); return $languages; }); -$container->set('promiseAdapter', function ($register) { - return $register->get('promiseAdapter'); -}, ['register']); +$container->set('promiseAdapter', fn ($register) => $register->get('promiseAdapter'), ['register']); -$container->set('gitHub', function (Cache $cache) { - return new VcsGitHub($cache); -}, ['cache']); +$container->set('gitHub', fn (Cache $cache) => new VcsGitHub($cache), ['cache']); -$container->set('plan', function () { - return []; -}); +$container->set('plan', fn () => []); -$container->set('smsRates', function () { - return []; -}); +$container->set('smsRates', fn () => []); $container->set( 'isResourceBlocked', diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 8c55eaf3e0..416c294140 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -3,6 +3,7 @@ use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Key; +use Appwrite\Database\Factory as DatabaseFactory; use Appwrite\Databases\TransactionState; use Appwrite\Event\Context\Audit as AuditContext; use Appwrite\Event\Event; @@ -21,7 +22,7 @@ use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; -use Utopia\Agents\Adapters\Ollama; +use Utopia\Agents\Adapters\Appwrite as AppwriteAdapter; use Utopia\Agents\Agent; use Utopia\Audit\Adapter\Database as AdapterDatabase; use Utopia\Audit\Audit; @@ -33,7 +34,6 @@ use Utopia\Auth\Proofs\Token; use Utopia\Auth\Store; use Utopia\Cache\Cache; use Utopia\Config\Config; -use Utopia\Database\Adapter\Pool as DatabasePool; use Utopia\Database\Database; use Utopia\Database\DateTime as DatabaseDateTime; use Utopia\Database\Document; @@ -41,7 +41,6 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\DI\Container; use Utopia\Domains\Domain; -use Utopia\DSN\DSN; use Utopia\Http\Http; use Utopia\Locale\Locale; use Utopia\Logger\Log; @@ -116,28 +115,21 @@ return function (Container $context): void { new Queue(System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME), 'utopia-queue', Event::FUNCTIONS_QUEUE_TTL) ), ['publisher']); $context->set('eventProcessor', fn () => new EventProcessor(), []); - $context->set('dbForPlatform', function (Group $pools, Cache $cache, Authorization $authorization) { - $adapter = new DatabasePool($pools->get('console')); - $database = new Database($adapter, $cache); + $context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( + $pools, + $cache, + $authorization + ), ['pools', 'cache', 'authorization']); - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setNamespace('_console') - ->setMetadata('host', \gethostname()) - ->setMetadata('project', 'console') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); + $context->set('dbForPlatform', fn (DatabaseFactory $databaseFactory) => $databaseFactory->platform( + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES, + ['host' => \gethostname(), 'project' => 'console'] + ), ['databaseFactory']); - $database->setDocumentType('users', User::class); + $context->set('getProjectDB', function (DatabaseFactory $databaseFactory, Database $dbForPlatform) { - return $database; - }, ['pools', 'cache', 'authorization']); - - $context->set('getProjectDB', function (Group $pools, Database $dbForPlatform, Cache $cache, Authorization $authorization) { - $adapters = []; - - return function (Document $project) use ($pools, $dbForPlatform, $cache, $authorization, &$adapters) { + return function (Document $project) use ($databaseFactory, $dbForPlatform) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -147,78 +139,25 @@ return function (Container $context): void { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Project database is not configured'); } - try { - $dsn = new DSN($database); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $database); - } - - $adapter = $adapters[$dsn->getHost()] ??= new DatabasePool($pools->get($dsn->getHost())); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES) - ->setDocumentType('users', User::class); - - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; - - $database - ->setSharedTables(true) - ->setTenant($project->getSequence()) - ->setGlobalCollections($projectsGlobalCollections) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } - - return $database; + return $databaseFactory->project( + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES, + ['host' => \gethostname(), 'project' => $project->getId()] + ); }; - }, ['pools', 'dbForPlatform', 'cache', 'authorization']); + }, ['databaseFactory', 'dbForPlatform']); - $context->set('getLogsDB', function (Group $pools, Cache $cache, Authorization $authorization) { - $adapter = null; + $context->set('getLogsDB', function (DatabaseFactory $databaseFactory) { - return function (?Document $project = null) use ($pools, $cache, $authorization, &$adapter) { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $logsCollections = $collections['logs'] ?? []; - $logsCollections = array_keys($logsCollections); - - $adapter ??= new DatabasePool($pools->get('logs')); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setSharedTables(true) - ->setGlobalCollections($logsCollections) - ->setNamespace('logsV1') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant($project->getSequence()); - } - - return $database; + return function (?Document $project = null) use ($databaseFactory) { + return $databaseFactory->logs( + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES + ); }; - }, ['pools', 'cache', 'authorization']); + }, ['databaseFactory']); /** * List of allowed request hostnames for the request. @@ -636,7 +575,7 @@ return function (Container $context): void { return; }, ['user', 'store', 'proofForToken']); - $context->set('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project, Response $response, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, FunctionPublisher $publisherForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, UsageContext $usage, Authorization $authorization, Request $request) { + $context->set('dbForProject', function (DatabaseFactory $databaseFactory, Database $dbForPlatform, Document $project, Response $response, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, FunctionPublisher $publisherForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, UsageContext $usage, Request $request) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -646,45 +585,12 @@ return function (Container $context): void { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Project database is not configured'); } - try { - $dsn = new DSN($database); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $database); - } - - $adapter = new DatabasePool($pools->get($dsn->getHost())); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - $database->setDocumentType('users', User::class); - - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; - - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } + $database = $databaseFactory->project( + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES, + ['host' => \gethostname(), 'project' => $project->getId()] + ); /** * This isolated event handling for `users.*.create` which is based on a `Database::EVENT_DOCUMENT_CREATE` listener may look odd, but it is **intentional**. @@ -915,7 +821,7 @@ return function (Container $context): void { ->on(Database::EVENT_DOCUMENT_DELETE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)); return $database; - }, ['pools', 'dbForPlatform', 'cache', 'project', 'response', 'publisher', 'publisherFunctions', 'publisherWebhooks', 'queueForEvents', 'publisherForFunctions', 'queueForWebhooks', 'queueForRealtime', 'usage', 'authorization', 'request']); + }, ['databaseFactory', 'dbForPlatform', 'project', 'response', 'publisher', 'publisherFunctions', 'publisherWebhooks', 'queueForEvents', 'publisherForFunctions', 'queueForWebhooks', 'queueForRealtime', 'usage', 'request']); $context->set('schema', function ($utopia, $dbForProject, $authorization) { @@ -1251,82 +1157,19 @@ return function (Container $context): void { return new Document([]); }, ['project', 'dbForProject', 'request', 'authorization']); - $context->set('getDatabasesDB', function (Group $pools, Cache $cache, Document $project, Request $request, UsageContext $usage, Authorization $authorization) { + $context->set('getDatabasesDB', function (DatabaseFactory $databaseFactory, Document $project, Request $request, UsageContext $usage) { - return function (Document $database) use ($pools, $cache, $project, $request, $usage, $authorization): Database { - $databaseDSN = $database->getAttribute('database') ?: $project->getAttribute('database', ''); + return function (Document $database) use ($databaseFactory, $project, $request, $usage): Database { $databaseType = $database->getAttribute('type', ''); - try { - $databaseDSN = new DSN($databaseDSN); - } catch (\InvalidArgumentException) { - // for old databases migrated through patch script - // databaseDSN determines the adapter - $databaseDSN = new DSN('mysql://' . $databaseDSN); - } - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } + $database = $databaseFactory->tenant( + $database, + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES, + ['host' => \gethostname(), 'project' => $project->getId()] + ); - $databaseHost = $databaseDSN->getHost(); - $pool = $pools->get($databaseHost); - - $adapter = new DatabasePool($pool); - $database = new Database($adapter, $cache); - $sharedTables = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''))); - - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - // inside pools authorization needs to be set first - $database->getAdapter()->setSupportForAttributes($databaseType !== DOCUMENTSDB); - - // For separate pools (documentsdb/vectorsdb), check their own shared tables config. - // If not configured, use dedicated mode to avoid cross-engine tenant type mismatches. - if ($databaseHost !== $dsn->getHost()) { - $dbTypeSharedTables = match ($databaseType) { - DOCUMENTSDB => \array_filter(\explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES', ''))), - VECTORSDB => \array_filter(\explode(',', System::getEnv('_APP_DATABASE_VECTORSDB_SHARED_TABLES', ''))), - default => [], - }; - - if (\in_array($databaseHost, $dbTypeSharedTables)) { - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($project->getSequence()) - ->setNamespace($databaseDSN->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } - } elseif (\in_array($dsn->getHost(), $sharedTables)) { - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } $timeout = \intval($request->getHeader('x-appwrite-timeout')); if (!empty($timeout) && Http::isDevelopment()) { $database->setTimeout($timeout); @@ -1411,7 +1254,7 @@ return function (Container $context): void { return $database; }; - }, ['pools', 'cache', 'project', 'request', 'usage', 'authorization']); + }, ['databaseFactory', 'project', 'request', 'usage']); $context->set( 'transactionState', @@ -1434,8 +1277,8 @@ return function (Container $context): void { $context->set('deviceForBuilds', fn ($project, Telemetry $telemetry) => new Device\Telemetry($telemetry, getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId())), ['project', 'telemetry']); $context->set('embeddingAgent', function ($register) { - $adapter = new Ollama(); - $adapter->setEndpoint(System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://ollama:11434/api/embed')); + $adapter = new AppwriteAdapter(); + $adapter->setEndpoint(System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://appwrite-embedding:3000/embed')); $adapter->setTimeout((int) System::getEnv('_APP_EMBEDDING_TIMEOUT', '30000')); return new Agent($adapter); }, ['register']); diff --git a/app/init/worker/message.php b/app/init/worker/message.php index 5cabfc7859..7cbfe19b0f 100644 --- a/app/init/worker/message.php +++ b/app/init/worker/message.php @@ -1,26 +1,24 @@ set('dbForPlatform', function (Cache $cache, Group $pools, Authorization $authorization) { - $adapter = new DatabasePool($pools->get('console')); - $dbForPlatform = new Database($adapter, $cache); + $container->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( + $pools, + $cache, + $authorization + ), ['pools', 'cache', 'authorization']); - $dbForPlatform - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setNamespace('_console') - ->setDocumentType('users', User::class); - - return $dbForPlatform; - }, ['cache', 'pools', 'authorization']); + $container->set('dbForPlatform', fn (DatabaseFactory $databaseFactory) => $databaseFactory->platform(), ['databaseFactory']); $container->set('project', function ($message, Database $dbForPlatform) { $payload = $message->getPayload() ?? []; @@ -63,250 +56,70 @@ return function (Container $container): void { return $project; } - return $dbForPlatform->getDocument('projects', $project->getId()); + $project = $dbForPlatform->getDocument('projects', $project->getId()); + + Span::add('project.id', $project->getId()); + + return $project; }, ['message', 'dbForPlatform']); - $container->set('dbForProject', function (Cache $cache, Group $pools, Document $project, Database $dbForPlatform, Authorization $authorization) { - if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; - } + $container->set('dbForProject', fn (DatabaseFactory $databaseFactory, Document $project, Database $dbForPlatform) => $project->isEmpty() || $project->getId() === 'console' + ? $dbForPlatform + : $databaseFactory->project( + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER, + ), ['databaseFactory', 'project', 'dbForPlatform']); - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - $adapter = new DatabasePool($pools->get($dsn->getHost())); - $database = new Database($adapter, $cache); - $database->setDocumentType('users', User::class); - - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; - - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER); - - return $database; - }, ['cache', 'pools', 'project', 'dbForPlatform', 'authorization']); - - $container->set('getProjectDB', function (Group $pools, Database $dbForPlatform, Cache $cache, Authorization $authorization) { - $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - - return function (Document $project) use ($pools, $dbForPlatform, $cache, $authorization, &$databases): Database { + $container->set('getProjectDB', function (DatabaseFactory $databaseFactory, Database $dbForPlatform) { + return function (Document $project) use ($databaseFactory, $dbForPlatform): Database { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - if (isset($databases[$dsn->getHost()])) { - $database = $databases[$dsn->getHost()]; - $database->setAuthorization($authorization); - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; - - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } - - return $database; - } - - $adapter = new DatabasePool($pools->get($dsn->getHost())); - $database = new Database($adapter, $cache); - - $databases[$dsn->getHost()] = $database; - - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; - - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER); - - return $database; + return $databaseFactory->project( + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER + ); }; - }, ['pools', 'dbForPlatform', 'cache', 'authorization']); + }, ['databaseFactory', 'dbForPlatform']); - $container->set('getDatabasesDB', function (Cache $cache, Registry $register, Document $project, Authorization $authorization) { - return function (Document $database, ?Document $projectDocument = null) use ($cache, $register, $project, $authorization): Database { + $container->set('getDatabasesDB', function (DatabaseFactory $databaseFactory, Document $project) { + return function (Document $database, ?Document $projectDocument = null) use ($databaseFactory, $project): Database { $projectDocument ??= $project; - $databaseDSN = $database->getAttribute('database', $project->getAttribute('database', '')); - $databaseType = $database->getAttribute('type', ''); // Backwards-compatibility: older or seeded legacy databases may not have a DSN stored // in the "database" attribute. In that case, fall back to the project's database DSN. - if ($databaseDSN === '') { - $databaseDSN = $projectDocument->getAttribute('database', ''); - } + $databaseConfig = $database->getAttribute('database', '') === '' + ? new Document(\array_merge($database->getArrayCopy(), ['database' => $projectDocument->getAttribute('database', '')])) + : $database; - try { - $databaseDSN = new DSN($databaseDSN); - } catch (\InvalidArgumentException) { - $databaseDSN = new DSN('mysql://' . $databaseDSN); - } - - try { - $dsn = new DSN($projectDocument->getAttribute('database')); - } catch (\InvalidArgumentException) { - // Temporary fallback until all projects use shared tables - $dsn = new DSN('mysql://' . $projectDocument->getAttribute('database')); - } - - $pools = $register->get('pools'); - $databaseHost = $databaseDSN->getHost(); - $pool = $pools->get($databaseHost); - - $adapter = new DatabasePool($pool); - $database = new Database($adapter, $cache); - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization); - $database->getAdapter()->setSupportForAttributes($databaseType !== DOCUMENTSDB); - - $sharedTables = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''))); - - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; - - $database->setGlobalCollections($projectsGlobalCollections); - - // For separate pools (documentsdb/vectorsdb), check their own shared tables config. - // If not configured, use dedicated mode to avoid cross-engine tenant type mismatches. - if ($databaseHost !== $dsn->getHost()) { - $dbTypeSharedTables = match ($databaseType) { - DOCUMENTSDB => \array_filter(\explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES', ''))), - VECTORSDB => \array_filter(\explode(',', System::getEnv('_APP_DATABASE_VECTORSDB_SHARED_TABLES', ''))), - default => [], - }; - - if (\in_array($databaseHost, $dbTypeSharedTables)) { - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($projectDocument->getSequence()) - ->setNamespace($databaseDSN->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $projectDocument->getSequence()); - } - } elseif (\in_array($dsn->getHost(), $sharedTables, true)) { - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($projectDocument->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $projectDocument->getSequence()); - } - - $database->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER); - return $database; + return $databaseFactory->tenant( + $databaseConfig, + $projectDocument, + APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER, + ); }; - }, ['cache', 'register', 'project', 'authorization']); + }, ['databaseFactory', 'project']); - $container->set('getLogsDB', function (Group $pools, Cache $cache, Authorization $authorization) { + $container->set('getLogsDB', function (DatabaseFactory $databaseFactory) { $database = null; - return function (?Document $project = null) use ($pools, $cache, $authorization, &$database) { + return function (?Document $project = null) use ($databaseFactory, &$database) { if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { $database->setTenant($project->getSequence()); return $database; } - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $logsCollections = $collections['logs'] ?? []; - $logsCollections = array_keys($logsCollections); - - $adapter = new DatabasePool($pools->get('logs')); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setSharedTables(true) - ->setGlobalCollections($logsCollections) - ->setNamespace('logsV1') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES_WORKER); - - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant($project->getSequence()); - } + $database = $databaseFactory->logs( + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER, + APP_DATABASE_QUERY_MAX_VALUES_WORKER + ); return $database; }; - }, ['pools', 'cache', 'authorization']); + }, ['databaseFactory']); $container->set('abuseRetention', function () { return \time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); // 1 day diff --git a/app/realtime.php b/app/realtime.php index ce2dc41e54..c99e4c0857 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -657,50 +657,58 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $userId = $event['userId']; if ($realtime->hasSubscriber($projectId, 'user:' . $userId)) { - $connection = array_key_first(reset($realtime->subscriptions[$projectId]['user:' . $userId])); - $subscriptionsBefore = \count($realtime->getSubscriptionMetadata($connection)); + $connections = []; + foreach ($realtime->subscriptions[$projectId]['user:' . $userId] as $byConnection) { + foreach (\array_keys($byConnection) as $connectionId) { + $connections[$connectionId] = true; + } + } + $consoleDatabase = getConsoleDB(); $project = $consoleDatabase->getAuthorization()->skip(fn () => $consoleDatabase->getDocument('projects', $projectId)); $database = getProjectDB($project); /** @var User $user */ $user = $database->getDocument('users', $userId); - $roles = $user->getRoles($database->getAuthorization()); - $authorization = $realtime->connections[$connection]['authorization'] ?? null; - $previousUserId = $realtime->connections[$connection]['userId'] ?? ''; - $meta = $realtime->getSubscriptionMetadata($connection); + foreach (\array_keys($connections) as $connection) { + $subscriptionsBefore = \count($realtime->getSubscriptionMetadata($connection)); + $authorization = $realtime->connections[$connection]['authorization'] ?? null; + $previousUserId = $realtime->connections[$connection]['userId'] ?? ''; - $realtime->unsubscribe($connection); + $meta = $realtime->getSubscriptionMetadata($connection); - foreach ($meta as $subscriptionId => $subscription) { - $queries = Query::parseQueries($subscription['queries'] ?? []); - $channels = Realtime::rebindAccountChannels( - $subscription['channels'] ?? [], - $previousUserId, - $userId - ); - $realtime->subscribe( - $projectId, - $connection, - $subscriptionId, - $roles, - $channels, - $queries, - $userId - ); - } + $realtime->unsubscribe($connection); - // Restore authorization after subscribe - if ($authorization !== null) { - $realtime->connections[$connection]['authorization'] = $authorization; - } + foreach ($meta as $subscriptionId => $subscription) { + $queries = Query::parseQueries($subscription['queries'] ?? []); + $channels = Realtime::rebindAccountChannels( + $subscription['channels'] ?? [], + $previousUserId, + $userId + ); + $realtime->subscribe( + $projectId, + $connection, + $subscriptionId, + $roles, + $channels, + $queries, + $userId + ); + } - $subscriptionsAfter = \count($realtime->getSubscriptionMetadata($connection)); - $subscriptionDelta = $subscriptionsAfter - $subscriptionsBefore; - if ($subscriptionDelta !== 0) { - $register->get('telemetry.workerSubscriptionCounter')->add($subscriptionDelta, $register->get('telemetry.workerAttributes')); + // Restore authorization after subscribe + if ($authorization !== null) { + $realtime->connections[$connection]['authorization'] = $authorization; + } + + $subscriptionsAfter = \count($realtime->getSubscriptionMetadata($connection)); + $subscriptionDelta = $subscriptionsAfter - $subscriptionsBefore; + if ($subscriptionDelta !== 0) { + $register->get('telemetry.workerSubscriptionCounter')->add($subscriptionDelta, $register->get('telemetry.workerAttributes')); + } } } } diff --git a/app/worker.php b/app/worker.php index 169b8e9770..063d4746ac 100644 --- a/app/worker.php +++ b/app/worker.php @@ -83,17 +83,25 @@ if (\str_starts_with($workerName, 'databases')) { /** @var \Utopia\Pools\Group $pools */ $pools = $container->get('pools'); +// Each consume coroutine leases its own broker connection from the pool for +// the lifetime of its consume loop, so concurrent coroutines never share a +// connection. The publisher side backs the queue-depth telemetry gauge. $adapter = new Swoole( - $pools->get('consumer')->pop()->getResource(), + new BrokerPool( + publisher: $pools->get('consumer'), + consumer: $pools->get('consumer'), + ), System::getEnv('_APP_WORKERS_NUM', 1), - $queueName + $queueName, + maxCoroutines: (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1), + resources: $container, ); -$worker = new Server($adapter, $container); +$worker = new Server($adapter); try { $worker->init()->action(function () use ($worker, $registerWorkerMessageResources, $queueName) { - $registerWorkerMessageResources($worker->getContainer()); + $registerWorkerMessageResources($worker->context()); Span::init("worker.{$queueName}"); }); @@ -103,7 +111,7 @@ try { $container->set('bus', function ($register) use ($worker) { return $register->get('bus')->setResolver( - fn (string $name) => $worker->getContainer()->get($name) + fn (string $name) => $worker->context()->get($name) ); }, ['register']); diff --git a/composer.json b/composer.json index 0b9b2c8e1b..b78637ccd9 100644 --- a/composer.json +++ b/composer.json @@ -52,7 +52,7 @@ "appwrite/php-runtimes": "0.20.*", "appwrite/php-clamav": "2.0.*", "utopia-php/abuse": "1.3.*", - "utopia-php/agents": "1.2.*", + "utopia-php/agents": "2.*", "utopia-php/analytics": "0.15.*", "utopia-php/audit": "^2.4", "utopia-php/auth": "0.5.*", @@ -63,29 +63,29 @@ "utopia-php/console": "0.1.*", "utopia-php/database": "5.*", "utopia-php/detector": "0.2.*", - "utopia-php/domains": "^2.1", + "utopia-php/domains": "^2.2", "utopia-php/emails": "0.7.*", "utopia-php/dns": "1.7.*", "utopia-php/dsn": "0.2.1", - "utopia-php/http": "2.0.0-rc3", + "utopia-php/http": "^2.0@RC", "utopia-php/fetch": "^1.1", "utopia-php/validators": "0.2.*", "utopia-php/image": "0.8.*", "utopia-php/locale": "0.8.*", "utopia-php/lock": "0.2.*", "utopia-php/logger": "0.8.*", - "utopia-php/messaging": "0.22.*", - "utopia-php/migration": "dev-add-webhook-migration as 1.12.0", - "utopia-php/platform": "1.0.0-rc3", + "utopia-php/messaging": "1.0.*", + "utopia-php/migration": "1.*", + "utopia-php/platform": "1.0.0-rc4", "utopia-php/pools": "1.*", "utopia-php/span": "1.1.*", "utopia-php/preloader": "0.2.*", - "utopia-php/queue": "0.18.*", + "utopia-php/queue": "0.19.*", "utopia-php/servers": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/storage": "2.*", "utopia-php/system": "0.10.*", - "utopia-php/telemetry": "0.2.*", + "utopia-php/telemetry": "^0.4", "utopia-php/vcs": "4.*", "utopia-php/websocket": "1.0.*", "matomo/device-detector": "6.4.*", @@ -113,17 +113,11 @@ "ext-phpiredis": "*" }, "config": { - "platform": {}, + "platform": { + }, "allow-plugins": { "php-http/discovery": true, "tbachert/spi": true } - }, - "repositories": [ - { - "name": "migration", - "type": "vcs", - "url": "https://github.com/utopia-php/migration" - } - ] + } } diff --git a/composer.lock b/composer.lock index 537fb56ea3..335558fa9f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7bd4c82e7547f2983f272c86787b4619", + "content-hash": "0c23c30bba86ad84377d4d1abce0992a", "packages": [ { "name": "adhocore/jwt", @@ -69,16 +69,16 @@ }, { "name": "appwrite/appwrite", - "version": "23.1.1", + "version": "24.2.0", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-for-php.git", - "reference": "fd7c0f0bf5ddf334533534b20ed967cfb400f6ea" + "reference": "d3217b6524b9d174ab91fb7ffc555c68bfa1da97" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-for-php/zipball/fd7c0f0bf5ddf334533534b20ed967cfb400f6ea", - "reference": "fd7c0f0bf5ddf334533534b20ed967cfb400f6ea", + "url": "https://api.github.com/repos/appwrite/sdk-for-php/zipball/d3217b6524b9d174ab91fb7ffc555c68bfa1da97", + "reference": "d3217b6524b9d174ab91fb7ffc555c68bfa1da97", "shasum": "" }, "require": { @@ -104,10 +104,10 @@ "support": { "email": "team@appwrite.io", "issues": "https://github.com/appwrite/sdk-for-php/issues", - "source": "https://github.com/appwrite/sdk-for-php/tree/23.1.1", + "source": "https://github.com/appwrite/sdk-for-php/tree/24.2.0", "url": "https://appwrite.io/support" }, - "time": "2026-05-12T11:03:36+00:00" + "time": "2026-05-21T18:56:51+00:00" }, { "name": "appwrite/php-clamav", @@ -3359,20 +3359,20 @@ }, { "name": "utopia-php/abuse", - "version": "1.3.0", + "version": "1.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "5d7efbe5c6b0cf7d06003114fd86e24ba785582f" + "reference": "00e59c89a1ac5f4ad771695e5dbee020b3c7ac44" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/5d7efbe5c6b0cf7d06003114fd86e24ba785582f", - "reference": "5d7efbe5c6b0cf7d06003114fd86e24ba785582f", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/00e59c89a1ac5f4ad771695e5dbee020b3c7ac44", + "reference": "00e59c89a1ac5f4ad771695e5dbee020b3c7ac44", "shasum": "" }, "require": { - "appwrite/appwrite": "23.*", + "appwrite/appwrite": "24.*", "ext-curl": "*", "ext-pdo": "*", "ext-redis": "*", @@ -3405,22 +3405,22 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/1.3.0" + "source": "https://github.com/utopia-php/abuse/tree/1.3.1" }, - "time": "2026-05-11T08:07:02+00:00" + "time": "2026-06-03T07:30:05+00:00" }, { "name": "utopia-php/agents", - "version": "1.2.2", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/utopia-php/agents.git", - "reference": "0703f4cae02261e09a1bf0d39a4b1ce649cae634" + "reference": "c9785833948639c09c8ccc3e954178c66ed9a896" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/agents/zipball/0703f4cae02261e09a1bf0d39a4b1ce649cae634", - "reference": "0703f4cae02261e09a1bf0d39a4b1ce649cae634", + "url": "https://api.github.com/repos/utopia-php/agents/zipball/c9785833948639c09c8ccc3e954178c66ed9a896", + "reference": "c9785833948639c09c8ccc3e954178c66ed9a896", "shasum": "" }, "require": { @@ -3458,9 +3458,9 @@ ], "support": { "issues": "https://github.com/utopia-php/agents/issues", - "source": "https://github.com/utopia-php/agents/tree/1.2.2" + "source": "https://github.com/utopia-php/agents/tree/2.1.0" }, - "time": "2026-05-08T10:38:23+00:00" + "time": "2026-05-29T10:44:54+00:00" }, { "name": "utopia-php/analytics", @@ -3669,16 +3669,16 @@ }, { "name": "utopia-php/circuit-breaker", - "version": "0.3.0", + "version": "0.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/circuit-breaker.git", - "reference": "064243c1667778c00abf027ff53a735a228776de" + "reference": "db5d77f6c99ebce2ee81bd8ed4ae8f41bd2b0828" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/circuit-breaker/zipball/064243c1667778c00abf027ff53a735a228776de", - "reference": "064243c1667778c00abf027ff53a735a228776de", + "url": "https://api.github.com/repos/utopia-php/circuit-breaker/zipball/db5d77f6c99ebce2ee81bd8ed4ae8f41bd2b0828", + "reference": "db5d77f6c99ebce2ee81bd8ed4ae8f41bd2b0828", "shasum": "" }, "require": { @@ -3688,7 +3688,7 @@ "laravel/pint": "^1.29", "phpstan/phpstan": "^2.1", "phpunit/phpunit": "^10.0", - "utopia-php/telemetry": "0.2.*" + "utopia-php/telemetry": "^0.4" }, "suggest": { "ext-opentelemetry": "Required by utopia-php/telemetry when using OpenTelemetry metrics.", @@ -3725,9 +3725,9 @@ ], "support": { "issues": "https://github.com/utopia-php/circuit-breaker/issues", - "source": "https://github.com/utopia-php/circuit-breaker/tree/0.3.0" + "source": "https://github.com/utopia-php/circuit-breaker/tree/0.3.1" }, - "time": "2026-05-12T04:27:08+00:00" + "time": "2026-05-29T12:12:23+00:00" }, { "name": "utopia-php/cli", @@ -4136,16 +4136,16 @@ }, { "name": "utopia-php/domains", - "version": "2.1.0", + "version": "2.2.0", "source": { "type": "git", "url": "https://github.com/utopia-php/domains.git", - "reference": "1b1fea8674e8712e0344d3abb5a7acd558dede50" + "reference": "77fcf4b63e33b2c1ddf8c67cfc1a57eef2dbec8f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/domains/zipball/1b1fea8674e8712e0344d3abb5a7acd558dede50", - "reference": "1b1fea8674e8712e0344d3abb5a7acd558dede50", + "url": "https://api.github.com/repos/utopia-php/domains/zipball/77fcf4b63e33b2c1ddf8c67cfc1a57eef2dbec8f", + "reference": "77fcf4b63e33b2c1ddf8c67cfc1a57eef2dbec8f", "shasum": "" }, "require": { @@ -4192,9 +4192,9 @@ ], "support": { "issues": "https://github.com/utopia-php/domains/issues", - "source": "https://github.com/utopia-php/domains/tree/2.1.0" + "source": "https://github.com/utopia-php/domains/tree/2.2.0" }, - "time": "2026-05-14T14:33:46+00:00" + "time": "2026-05-28T15:16:46+00:00" }, { "name": "utopia-php/dsn", @@ -4346,16 +4346,16 @@ }, { "name": "utopia-php/http", - "version": "2.0.0-rc3", + "version": "2.0.0-rc4", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "e1a51a2eb906a3d461378fb551a647ef01714f1c" + "reference": "0396a959c6a3be7c16b4b72283347327a75d1f92" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/e1a51a2eb906a3d461378fb551a647ef01714f1c", - "reference": "e1a51a2eb906a3d461378fb551a647ef01714f1c", + "url": "https://api.github.com/repos/utopia-php/http/zipball/0396a959c6a3be7c16b4b72283347327a75d1f92", + "reference": "0396a959c6a3be7c16b4b72283347327a75d1f92", "shasum": "" }, "require": { @@ -4363,7 +4363,7 @@ "utopia-php/compression": "0.1.*", "utopia-php/di": "0.3.*", "utopia-php/servers": "0.4.*", - "utopia-php/telemetry": "0.2.*", + "utopia-php/telemetry": "^0.4", "utopia-php/validators": "0.2.*" }, "require-dev": { @@ -4396,9 +4396,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/2.0.0-rc3" + "source": "https://github.com/utopia-php/http/tree/2.0.0-rc4" }, - "time": "2026-05-21T14:42:41+00:00" + "time": "2026-06-02T04:17:22+00:00" }, { "name": "utopia-php/image", @@ -4606,16 +4606,16 @@ }, { "name": "utopia-php/messaging", - "version": "0.22.3", + "version": "1.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "67366d5f45cc92efe7adb6aab5d6dcd2342f2f9e" + "reference": "1aca8968c3bf3c78c428a399dfe97cbe9b366ec4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/67366d5f45cc92efe7adb6aab5d6dcd2342f2f9e", - "reference": "67366d5f45cc92efe7adb6aab5d6dcd2342f2f9e", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/1aca8968c3bf3c78c428a399dfe97cbe9b366ec4", + "reference": "1aca8968c3bf3c78c428a399dfe97cbe9b366ec4", "shasum": "" }, "require": { @@ -4651,26 +4651,26 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.22.3" + "source": "https://github.com/utopia-php/messaging/tree/1.0.0" }, - "time": "2026-05-19T05:31:20+00:00" + "time": "2026-06-02T14:54:22+00:00" }, { "name": "utopia-php/migration", - "version": "dev-add-webhook-migration", + "version": "1.13.0", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "a351e83121d0241737d99102e4d63e15c75da5b0" + "reference": "0395af61922b52fb73bd9dbfb2dd10ac7aed2b01" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/a351e83121d0241737d99102e4d63e15c75da5b0", - "reference": "a351e83121d0241737d99102e4d63e15c75da5b0", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/0395af61922b52fb73bd9dbfb2dd10ac7aed2b01", + "reference": "0395af61922b52fb73bd9dbfb2dd10ac7aed2b01", "shasum": "" }, "require": { - "appwrite/appwrite": "23.*", + "appwrite/appwrite": "24.*", "ext-curl": "*", "ext-openssl": "*", "halaxa/json-machine": "^1.2", @@ -4692,25 +4692,7 @@ "Utopia\\Migration\\": "src/Migration" } }, - "autoload-dev": { - "psr-4": { - "Utopia\\Tests\\": "tests/Migration" - } - }, - "scripts": { - "test": [ - "./vendor/bin/phpunit" - ], - "lint": [ - "./vendor/bin/pint --test" - ], - "format": [ - "./vendor/bin/pint" - ], - "check": [ - "./vendor/bin/phpstan analyse --level 3 src tests --memory-limit 2G" - ] - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -4723,10 +4705,10 @@ "utopia" ], "support": { - "source": "https://github.com/utopia-php/migration/tree/add-webhook-migration", - "issues": "https://github.com/utopia-php/migration/issues" + "issues": "https://github.com/utopia-php/migration/issues", + "source": "https://github.com/utopia-php/migration/tree/1.13.0" }, - "time": "2026-05-27T20:34:30+00:00" + "time": "2026-06-03T06:15:27+00:00" }, { "name": "utopia-php/mongo", @@ -4791,16 +4773,16 @@ }, { "name": "utopia-php/platform", - "version": "1.0.0-rc3", + "version": "1.0.0-rc4", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9" + "reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/674fd3cea6d1f8db660b01c5be89e8333dfb6de9", - "reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0", + "reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0", "shasum": "" }, "require": { @@ -4809,7 +4791,7 @@ "php": ">=8.3", "utopia-php/cli": "0.23.*", "utopia-php/http": "^2.0@RC", - "utopia-php/queue": "0.18.*", + "utopia-php/queue": "^0.19", "utopia-php/servers": "0.4.*" }, "require-dev": { @@ -4836,27 +4818,27 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc3" + "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc4" }, - "time": "2026-05-25T11:48:18+00:00" + "time": "2026-06-02T20:19:26+00:00" }, { "name": "utopia-php/pools", - "version": "1.0.3", + "version": "1.0.4", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "74de7c5457a2c447f27e7ec4d72e8412a7d68c10" + "reference": "19d8b82f6772a901eb1e16aa7c0e7dfba0ba711b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/74de7c5457a2c447f27e7ec4d72e8412a7d68c10", - "reference": "74de7c5457a2c447f27e7ec4d72e8412a7d68c10", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/19d8b82f6772a901eb1e16aa7c0e7dfba0ba711b", + "reference": "19d8b82f6772a901eb1e16aa7c0e7dfba0ba711b", "shasum": "" }, "require": { "php": ">=8.4", - "utopia-php/telemetry": "*" + "utopia-php/telemetry": "^0.4" }, "require-dev": { "laravel/pint": "1.*", @@ -4889,9 +4871,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/1.0.3" + "source": "https://github.com/utopia-php/pools/tree/1.0.4" }, - "time": "2026-02-26T08:42:40+00:00" + "time": "2026-06-03T05:20:57+00:00" }, { "name": "utopia-php/preloader", @@ -4994,25 +4976,26 @@ }, { "name": "utopia-php/queue", - "version": "0.18.4", + "version": "0.19.0", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "1323b52f39b899c580dbb7af8562fc53d4e13e62" + "reference": "cd4248af69006b2568c3486a675f27cb21191859" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/1323b52f39b899c580dbb7af8562fc53d4e13e62", - "reference": "1323b52f39b899c580dbb7af8562fc53d4e13e62", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/cd4248af69006b2568c3486a675f27cb21191859", + "reference": "cd4248af69006b2568c3486a675f27cb21191859", "shasum": "" }, "require": { "php": ">=8.3", "php-amqplib/php-amqplib": "^3.7", "utopia-php/di": "0.3.*", + "utopia-php/lock": "0.2.*", "utopia-php/pools": "1.*", "utopia-php/servers": "0.4.*", - "utopia-php/telemetry": "0.2.*", + "utopia-php/telemetry": "0.4.*", "utopia-php/validators": "0.2.*" }, "require-dev": { @@ -5054,9 +5037,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.18.4" + "source": "https://github.com/utopia-php/queue/tree/0.19.0" }, - "time": "2026-05-21T14:43:24+00:00" + "time": "2026-06-02T20:12:18+00:00" }, { "name": "utopia-php/registry", @@ -5210,16 +5193,16 @@ }, { "name": "utopia-php/storage", - "version": "2.0.3", + "version": "2.0.5", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "37129cf0bfcc03210172000e4388d4d3495ae013" + "reference": "b8af52e8ae1f9e670b32255a8936bbb46634b711" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/37129cf0bfcc03210172000e4388d4d3495ae013", - "reference": "37129cf0bfcc03210172000e4388d4d3495ae013", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/b8af52e8ae1f9e670b32255a8936bbb46634b711", + "reference": "b8af52e8ae1f9e670b32255a8936bbb46634b711", "shasum": "" }, "require": { @@ -5229,7 +5212,7 @@ "ext-zlib": "*", "php": ">=8.1", "utopia-php/system": "0.*.*", - "utopia-php/telemetry": "0.2.*", + "utopia-php/telemetry": "^0.4", "utopia-php/validators": "0.2.*" }, "require-dev": { @@ -5256,9 +5239,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/2.0.3" + "source": "https://github.com/utopia-php/storage/tree/2.0.5" }, - "time": "2026-05-15T09:42:32+00:00" + "time": "2026-06-03T05:21:46+00:00" }, { "name": "utopia-php/system", @@ -5318,16 +5301,16 @@ }, { "name": "utopia-php/telemetry", - "version": "0.2.0", + "version": "0.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/telemetry.git", - "reference": "9997ebf59bb77920a7223ad73d834a76b09152c3" + "reference": "e0630df7d8176833cd4882f78814a5b893dcb0e0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/9997ebf59bb77920a7223ad73d834a76b09152c3", - "reference": "9997ebf59bb77920a7223ad73d834a76b09152c3", + "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/e0630df7d8176833cd4882f78814a5b893dcb0e0", + "reference": "e0630df7d8176833cd4882f78814a5b893dcb0e0", "shasum": "" }, "require": { @@ -5367,9 +5350,9 @@ ], "support": { "issues": "https://github.com/utopia-php/telemetry/issues", - "source": "https://github.com/utopia-php/telemetry/tree/0.2.0" + "source": "https://github.com/utopia-php/telemetry/tree/0.4.0" }, - "time": "2025-12-17T07:56:38+00:00" + "time": "2026-05-29T11:57:04+00:00" }, { "name": "utopia-php/validators", @@ -5656,16 +5639,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.31.6", + "version": "1.31.8", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "54f8bb457c3076530f48833a01be241847d8ef9a" + "reference": "3bc6c697e60807dd036b5c02c66d03183afad79f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/54f8bb457c3076530f48833a01be241847d8ef9a", - "reference": "54f8bb457c3076530f48833a01be241847d8ef9a", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/3bc6c697e60807dd036b5c02c66d03183afad79f", + "reference": "3bc6c697e60807dd036b5c02c66d03183afad79f", "shasum": "" }, "require": { @@ -5674,7 +5657,7 @@ "ext-mbstring": "*", "matthiasmullie/minify": "1.3.*", "php": ">=8.3", - "twig/twig": "3.26.*" + "twig/twig": "3.27.*" }, "require-dev": { "brianium/paratest": "7.*", @@ -5701,9 +5684,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.31.6" + "source": "https://github.com/appwrite/sdk-generator/tree/1.31.8" }, - "time": "2026-05-24T02:50:38+00:00" + "time": "2026-05-28T05:01:17+00:00" }, { "name": "brianium/paratest", @@ -6412,11 +6395,11 @@ }, { "name": "phpstan/phpstan", - "version": "2.1.56", + "version": "2.1.55", "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/93a603c9fc3be8c3c93bbc8d22170ad766685537", - "reference": "93a603c9fc3be8c3c93bbc8d22170ad766685537", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/9eaac3826ed5e9b8427350a43cac825eeca3f566", + "reference": "9eaac3826ed5e9b8427350a43cac825eeca3f566", "shasum": "" }, "require": { @@ -6461,7 +6444,7 @@ "type": "github" } ], - "time": "2026-05-26T17:04:57+00:00" + "time": "2026-05-18T11:57:34+00:00" }, { "name": "phpunit/php-code-coverage", @@ -6810,16 +6793,16 @@ }, { "name": "phpunit/phpunit", - "version": "12.5.27", + "version": "12.5.25", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "f37c01edaf3a0cd820331462506af93f1495696e" + "reference": "792c2980442dfce319226b88fa845b8b6de3b333" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/f37c01edaf3a0cd820331462506af93f1495696e", - "reference": "f37c01edaf3a0cd820331462506af93f1495696e", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/792c2980442dfce319226b88fa845b8b6de3b333", + "reference": "792c2980442dfce319226b88fa845b8b6de3b333", "shasum": "" }, "require": { @@ -6838,15 +6821,15 @@ "phpunit/php-invoker": "^6.0.0", "phpunit/php-text-template": "^5.0.0", "phpunit/php-timer": "^8.0.0", - "sebastian/cli-parser": "^4.2.1", - "sebastian/comparator": "^7.1.8", + "sebastian/cli-parser": "^4.2.0", + "sebastian/comparator": "^7.1.6", "sebastian/diff": "^7.0.0", - "sebastian/environment": "^8.1.2", - "sebastian/exporter": "^7.0.3", + "sebastian/environment": "^8.1.0", + "sebastian/exporter": "^7.0.2", "sebastian/global-state": "^8.0.2", "sebastian/object-enumerator": "^7.0.0", "sebastian/recursion-context": "^7.0.1", - "sebastian/type": "^6.0.4", + "sebastian/type": "^6.0.3", "sebastian/version": "^6.0.0", "staabm/side-effects-detector": "^1.0.5" }, @@ -6888,7 +6871,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/12.5.27" + "source": "https://github.com/sebastianbergmann/phpunit/tree/12.5.25" }, "funding": [ { @@ -6896,7 +6879,7 @@ "type": "other" } ], - "time": "2026-05-25T15:35:34+00:00" + "time": "2026-05-13T03:56:57+00:00" }, { "name": "sebastian/cli-parser", @@ -6969,16 +6952,16 @@ }, { "name": "sebastian/comparator", - "version": "7.1.8", + "version": "7.1.6", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "7c65c1e79836812819705b473a90c12399542485" + "reference": "c769009dee98f494e0edc3fd4f4087501688f11e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/7c65c1e79836812819705b473a90c12399542485", - "reference": "7c65c1e79836812819705b473a90c12399542485", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/c769009dee98f494e0edc3fd4f4087501688f11e", + "reference": "c769009dee98f494e0edc3fd4f4087501688f11e", "shasum": "" }, "require": { @@ -6986,10 +6969,10 @@ "ext-mbstring": "*", "php": ">=8.3", "sebastian/diff": "^7.0", - "sebastian/exporter": "^7.0.3" + "sebastian/exporter": "^7.0" }, "require-dev": { - "phpunit/phpunit": "^12.5.25" + "phpunit/phpunit": "^12.2" }, "suggest": { "ext-bcmath": "For comparing BcMath\\Number objects" @@ -7037,7 +7020,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/comparator/issues", "security": "https://github.com/sebastianbergmann/comparator/security/policy", - "source": "https://github.com/sebastianbergmann/comparator/tree/7.1.8" + "source": "https://github.com/sebastianbergmann/comparator/tree/7.1.6" }, "funding": [ { @@ -7057,7 +7040,7 @@ "type": "tidelift" } ], - "time": "2026-05-21T04:45:25+00:00" + "time": "2026-04-14T08:23:15+00:00" }, { "name": "sebastian/complexity", @@ -7186,23 +7169,23 @@ }, { "name": "sebastian/environment", - "version": "8.1.2", + "version": "8.1.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/environment.git", - "reference": "9d32c685773823b1983e256ae4ecd48a10d6e439" + "reference": "b121608b28a13f721e76ffbbd386d08eff58f3f6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/9d32c685773823b1983e256ae4ecd48a10d6e439", - "reference": "9d32c685773823b1983e256ae4ecd48a10d6e439", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/b121608b28a13f721e76ffbbd386d08eff58f3f6", + "reference": "b121608b28a13f721e76ffbbd386d08eff58f3f6", "shasum": "" }, "require": { "php": ">=8.3" }, "require-dev": { - "phpunit/phpunit": "^12.5.26" + "phpunit/phpunit": "^12.0" }, "suggest": { "ext-posix": "*" @@ -7238,7 +7221,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/environment/issues", "security": "https://github.com/sebastianbergmann/environment/security/policy", - "source": "https://github.com/sebastianbergmann/environment/tree/8.1.2" + "source": "https://github.com/sebastianbergmann/environment/tree/8.1.0" }, "funding": [ { @@ -7258,7 +7241,7 @@ "type": "tidelift" } ], - "time": "2026-05-25T13:40:20+00:00" + "time": "2026-04-15T12:13:01+00:00" }, { "name": "sebastian/exporter", @@ -8066,16 +8049,16 @@ }, { "name": "symfony/polyfill-intl-grapheme", - "version": "v1.38.1", + "version": "v1.37.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", - "reference": "e9247d281d694a5120554d9afaf54e070e88a603" + "reference": "4864388bfbd3001ce88e234fab652acd91fdc57e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/e9247d281d694a5120554d9afaf54e070e88a603", - "reference": "e9247d281d694a5120554d9afaf54e070e88a603", + "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/4864388bfbd3001ce88e234fab652acd91fdc57e", + "reference": "4864388bfbd3001ce88e234fab652acd91fdc57e", "shasum": "" }, "require": { @@ -8124,7 +8107,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.38.1" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.37.0" }, "funding": [ { @@ -8144,20 +8127,20 @@ "type": "tidelift" } ], - "time": "2026-05-26T05:58:03+00:00" + "time": "2026-04-26T13:13:48+00:00" }, { "name": "symfony/polyfill-intl-normalizer", - "version": "v1.38.0", + "version": "v1.37.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-normalizer.git", - "reference": "2d446c214bdbe5b71bde5011b060a05fece3ae6b" + "reference": "3833d7255cc303546435cb650316bff708a1c75c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/2d446c214bdbe5b71bde5011b060a05fece3ae6b", - "reference": "2d446c214bdbe5b71bde5011b060a05fece3ae6b", + "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/3833d7255cc303546435cb650316bff708a1c75c", + "reference": "3833d7255cc303546435cb650316bff708a1c75c", "shasum": "" }, "require": { @@ -8209,7 +8192,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.38.0" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.37.0" }, "funding": [ { @@ -8229,7 +8212,7 @@ "type": "tidelift" } ], - "time": "2026-05-25T13:48:31+00:00" + "time": "2024-09-09T11:45:10+00:00" }, { "name": "symfony/process", @@ -8487,16 +8470,16 @@ }, { "name": "twig/twig", - "version": "v3.26.0", + "version": "v3.27.0", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "1fcae487b180d78e6351f4e0afa91f9eab96a2bc" + "reference": "04ae1bfe9463c816cf72ca0abe7eae2c77a9a9ed" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/1fcae487b180d78e6351f4e0afa91f9eab96a2bc", - "reference": "1fcae487b180d78e6351f4e0afa91f9eab96a2bc", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/04ae1bfe9463c816cf72ca0abe7eae2c77a9a9ed", + "reference": "04ae1bfe9463c816cf72ca0abe7eae2c77a9a9ed", "shasum": "" }, "require": { @@ -8551,7 +8534,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v3.26.0" + "source": "https://github.com/twigphp/Twig/tree/v3.27.0" }, "funding": [ { @@ -8563,20 +8546,13 @@ "type": "tidelift" } ], - "time": "2026-05-20T07:31:59+00:00" - } - ], - "aliases": [ - { - "package": "utopia-php/migration", - "version": "dev-add-webhook-migration", - "alias": "1.12.0", - "alias_normalized": "1.12.0.0" + "time": "2026-05-27T13:05:51+00:00" } ], + "aliases": [], "minimum-stability": "dev", "stability-flags": { - "utopia-php/migration": 20 + "utopia-php/http": 5 }, "prefer-stable": true, "prefer-lowest": false, diff --git a/docker-compose.yml b/docker-compose.yml index 7d3f6bdc5f..90bea1dc5d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,7 +112,7 @@ services: condition: service_healthy coredns: condition: service_started - ollama: + appwrite-embedding: condition: service_started entrypoint: - php @@ -304,7 +304,7 @@ services: depends_on: - ${_APP_DB_HOST:-mongodb} - redis - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -346,9 +346,11 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} - - ollama + - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -380,9 +382,11 @@ services: - ${_APP_DB_HOST:-mongodb} - request-catcher-sms - request-catcher-webhook - - ollama + - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -411,7 +415,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} - - ollama + - appwrite-embedding volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw @@ -424,6 +428,8 @@ services: environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -481,9 +487,11 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} - - ollama + - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -504,7 +512,6 @@ services: - _APP_DB_USER_VECTORSDB - _APP_DB_PASS_VECTORSDB - _APP_LOGGING_CONFIG - - _APP_WORKERS_NUM - _APP_QUEUE_NAME - _APP_DATABASE_SHARED_TABLES @@ -524,9 +531,11 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} - - ollama + - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -605,6 +614,8 @@ services: - _APP_OPTIONS_FORCE_HTTPS # Basic - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_LOGGING_CONFIG @@ -657,7 +668,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} - - ollama + - appwrite-embedding volumes: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -665,6 +676,8 @@ services: - ./src:/usr/src/code/src environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -705,6 +718,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -736,6 +751,8 @@ services: - redis environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -782,6 +799,8 @@ services: # - smtp environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -823,6 +842,8 @@ services: - redis environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -883,6 +904,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1044,6 +1067,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1075,9 +1100,11 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} - - ollama + - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1109,7 +1136,7 @@ services: depends_on: - ${_APP_DB_HOST:-mongodb} - redis - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -1147,7 +1174,7 @@ services: depends_on: - ${_APP_DB_HOST:-mongodb} - redis - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -1185,7 +1212,7 @@ services: depends_on: - ${_APP_DB_HOST:-mongodb} - redis - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -1358,17 +1385,17 @@ services: start_period: 10s command: "postgres" - ollama: - image: appwrite/ollama:0.1.1 - container_name: ollama - ports: - - "11434:11434" - restart: unless-stopped + appwrite-embedding: + image: appwrite/embedding:0.1.0 environment: - MODELS: ${_APP_EMBEDDING_MODELS:-embeddinggemma} - OLLAMA_KEEP_ALIVE: 24h + EMBEDDING_MODELS: ${_APP_EMBEDDING_MODELS:-nomic-embed-text} + container_name: appwrite-embedding + ports: + - "3000:3000" + restart: unless-stopped + # persistent for caching models across restarts and preloading volumes: - - appwrite-models:/root/.ollama + - appwrite-models:/home/embedder/models networks: - appwrite diff --git a/docs/references/messaging/create-ses-provider.md b/docs/references/messaging/create-ses-provider.md new file mode 100644 index 0000000000..c612414f99 --- /dev/null +++ b/docs/references/messaging/create-ses-provider.md @@ -0,0 +1 @@ +Create a new Amazon SES provider. \ No newline at end of file diff --git a/docs/references/messaging/update-ses-provider.md b/docs/references/messaging/update-ses-provider.md new file mode 100644 index 0000000000..79947186e8 --- /dev/null +++ b/docs/references/messaging/update-ses-provider.md @@ -0,0 +1 @@ +Update an Amazon SES provider by its unique ID. \ No newline at end of file diff --git a/src/Appwrite/Database/Factory.php b/src/Appwrite/Database/Factory.php new file mode 100644 index 0000000000..93b4e4ff53 --- /dev/null +++ b/src/Appwrite/Database/Factory.php @@ -0,0 +1,232 @@ +newDatabase($this->adapter('console')); + + $database + ->setDatabase($this->database) + ->setAuthorization($this->authorization) + ->setNamespace($this->platformNamespace); + + $this->configureDocumentTypes($database); + $this->configureOptions($database, $timeout, $maxQueryValues, $metadata); + + return $database; + } + + public function project( + Document $project, + int $timeout = 0, + int $maxQueryValues = 0, + array $metadata = [], + ): Database { + $dsn = $this->dsn($project->getAttribute('database')); + $database = $this->newDatabase( + $this->adapter($dsn->getHost()), + $this->destinationFor($dsn) + ); + + $database + ->setDatabase($this->database) + ->setAuthorization($this->authorization); + + $this->configureDocumentTypes($database); + $this->configureOptions($database, $timeout, $maxQueryValues, $metadata); + + return $this->configureProject($database, $project, $dsn); + } + + public function logs( + ?Document $project = null, + int $timeout = 0, + int $maxQueryValues = 0, + array $metadata = [], + ): Database { + /** @var array $collections */ + $collections = Config::getParam('collections', []); + $logsCollections = \array_keys($collections['logs'] ?? []); + + $database = $this->newDatabase($this->adapter('logs')); + + $database + ->setDatabase($this->database) + ->setAuthorization($this->authorization) + ->setSharedTables(true) + ->setGlobalCollections($logsCollections) + ->setNamespace('logsV1'); + + if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant($project->getSequence()); + } + + $this->configureOptions($database, $timeout, $maxQueryValues, $metadata); + + return $database; + } + + public function tenant( + Document $databaseDocument, + Document $project, + int $timeout = 0, + int $maxQueryValues = 0, + array $metadata = [], + bool $preserveDates = false, + ): Database { + $databaseDsn = $this->dsn($databaseDocument->getAttribute('database') ?: $project->getAttribute('database', '')); + $projectDsn = $this->dsn($project->getAttribute('database')); + $databaseType = $databaseDocument->getAttribute('type', ''); + + $database = $this->newDatabase($this->adapter($databaseDsn->getHost())); + + $database + ->setDatabase($this->database) + ->setAuthorization($this->authorization); + + $database->getAdapter()->setSupportForAttributes($databaseType !== DOCUMENTSDB); + + if ($preserveDates) { + $database->setPreserveDates(true); + } + + $this->configureOptions($database, $timeout, $maxQueryValues, $metadata); + + $sharedTables = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''))); + $globalCollections = $this->projectGlobalCollections(); + $databaseHost = $databaseDsn->getHost(); + + if ($databaseHost !== $projectDsn->getHost()) { + $dbTypeSharedTables = match ($databaseType) { + DOCUMENTSDB => \array_filter(\explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES', ''))), + VECTORSDB => \array_filter(\explode(',', System::getEnv('_APP_DATABASE_VECTORSDB_SHARED_TABLES', ''))), + default => [], + }; + + if (\in_array($databaseHost, $dbTypeSharedTables, true)) { + return $database + ->setSharedTables(true) + ->setGlobalCollections($globalCollections) + ->setTenant($project->getSequence()) + ->setNamespace($databaseDsn->getParam('namespace')); + } + + return $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getSequence()); + } + + if (\in_array($projectDsn->getHost(), $sharedTables, true)) { + return $database + ->setSharedTables(true) + ->setGlobalCollections($globalCollections) + ->setTenant($project->getSequence()) + ->setNamespace($projectDsn->getParam('namespace')); + } + + return $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getSequence()); + } + + protected function newDatabase(Adapter $adapter, ?Database $destination = null): Database + { + return new Database($adapter, $this->cache); + } + + protected function adapter(string $name): DatabasePool + { + return new DatabasePool($this->pools->get($name)); + } + + protected function configureDocumentTypes(Database $database): Database + { + return $database->setDocumentType('users', User::class); + } + + protected function destinationFor(DSN $dsn): ?Database + { + return null; + } + + protected function configureProject(Database $database, Document $project, DSN $dsn): Database + { + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + if (\in_array($dsn->getHost(), $sharedTables, true)) { + return $database + ->setSharedTables(true) + ->setGlobalCollections($this->projectGlobalCollections()) + ->setTenant($project->getSequence()) + ->setNamespace($dsn->getParam('namespace')); + } + + return $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getSequence()); + } + + private function configureOptions(Database $database, int $timeout, int $maxQueryValues, array $metadata): void + { + foreach ($metadata as $key => $value) { + $database->setMetadata($key, $value); + } + + if ($timeout > 0) { + $database->setTimeout($timeout); + } + + if ($maxQueryValues > 0) { + $database->setMaxQueryValues($maxQueryValues); + } + } + + private function dsn(?string $value): DSN + { + try { + return new DSN($value ?? ''); + } catch (\InvalidArgumentException) { + return new DSN('mysql://' . $value); + } + } + + private function projectGlobalCollections(): array + { + /** @var array $collections */ + $collections = Config::getParam('collections', []); + $globalCollections = \array_keys($collections['projects'] ?? []); + $globalCollections[] = 'audit'; + + return $globalCollections; + } +} diff --git a/src/Appwrite/Event/Realtime.php b/src/Appwrite/Event/Realtime.php index f040d91468..1b21ef1e44 100644 --- a/src/Appwrite/Event/Realtime.php +++ b/src/Appwrite/Event/Realtime.php @@ -10,6 +10,8 @@ use Utopia\Database\Exception; class Realtime extends Event { + public const CONSOLE_ALLOWLIST = ['presences']; + protected array $subscribers = []; private Adapter $realtime; diff --git a/src/Appwrite/Network/Validator/PublicHostname.php b/src/Appwrite/Network/Validator/PublicHostname.php new file mode 100644 index 0000000000..e807315521 --- /dev/null +++ b/src/Appwrite/Network/Validator/PublicHostname.php @@ -0,0 +1,202 @@ +reason !== '' + ? $this->reason + : 'Value must be a publicly routable hostname or address.'; + } + + public function isArray(): bool + { + return false; + } + + public function getType(): string + { + return self::TYPE_STRING; + } + + public function isValid(mixed $value): bool + { + $this->reason = ''; + + if (!\is_string($value) || $value === '') { + $this->reason = 'Hostname is empty.'; + return false; + } + + $hostname = \strtolower(\trim($value, " \t\n\r\0\x0B[]")); + + // IP literals are checked directly, no DNS round-trip. + if (\filter_var($hostname, FILTER_VALIDATE_IP) !== false) { + if (!self::isPublicIp($hostname)) { + $this->reason = "Address {$hostname} is in a private or reserved range."; + return false; + } + return true; + } + + $addresses = self::resolve($hostname); + + if (empty($addresses)) { + $this->reason = "Hostname {$hostname} does not resolve."; + return false; + } + + foreach ($addresses as $ip) { + if (!self::isPublicIp($ip)) { + $this->reason = "Hostname {$hostname} resolves to private or reserved address {$ip}."; + return false; + } + } + + return true; + } + + /** + * Resolves a hostname to all A and AAAA records. + * + * @return array + */ + public static function resolve(string $hostname): array + { + $ipv4 = []; + $ipv6 = []; + + $records = @\dns_get_record($hostname, DNS_A | DNS_AAAA); + if (\is_array($records)) { + foreach ($records as $record) { + if (!empty($record['ip'])) { + $ipv4[] = $record['ip']; + } + + if (!empty($record['ipv6'])) { + $ipv6[] = $record['ipv6']; + } + } + } + + return \array_values(\array_unique([...$ipv4, ...$ipv6])); + } + + /** + * Returns true only if the given IP literal sits in a globally routable range. + */ + public static function isPublicIp(string $ip): bool + { + $filtered = \filter_var( + $ip, + FILTER_VALIDATE_IP, + FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE + ); + + if ($filtered === false) { + return false; + } + + $isIpv4 = \filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) !== false; + $extra = $isIpv4 ? self::PRIVATE_IPV4_CIDRS : self::PRIVATE_IPV6_CIDRS; + + foreach ($extra as $cidr) { + if (self::ipInCidr($ip, $cidr)) { + return false; + } + } + + return true; + } + + /** + * Binary CIDR match. Works for both IPv4 and IPv6 as long as the + * IP and subnet are the same family. + */ + private static function ipInCidr(string $ip, string $cidr): bool + { + [$subnet, $maskBits] = \explode('/', $cidr, 2); + $maskBits = (int) $maskBits; + + $ipBinary = @\inet_pton($ip); + $subnetBinary = @\inet_pton($subnet); + + if ($ipBinary === false || $subnetBinary === false) { + return false; + } + + if (\strlen($ipBinary) !== \strlen($subnetBinary)) { + return false; + } + + $fullBytes = \intdiv($maskBits, 8); + $remainderBits = $maskBits % 8; + + if ($fullBytes > 0 && \substr($ipBinary, 0, $fullBytes) !== \substr($subnetBinary, 0, $fullBytes)) { + return false; + } + + if ($remainderBits > 0) { + $mask = \chr(0xFF << (8 - $remainderBits) & 0xFF); + if ((\ord($ipBinary[$fullBytes]) & \ord($mask)) !== (\ord($subnetBinary[$fullBytes]) & \ord($mask))) { + return false; + } + } + + return true; + } +} diff --git a/src/Appwrite/Platform/Modules/Avatars/Http/Favicon/Get.php b/src/Appwrite/Platform/Modules/Avatars/Http/Favicon/Get.php index 31ad572f18..6faa0fe223 100644 --- a/src/Appwrite/Platform/Modules/Avatars/Http/Favicon/Get.php +++ b/src/Appwrite/Platform/Modules/Avatars/Http/Favicon/Get.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Avatars\Http\Favicon; use Appwrite\Extend\Exception; +use Appwrite\Network\Validator\PublicHostname; use Appwrite\Platform\Modules\Avatars\Http\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -15,7 +16,9 @@ use DOMDocument; use DOMElement; use enshrined\svgSanitize\Sanitizer as SvgSanitizer; use Utopia\Domains\Domain; +use Utopia\Fetch\Adapter; use Utopia\Fetch\Client; +use Utopia\Fetch\Response as FetchResponse; use Utopia\Image\Image; use Utopia\Platform\Action as UtopiaAction; use Utopia\Platform\Scope\HTTP; @@ -26,6 +29,9 @@ class Get extends Action { use HTTP; + private const ALLOWED_SCHEMES = ['http', 'https']; + private const MAX_REDIRECTS = 5; + public static function getName(): string { return 'getFavicon'; @@ -56,7 +62,7 @@ class Get extends Action ], contentType: ContentType::IMAGE )) - ->param('url', '', new URL(['http', 'https']), 'Website URL which you want to fetch the favicon from.') + ->param('url', '', new URL(self::ALLOWED_SCHEMES), 'Website URL which you want to fetch the favicon from.') ->inject('response') ->callback($this->action(...)); } @@ -67,34 +73,25 @@ class Get extends Action $height = 56; $quality = 80; $output = 'png'; - $type = 'png'; if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); } - $domain = new Domain(\parse_url($url, PHP_URL_HOST)); + $userAgent = \sprintf( + APP_USERAGENT, + System::getEnv('_APP_VERSION', 'UNKNOWN'), + System::getEnv('_APP_EMAIL_SECURITY', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY)) + ); - if (!$domain->isKnown()) { - throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); - } - - $client = new Client(); try { - $res = $client - ->setAllowRedirects(true) - ->setMaxRedirects(5) - ->setUserAgent(\sprintf( - APP_USERAGENT, - System::getEnv('_APP_VERSION', 'UNKNOWN'), - System::getEnv('_APP_EMAIL_SECURITY', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY)) - )) - ->fetch($url); + $pageResponse = $this->safeFetch($url, $userAgent); } catch (\Throwable) { throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); } - $body = $res->getBody(); + $body = $pageResponse->getBody(); + $doc = new DOMDocument(); $doc->strictErrorChecking = false; if (!empty($body)) { @@ -110,12 +107,11 @@ class Get extends Action $href = $link->getAttribute('href'); $rel = $link->getAttribute('rel'); $sizes = $link->getAttribute('sizes'); - $absolute = URLParse::unparse(\array_merge(\parse_url($url), \parse_url($href))); + $absolute = URLParse::resolveLocation($url, $href); switch (\strtolower($rel)) { case 'icon': case 'shortcut icon': - //case 'apple-touch-icon': $ext = \pathinfo(\parse_url($absolute, PHP_URL_PATH), PATHINFO_EXTENSION); switch ($ext) { @@ -154,27 +150,17 @@ class Get extends Action $outputExt = 'ico'; } - $domain = new Domain(\parse_url($outputHref, PHP_URL_HOST)); - - if (!$domain->isKnown()) { - throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); - } - - $client = new Client(); try { - $res = $client - ->setAllowRedirects(true) - ->setMaxRedirects(5) - ->fetch($outputHref); + $iconResponse = $this->safeFetch($outputHref, $userAgent); } catch (\Throwable) { throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); } - if ($res->getStatusCode() !== 200) { + if ($iconResponse->getStatusCode() !== 200) { throw new Exception(Exception::AVATAR_ICON_NOT_FOUND); } - $data = $res->getBody(); + $data = $iconResponse->getBody(); if ('ico' === $outputExt) { // Skip crop, Imagick isn\'t supporting icon files if ( @@ -215,4 +201,74 @@ class Get extends Action ->file($data); unset($image); } + + /** + * @throws Exception + */ + protected static function assertSafeUrl(string $url): void + { + $parts = \parse_url($url); + if (!\is_array($parts)) { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED, 'Malformed URL.'); + } + + $scheme = \strtolower($parts['scheme'] ?? ''); + if (!\in_array($scheme, self::ALLOWED_SCHEMES, true)) { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED, "Scheme '{$scheme}' is not allowed."); + } + + $host = $parts['host'] ?? ''; + if ($host === '') { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED, 'URL has no host.'); + } + + $isIpLiteral = \filter_var(\trim($host, '[]'), FILTER_VALIDATE_IP) !== false; + if (!$isIpLiteral) { + try { + $domain = new Domain($host); + } catch (\Throwable) { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED, "Hostname '{$host}' is invalid."); + } + + if (!$domain->isKnown()) { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED, "Hostname '{$host}' is not a known public domain."); + } + } + + $validator = new PublicHostname(); + if (!$validator->isValid($host)) { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED, $validator->getDescription()); + } + } + + /** + * @throws Exception + */ + protected function safeFetch(string $url, string $userAgent, ?Adapter $adapter = null): FetchResponse + { + for ($hop = 0; $hop <= self::MAX_REDIRECTS; $hop++) { + self::assertSafeUrl($url); + + $client = $adapter !== null ? new Client($adapter) : new Client(); + $response = $client + ->setAllowRedirects(false) + ->setUserAgent($userAgent) + ->fetch($url); + + $status = $response->getStatusCode(); + if ($status < 300 || $status >= 400) { + return $response; + } + + $headers = \array_change_key_case($response->getHeaders(), CASE_LOWER); + $location = $headers['location'] ?? ''; + if ($location === '') { + return $response; + } + + $url = URLParse::resolveLocation($url, $location); + } + + throw new \RuntimeException('Too many redirects.'); + } } diff --git a/src/Appwrite/Platform/Modules/Avatars/Http/Image/Get.php b/src/Appwrite/Platform/Modules/Avatars/Http/Image/Get.php index 77ff9dd8ce..315202d4d5 100644 --- a/src/Appwrite/Platform/Modules/Avatars/Http/Image/Get.php +++ b/src/Appwrite/Platform/Modules/Avatars/Http/Image/Get.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Avatars\Http\Image; use Appwrite\Extend\Exception; +use Appwrite\Network\Validator\PublicHostname; use Appwrite\Platform\Modules\Avatars\Http\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -69,10 +70,19 @@ class Get extends Action throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); } - $domain = new Domain(\parse_url($url, PHP_URL_HOST)); + $host = \parse_url($url, PHP_URL_HOST) ?? ''; - if (!$domain->isKnown()) { - throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); + $isIpLiteral = \filter_var(\trim($host, '[]'), FILTER_VALIDATE_IP) !== false; + if (!$isIpLiteral) { + $domain = new Domain($host); + if (!$domain->isKnown()) { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); + } + } + + $hostnameValidator = new PublicHostname(); + if (!$hostnameValidator->isValid($host)) { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED, $hostnameValidator->getDescription()); } $client = new Client(); diff --git a/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php b/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php index 9b2e92efe5..51d4df2a49 100644 --- a/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php +++ b/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Avatars\Http\Screenshots; use Appwrite\Extend\Exception; +use Appwrite\Network\Validator\PublicHostname; use Appwrite\Platform\Modules\Avatars\Http\Action; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -95,10 +96,19 @@ class Get extends Action throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); } - $domain = new Domain(\parse_url($url, PHP_URL_HOST)); + $host = \parse_url($url, PHP_URL_HOST) ?? ''; - if (!$domain->isKnown()) { - throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); + $isIpLiteral = \filter_var(\trim($host, '[]'), FILTER_VALIDATE_IP) !== false; + if (!$isIpLiteral) { + $domain = new Domain($host); + if (!$domain->isKnown()) { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED); + } + } + + $hostnameValidator = new PublicHostname(); + if (!$hostnameValidator->isValid($host)) { + throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED, $hostnameValidator->getDescription()); } $client = new Client(); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Create.php index 3a53a49579..a0ee24db87 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Create.php @@ -172,7 +172,7 @@ class Create extends Action $indexDocuments = []; try { foreach ($indexes as $indexDef) { - $idxDoc = $this->buildIndexDocument($database, $collection, $indexDef, $collectionAttributes); + $idxDoc = $this->buildIndexDocument($database, $collection, $indexDef, $collectionAttributes, $dbForDatabases); $collectionIndexes[] = $idxDoc['collection']; $indexDocuments[] = $idxDoc['document']; } @@ -346,7 +346,7 @@ class Create extends Action * * @return array{collection: Document, document: Document} */ - protected function buildIndexDocument(Document $database, Document $collection, array $indexDef, array $attributeDocuments): array + protected function buildIndexDocument(Document $database, Document $collection, array $indexDef, array $attributeDocuments, Database $dbForDatabases): array { $key = $indexDef['key']; $type = $indexDef['type']; @@ -370,6 +370,11 @@ class Create extends Action if ($attrArray === true) { $lengths[$i] = Database::MAX_ARRAY_INDEX_LENGTH; $orders[$i] = null; + + if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { + // Because of a bug in MySQL, we cannot create indexes on array attributes for now, otherwise queries break. + throw new Exception(Exception::INDEX_INVALID, 'Creating indexes on array attributes is not currently supported.'); + } } } else { if (empty($lengths[$i])) { diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php index b5480f39e3..05512e3151 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php @@ -10,7 +10,7 @@ use Appwrite\SDK\Parameter; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Usage\Context; use Appwrite\Utopia\Response as UtopiaResponse; -use Utopia\Agents\Adapters\Ollama; +use Utopia\Agents\Adapters\Appwrite as AppwriteAdapter; use Utopia\Agents\Agent; use Utopia\Database\Document; use Utopia\Http\Adapter\Swoole\Response as SwooleResponse; @@ -75,7 +75,7 @@ class Create extends CreateDocumentAction ) ]) ->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesMaxEmbeddingTexts'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', false, ['plan']) - ->param('model', Ollama::MODEL_EMBEDDING_GEMMA, new WhiteList(Ollama::MODELS), 'The embedding model to use for generating vector embeddings.', true, enum: new Enum(name: 'EmbeddingModel')) + ->param('model', AppwriteAdapter::MODEL_NOMIC_EMBED_TEXT, new WhiteList(AppwriteAdapter::MODELS), 'The embedding model to use for generating vector embeddings.', true, enum: new Enum(name: 'EmbeddingModel')) ->inject('response') ->inject('project') ->inject('embeddingAgent') @@ -87,50 +87,43 @@ class Create extends CreateDocumentAction public function action(array $texts, string $model, UtopiaResponse $response, Document $project, Agent $embeddingAgent, Context $usage, Log $log, ?Logger $logger): void { - $results = []; - $embeddingAgent->getAdapter()->setModel($model); - $dimension = $embeddingAgent->getAdapter()->getEmbeddingDimension(); + $adapter = $embeddingAgent->getAdapter(); + $adapter->setModel($model); + $dimension = $adapter->getEmbeddingDimension(); + $results = []; $totalDuration = 0; $totalTokens = 0; $totalErrors = 0; - foreach ($texts as $text) { - $embedding = []; - $error = ''; + + foreach (array_chunk($texts, APP_EMBEDDING_BATCH_LIMIT) as $batch) { try { - $embedResult = $embeddingAgent->embed($text); - $embedding = $embedResult['embedding']; + $embedResult = $embeddingAgent->bulkEmbed($batch); $totalDuration += $embedResult['totalDuration'] ?? 0; $totalTokens += $embedResult['tokensProcessed'] ?? 0; + + foreach ($embedResult['embeddings'] as $embedding) { + $results[] = $this->embeddingResult($model, $dimension, $embedding); + } + + // Partial success: backend returned fewer embeddings than inputs. + // Fill the gap with error results so the response stays 1:1 with texts. + $missing = \count($batch) - \count($embedResult['embeddings']); + for ($i = 0; $i < $missing; $i++) { + $totalErrors++; + $results[] = $this->embeddingResult($model, $dimension, [], 'Error while generating embedding'); + } } catch (\Exception $e) { - $error = 'Error while generating embedding'; - $totalErrors += 1; - if ($logger) { - $log->setNamespace("http"); - $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); - $log->setVersion(System::getEnv('_APP_VERSION', 'UNKNOWN')); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($e->getMessage()); + $totalErrors += \count($batch); + $this->logError($e, $model, $project, $log, $logger); - $log->addTag('embeddingModel', $model); - $log->addTag('code', $e->getCode()); - $log->addTag('projectId', $project->getId()); - - $log->addExtra('file', $e->getFile()); - $log->addExtra('line', $e->getLine()); - $log->addExtra('trace', $e->getTraceAsString()); - - $logger->addLog($log); + // One error result per text in the failed batch. + foreach ($batch as $ignored) { + $results[] = $this->embeddingResult($model, $dimension, [], 'Error while generating embedding'); } } - - $results[] = new Document([ - 'model' => $model, - 'dimension' => $dimension, - 'embedding' => $embedding, - 'error' => $error - ]); } + $embeddings = new Document([ 'embeddings' => $results, 'total' => \count($results), @@ -150,4 +143,42 @@ class Create extends CreateDocumentAction ->addMetric(METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR, $totalErrors) ->addMetric(\str_replace('{embeddingModel}', $model, METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_ERROR), $totalErrors); } + + /** + * Build a single embedding response document. + * + * @param float[] $embedding + */ + private function embeddingResult(string $model, int $dimension, array $embedding, string $error = ''): Document + { + return new Document([ + 'model' => $model, + 'dimension' => $dimension, + 'embedding' => $embedding, + 'error' => $error, + ]); + } + + private function logError(\Throwable $e, string $model, Document $project, Log $log, ?Logger $logger): void + { + if ($logger === null) { + return; + } + + $log->setNamespace('http'); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); + $log->setVersion(System::getEnv('_APP_VERSION', 'UNKNOWN')); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($e->getMessage()); + + $log->addTag('embeddingModel', $model); + $log->addTag('code', $e->getCode()); + $log->addTag('projectId', $project->getId()); + + $log->addExtra('file', $e->getFile()); + $log->addExtra('line', $e->getLine()); + $log->addExtra('trace', $e->getTraceAsString()); + + $logger->addLog($log); + } } diff --git a/src/Appwrite/Platform/Modules/Databases/Workers/Databases.php b/src/Appwrite/Platform/Modules/Databases/Workers/Databases.php index ee8494b382..a6b13857a5 100644 --- a/src/Appwrite/Platform/Modules/Databases/Workers/Databases.php +++ b/src/Appwrite/Platform/Modules/Databases/Workers/Databases.php @@ -5,7 +5,6 @@ namespace Appwrite\Platform\Modules\Databases\Workers; use Appwrite\Event\Message\Database as DatabaseMessage; use Appwrite\Event\Realtime; use Exception; -use Utopia\Console; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception as DatabaseException; @@ -18,6 +17,7 @@ use Utopia\Database\Query; use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Span\Span; class Databases extends Action { @@ -62,10 +62,21 @@ class Databases extends Action } $databaseMessage = DatabaseMessage::fromArray($payload); + $type = $databaseMessage->type; + Span::add('payload.type', $type); + $document = $databaseMessage->row ?? $databaseMessage->document ?? new Document(); + Span::add('database.document.id', $document->getId()); + $collection = $databaseMessage->table ?? $databaseMessage->collection ?? new Document(); + Span::add('database.collection.id', $collection->getId()); + $database = $databaseMessage->database ?? new Document(); + Span::add('database.id', $database->getId()); + Span::add('database.namespace', $dbForProject->getNamespace()); + Span::add('database.tenant', $dbForProject->getTenant()); + /** * @var Database $dbForDatabases */ @@ -88,14 +99,6 @@ class Databases extends Action DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $dbForDatabases, $queueForRealtime), default => throw new Exception('No database operation for type: ' . \strval($type)), }; - - Console::info("Finished processing database operation: \n" . \json_encode([ - 'type' => $type, - 'projectId' => $project->getId(), - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId(), - 'documentId' => $document->getId(), - ], JSON_PRETTY_PRINT)); } /** @@ -206,8 +209,6 @@ class Databases extends Action $dbForProject->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'available')); } catch (\Throwable $e) { - Console::error($e->getMessage()); - if ($e instanceof DatabaseException) { $attribute->setAttribute('error', $e->getMessage()); if (! $relatedAttribute->isEmpty()) { @@ -309,7 +310,7 @@ class Databases extends Action } } catch (NotFound $e) { - Console::error($e->getMessage()); + Span::add('database.attribute.not_found', $e->getMessage()); $dbForProject->deleteDocument('attributes', $attribute->getId()); @@ -318,8 +319,6 @@ class Databases extends Action } } catch (\Throwable $e) { - Console::error($e->getMessage()); - if ($e instanceof DatabaseException) { $attribute->setAttribute('error', $e->getMessage()); if (!$relatedAttribute->isEmpty()) { @@ -456,7 +455,6 @@ class Databases extends Action } $dbForProject->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'available')); } catch (\Throwable $e) { - Console::error($e->getMessage()); if ($e instanceof DatabaseException) { $index->setAttribute('error', $e->getMessage()); } @@ -512,8 +510,6 @@ class Databases extends Action $dbForProject->deleteDocument('indexes', $index->getId()); $index->setAttribute('status', 'deleted'); } catch (\Throwable $e) { - Console::error($e->getMessage()); - if ($e instanceof DatabaseException) { $index->setAttribute('error', $e->getMessage()); } @@ -613,7 +609,7 @@ class Databases extends Action */ protected function deleteByGroup(string $collectionId, array $queries, Database $database, ?callable $callback = null): void { - $start = \microtime(true); + Span::add('delete_by_group.collection.id', $collectionId); try { $count = $database->deleteDocuments( @@ -623,13 +619,11 @@ class Databases extends Action $callback ); } catch (\Throwable $th) { - $tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : ''; - Console::error("Failed to delete documents/rows for collection/table: {$database->getNamespace()}_{$collectionId} {$tenant} :{$th->getMessage()}"); + Span::add('delete_by_group.error', $th->getMessage()); return; } - $end = \microtime(true); - Console::info("Deleted {$count} documents/rows by group in " . ($end - $start) . " seconds"); + Span::add('delete_by_group.count', $count); } /** diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php index e37f4120e1..eaafe561df 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php @@ -81,7 +81,7 @@ class Create extends Base )) ->param('functionId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Function ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) ->param('name', '', new Text(128), 'Function name. Max length: 128 chars.') - ->param('runtime', '', new WhiteList(array_keys(Config::getParam('runtimes')), true), 'Execution runtime.', enum: new Enum(name: 'FunctionRuntime')) + ->param('runtime', '', new WhiteList(array_keys(Config::getParam('runtimes')), true), 'Execution runtime.', enum: new Enum(name: 'Runtime')) ->param('execute', [], new Roles(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of role strings with execution permissions. By default no user is granted with any execute permissions. [learn more about roles](https://appwrite.io/docs/permissions#permission-roles). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 64 characters long.', true) ->param('events', [], new ArrayList(new FunctionEvent(), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Events list. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' events are allowed.', true) ->param('schedule', '', new Cron(), 'Schedule CRON syntax.', true) diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php index 9272c2863c..5fbe010f05 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php @@ -73,7 +73,7 @@ class Update extends Base )) ->param('functionId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Function ID.', false, ['dbForProject']) ->param('name', '', new Text(128), 'Function name. Max length: 128 chars.') - ->param('runtime', '', new WhiteList(array_keys(Config::getParam('runtimes')), true), 'Execution runtime.', true, enum: new Enum(name: 'FunctionRuntime')) + ->param('runtime', '', new WhiteList(array_keys(Config::getParam('runtimes')), true), 'Execution runtime.', true, enum: new Enum(name: 'Runtime')) ->param('execute', [], new Roles(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of role strings with execution permissions. By default no user is granted with any execute permissions. [learn more about roles](https://appwrite.io/docs/permissions#permission-roles). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 64 characters long.', true) ->param('events', [], new ArrayList(new FunctionEvent(), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Events list. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' events are allowed.', true) ->param('schedule', '', new Cron(), 'Schedule CRON syntax.', true) diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Templates/XList.php b/src/Appwrite/Platform/Modules/Functions/Http/Templates/XList.php index d77063e95a..f334351fef 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Templates/XList.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Templates/XList.php @@ -51,7 +51,7 @@ class XList extends Base ) ] )) - ->param('runtimes', [], new ArrayList(new WhiteList(array_keys(Config::getParam('runtimes')), true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'List of runtimes allowed for filtering function templates. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' runtimes are allowed.', true, enum: new Enum(name: 'FunctionRuntime')) + ->param('runtimes', [], new ArrayList(new WhiteList(array_keys(Config::getParam('runtimes')), true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'List of runtimes allowed for filtering function templates. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' runtimes are allowed.', true, enum: new Enum(name: 'Runtime')) ->param('useCases', [], new ArrayList(new WhiteList(FunctionUseCases::getAll()), APP_LIMIT_ARRAY_PARAMS_SIZE), 'List of use cases allowed for filtering function templates. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' use cases are allowed.', true, enum: new Enum(name: 'FunctionTemplateUseCase')) ->param('limit', 25, new Range(1, 5000), 'Limit the number of templates returned in the response. Default limit is 25, and maximum limit is 5000.', true) ->param('offset', 0, new Range(0, 5000), 'Offset the list of returned templates. Maximum offset is 5000.', true) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 1d4a8dd689..dd0f9d541b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -443,7 +443,8 @@ class Builds extends Action Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); // Commit and push - $exit = Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($tmpDirectory) . ' && git checkout -b ' . \escapeshellarg($branchName) . ' && git add . && git commit -m "Create ' . \escapeshellarg($resource->getAttribute('name', '')) . ' function" && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); + $commitMessage = \escapeshellarg('Create ' . $resource->getAttribute('name', '') . ' function'); + $exit = Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($tmpDirectory) . ' && git checkout -b ' . \escapeshellarg($branchName) . ' && git add . && git commit -m ' . $commitMessage . ' && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); if ($exit !== 0) { throw new \Exception('Unable to push code repository: ' . $stderr); diff --git a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/Appwrite/Create.php b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/Appwrite/Create.php index 2633f5f6fb..2e50f7966e 100644 --- a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/Appwrite/Create.php +++ b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/Appwrite/Create.php @@ -59,7 +59,7 @@ class Create extends Action ->param('endpoint', '', new URL(), 'Source Appwrite endpoint') ->param('projectId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Source Project ID', false, ['dbForProject']) ->param('apiKey', '', new Text(512), 'Source API Key') - ->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true, enum: new Enum(name: 'MigrationOnDuplicate')) + ->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true, enum: new Enum(name: 'OnDuplicate')) ->inject('response') ->inject('dbForProject') ->inject('project') diff --git a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Imports/Create.php b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Imports/Create.php index 7222a05703..66c1b96ae5 100644 --- a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Imports/Create.php +++ b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Imports/Create.php @@ -70,7 +70,7 @@ class Create extends Action ->param('fileId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'File ID.', false, ['dbForProject']) ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') ->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true) - ->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true, enum: new Enum(name: 'MigrationOnDuplicate')) + ->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true, enum: new Enum(name: 'OnDuplicate')) ->inject('response') ->inject('dbForProject') ->inject('dbForPlatform') diff --git a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Imports/Create.php b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Imports/Create.php index cec20fe61e..c1f2e25c8e 100644 --- a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Imports/Create.php +++ b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Imports/Create.php @@ -69,7 +69,7 @@ class Create extends Action ->param('fileId', '', new UID(), 'File ID.') ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') ->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true) - ->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true, enum: new Enum(name: 'MigrationOnDuplicate')) + ->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true, enum: new Enum(name: 'OnDuplicate')) ->inject('response') ->inject('dbForProject') ->inject('dbForPlatform') diff --git a/src/Appwrite/Platform/Modules/Presences/HTTP/Update.php b/src/Appwrite/Platform/Modules/Presences/HTTP/Update.php index 376359717b..cd2f0d5636 100644 --- a/src/Appwrite/Platform/Modules/Presences/HTTP/Update.php +++ b/src/Appwrite/Platform/Modules/Presences/HTTP/Update.php @@ -53,7 +53,7 @@ class Update extends PlatformAction name: 'update', desc: 'Update presence', description: '/docs/references/presences/update.md', - auth: [AuthType::SESSION], + auth: [AuthType::SESSION, AuthType::ADMIN], responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, @@ -73,10 +73,10 @@ class Update extends PlatformAction new Method( namespace: 'presences', group: 'presences', - name: 'updatePresence', + name: 'update', desc: 'Update presence', description: '/docs/references/presences/update.md', - auth: [AuthType::KEY, AuthType::JWT, AuthType::ADMIN], + auth: [AuthType::KEY, AuthType::JWT], responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, diff --git a/src/Appwrite/Platform/Modules/Presences/HTTP/Upsert.php b/src/Appwrite/Platform/Modules/Presences/HTTP/Upsert.php index be658adbd1..9b5163fdbf 100644 --- a/src/Appwrite/Platform/Modules/Presences/HTTP/Upsert.php +++ b/src/Appwrite/Platform/Modules/Presences/HTTP/Upsert.php @@ -54,7 +54,7 @@ class Upsert extends PlatformAction name: 'upsert', desc: 'Upsert presence', description: '/docs/references/presences/upsert.md', - auth: [AuthType::SESSION], + auth: [AuthType::SESSION, AuthType::ADMIN], responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, @@ -76,7 +76,7 @@ class Upsert extends PlatformAction name: 'upsert', desc: 'Upsert presence', description: '/docs/references/presences/upsert.md', - auth: [AuthType::KEY, AuthType::JWT, AuthType::ADMIN], + auth: [AuthType::KEY, AuthType::JWT], responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, @@ -134,15 +134,13 @@ class Upsert extends PlatformAction throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, "userId is not allowed for non-API key and non-privileged users"); } - if (($isAPIKey || $isPrivilegedUser) && !$userId) { - throw new Exception(Exception::GENERAL_BAD_REQUEST, "userId is required for API key and privileged users"); + // API keys have no associated session user — they must target one explicitly + if ($isAPIKey && !$userId) { + throw new Exception(Exception::GENERAL_BAD_REQUEST, "userId is required for API key authentication"); } $userInternalId = null; $resolvedUserId = $userId; - if (!$isAPIKey && !$isPrivilegedUser) { - $userInternalId = $user->getSequence(); - $resolvedUserId = $user->getId(); - } else { + if ($userId) { $fetchedUser = $dbForProject->getDocument('users', $userId); if ($fetchedUser->isEmpty()) { throw new Exception(Exception::USER_NOT_FOUND, params: [$userId]); @@ -150,6 +148,9 @@ class Upsert extends PlatformAction $userInternalId = (string) $fetchedUser->getSequence(); $resolvedUserId = $fetchedUser->getId(); + } else { + $userInternalId = $user->getSequence(); + $resolvedUserId = $user->getId(); } if (empty($userInternalId)) { diff --git a/src/Appwrite/Platform/Modules/Project/Http/Project/Policies/Get.php b/src/Appwrite/Platform/Modules/Project/Http/Project/Policies/Get.php index 7505cf62c4..c0c247936d 100644 --- a/src/Appwrite/Platform/Modules/Project/Http/Project/Policies/Get.php +++ b/src/Appwrite/Platform/Modules/Project/Http/Project/Policies/Get.php @@ -23,6 +23,9 @@ class Get extends Action public function __construct() { + $policies = $this->getPolicies(); + $policyIds = \array_keys($policies); + $this ->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) ->setHttpPath('/v1/project/policies/:policyId') @@ -40,31 +43,11 @@ class Get extends Action responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: [ - Response::MODEL_POLICY_PASSWORD_DICTIONARY, - Response::MODEL_POLICY_PASSWORD_HISTORY, - Response::MODEL_POLICY_PASSWORD_PERSONAL_DATA, - Response::MODEL_POLICY_SESSION_ALERT, - Response::MODEL_POLICY_SESSION_DURATION, - Response::MODEL_POLICY_SESSION_INVALIDATION, - Response::MODEL_POLICY_SESSION_LIMIT, - Response::MODEL_POLICY_USER_LIMIT, - Response::MODEL_POLICY_MEMBERSHIP_PRIVACY, - ], + model: \array_values($policies), ) ] )) - ->param('policyId', '', new WhiteList([ - 'password-dictionary', - 'password-history', - 'password-personal-data', - 'session-alert', - 'session-duration', - 'session-invalidation', - 'session-limit', - 'user-limit', - 'membership-privacy', - ], true), 'Policy ID. Can be one of: password-dictionary, password-history, password-personal-data, session-alert, session-duration, session-invalidation, session-limit, user-limit, membership-privacy.', enum: new Enum(name: 'ProjectPolicyId')) + ->param('policyId', '', new WhiteList($policyIds, true), 'Policy ID. Can be one of: ' . \implode(', ', $policyIds) . '.', enum: new Enum(name: 'ProjectPolicyId')) ->inject('response') ->inject('project') ->callback($this->action(...)); @@ -75,9 +58,43 @@ class Get extends Action Response $response, Document $project, ): void { + $resolved = $this->resolvePolicy($policyId, $project); + + if ($resolved === null) { + throw new \LogicException('Unknown policy ID: ' . $policyId); + } + + [$policy, $model] = $resolved; + + $response->dynamic($policy, $model); + } + + /** + * @return array + */ + protected function getPolicies(): array + { + return [ + 'password-dictionary' => Response::MODEL_POLICY_PASSWORD_DICTIONARY, + 'password-history' => Response::MODEL_POLICY_PASSWORD_HISTORY, + 'password-personal-data' => Response::MODEL_POLICY_PASSWORD_PERSONAL_DATA, + 'session-alert' => Response::MODEL_POLICY_SESSION_ALERT, + 'session-duration' => Response::MODEL_POLICY_SESSION_DURATION, + 'session-invalidation' => Response::MODEL_POLICY_SESSION_INVALIDATION, + 'session-limit' => Response::MODEL_POLICY_SESSION_LIMIT, + 'user-limit' => Response::MODEL_POLICY_USER_LIMIT, + 'membership-privacy' => Response::MODEL_POLICY_MEMBERSHIP_PRIVACY, + ]; + } + + /** + * @return ?array + */ + protected function resolvePolicy(string $policyId, Document $project): ?array + { $auths = $project->getAttribute('auths', []); - [$policy, $model] = match ($policyId) { + return match ($policyId) { 'password-dictionary' => [ new Document([ '$id' => 'password-dictionary', @@ -145,9 +162,7 @@ class Get extends Action ]), Response::MODEL_POLICY_MEMBERSHIP_PRIVACY, ], - default => throw new \LogicException('Unknown policy ID: ' . $policyId), + default => null, }; - - $response->dynamic($policy, $model); } } diff --git a/src/Appwrite/Platform/Modules/Project/Http/Project/Policies/XList.php b/src/Appwrite/Platform/Modules/Project/Http/Project/Policies/XList.php index 3020fa79dd..7057803c8a 100644 --- a/src/Appwrite/Platform/Modules/Project/Http/Project/Policies/XList.php +++ b/src/Appwrite/Platform/Modules/Project/Http/Project/Policies/XList.php @@ -71,9 +71,30 @@ class XList extends Action throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } + $policies = $this->getPolicies($project); + + $total = $includeTotal ? \count($policies) : 0; + + $grouped = Query::groupByType($queries); + $offset = $grouped['offset'] ?? 0; + $limit = $grouped['limit'] ?? null; + + $policies = \array_slice($policies, $offset, $limit); + + $response->dynamic(new Document([ + 'policies' => $policies, + 'total' => $total, + ]), Response::MODEL_POLICY_LIST); + } + + /** + * @return array + */ + protected function getPolicies(Document $project): array + { $auths = $project->getAttribute('auths', []); - $policies = [ + return [ new Document([ '$id' => 'password-dictionary', 'enabled' => $auths['passwordDictionary'] ?? false, @@ -115,18 +136,5 @@ class XList extends Action 'userMFA' => $auths['membershipsMfa'] ?? false, ]), ]; - - $total = $includeTotal ? \count($policies) : 0; - - $grouped = Query::groupByType($queries); - $offset = $grouped['offset'] ?? 0; - $limit = $grouped['limit'] ?? null; - - $policies = \array_slice($policies, $offset, $limit); - - $response->dynamic(new Document([ - 'policies' => $policies, - 'total' => $total, - ]), Response::MODEL_POLICY_LIST); } } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php index 8dd15d3f03..24abbf49ba 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php @@ -68,7 +68,7 @@ class Create extends Action ->param('domain', null, new ValidatorDomain(), 'Domain name.') ->param('url', null, new URL(), 'Target URL of redirection') ->param('statusCode', null, new WhiteList([301, 302, 307, 308]), 'Status code of redirection', enum: new Enum( - name: 'RedirectStatusCode', + name: 'StatusCode', map: [ '301' => 'MovedPermanently', '302' => 'Found', diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php index 064f2e213e..e167c88af2 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php @@ -64,7 +64,7 @@ class Create extends Base )) ->param('siteId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Site ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) ->param('name', '', new Text(128), 'Site name. Max length: 128 chars.') - ->param('framework', '', new WhiteList(\array_keys(Config::getParam('frameworks')), true), 'Sites framework.', enum: new Enum(name: 'SiteFramework')) + ->param('framework', '', new WhiteList(\array_keys(Config::getParam('frameworks')), true), 'Sites framework.', enum: new Enum(name: 'Framework')) ->param('enabled', true, new Boolean(), 'Is site enabled? When set to \'disabled\', users cannot access the site but Server SDKs with and API key can still access the site. No data is lost when this is toggled.', true) ->param('logging', true, new Boolean(), 'When disabled, request logs will exclude logs and errors, and site responses will be slightly faster.', true) ->param('timeout', 30, new Range(1, (int) System::getEnv('_APP_SITES_TIMEOUT', 30)), 'Maximum request time in seconds.', true) @@ -72,8 +72,8 @@ class Create extends Base ->param('buildCommand', '', new Text(8192, 0), 'Build Command.', true) ->param('startCommand', '', new Text(8192, 0), 'Custom start command. Leave empty to use default.', true) ->param('outputDirectory', '', new Text(8192, 0), 'Output Directory for site.', true) - ->param('buildRuntime', '', new WhiteList(array_keys(Config::getParam('runtimes')), true), 'Runtime to use during build step.', enum: new Enum(name: 'SiteBuildRuntime')) - ->param('adapter', '', new WhiteList(['static', 'ssr']), 'Framework adapter defining rendering strategy. Allowed values are: static, ssr', true, enum: new Enum(name: 'SiteAdapter')) + ->param('buildRuntime', '', new WhiteList(array_keys(Config::getParam('runtimes')), true), 'Runtime to use during build step.', enum: new Enum(name: 'BuildRuntime')) + ->param('adapter', '', new WhiteList(['static', 'ssr']), 'Framework adapter defining rendering strategy. Allowed values are: static, ssr', true, enum: new Enum(name: 'Adapter')) ->param('installationId', '', new Text(128, 0), 'Appwrite Installation ID for VCS (Version Control System) deployment.', true) ->param('fallbackFile', '', new Text(255, 0), 'Fallback file for single page application sites.', true) ->param('providerRepositoryId', '', new Text(128, 0), 'Repository ID of the repo linked to the site.', true) diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php index 66537edb04..6cb1905550 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php @@ -68,7 +68,7 @@ class Update extends Base )) ->param('siteId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Site ID.', false, ['dbForProject']) ->param('name', '', new Text(128), 'Site name. Max length: 128 chars.') - ->param('framework', '', new WhiteList(\array_keys(Config::getParam('frameworks')), true), 'Sites framework.', enum: new Enum(name: 'SiteFramework')) + ->param('framework', '', new WhiteList(\array_keys(Config::getParam('frameworks')), true), 'Sites framework.', enum: new Enum(name: 'Framework')) ->param('enabled', true, new Boolean(), 'Is site enabled? When set to \'disabled\', users cannot access the site but Server SDKs with and API key can still access the site. No data is lost when this is toggled.', true) ->param('logging', true, new Boolean(), 'When disabled, request logs will exclude logs and errors, and site responses will be slightly faster.', true) ->param('timeout', 30, new Range(1, (int) System::getEnv('_APP_SITES_TIMEOUT', 30)), 'Maximum request time in seconds.', true) @@ -76,8 +76,8 @@ class Update extends Base ->param('buildCommand', '', new Text(8192, 0), 'Build Command.', true) ->param('startCommand', '', new Text(8192, 0), 'Custom start command. Leave empty to use default.', true) ->param('outputDirectory', '', new Text(8192, 0), 'Output Directory for site.', true) - ->param('buildRuntime', '', new WhiteList(array_keys(Config::getParam('runtimes')), true), 'Runtime to use during build step.', true, enum: new Enum(name: 'SiteBuildRuntime')) - ->param('adapter', '', new WhiteList(['static', 'ssr']), 'Framework adapter defining rendering strategy. Allowed values are: static, ssr', true, enum: new Enum(name: 'SiteAdapter')) + ->param('buildRuntime', '', new WhiteList(array_keys(Config::getParam('runtimes')), true), 'Runtime to use during build step.', true, enum: new Enum(name: 'BuildRuntime')) + ->param('adapter', '', new WhiteList(['static', 'ssr']), 'Framework adapter defining rendering strategy. Allowed values are: static, ssr', true, enum: new Enum(name: 'Adapter')) ->param('fallbackFile', '', new Text(255, 0), 'Fallback file for single page application sites.', true) ->param('installationId', '', new Text(128, 0), 'Appwrite Installation ID for VCS (Version Control System) deployment.', true) ->param('providerRepositoryId', '', new Text(128, 0), 'Repository ID of the repo linked to the site.', true) diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Templates/XList.php b/src/Appwrite/Platform/Modules/Sites/Http/Templates/XList.php index a5c6ece560..722cf986f8 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Templates/XList.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Templates/XList.php @@ -50,7 +50,7 @@ class XList extends Base ) ] )) - ->param('frameworks', [], new ArrayList(new WhiteList(\array_keys(Config::getParam('frameworks')), true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'List of frameworks allowed for filtering site templates. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' frameworks are allowed.', true, enum: new Enum(name: 'SiteFramework')) + ->param('frameworks', [], new ArrayList(new WhiteList(\array_keys(Config::getParam('frameworks')), true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'List of frameworks allowed for filtering site templates. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' frameworks are allowed.', true, enum: new Enum(name: 'Framework')) ->param('useCases', [], new ArrayList(new WhiteList(SiteUseCases::getAll()), APP_LIMIT_ARRAY_PARAMS_SIZE), 'List of use cases allowed for filtering site templates. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' use cases are allowed.', true, enum: new Enum(name: 'SiteTemplateUseCase')) ->param('limit', 25, new Range(1, 5000), 'Limit the number of templates returned in the response. Default limit is 25, and maximum limit is 5000.', true) ->param('offset', 0, new Range(0, 5000), 'Offset the list of returned templates. Maximum offset is 5000.', true) diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index 348b19c039..11f388d3c0 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -260,8 +260,8 @@ class Create extends Action }; try { - $locks($lockKey, 600, function () use ($bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $response, &$completed): void { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + $locks($lockKey, 600, function () use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $response, &$completed): void { + $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); if (!$file->isEmpty()) { $chunks = $file->getAttribute('chunksTotal', 1); $uploaded = $file->getAttribute('chunksUploaded', 0); @@ -325,7 +325,7 @@ class Create extends Action } $finalizeUpload = function (int $chunksUploaded) use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $mergeUploadMetadata, $path, $permissions, $queueForEvents, $response): void { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); $uploaded = 0; if (!$file->isEmpty()) { @@ -497,6 +497,8 @@ class Create extends Action ->setParam('bucketId', $bucket->getId()) ->setParam('fileId', $file->getId()) ->setContext('bucket', $bucket); + } else { + $queueForEvents->reset(); } $metadata = null; // was causing leaks as it was passed by reference diff --git a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Get.php b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Get.php index 556f6de52c..0a856bb5a8 100644 --- a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Get.php +++ b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Get.php @@ -70,6 +70,10 @@ class Get extends Action throw new Exception(Exception::MEMBERSHIP_NOT_FOUND); } + if ($membership->getAttribute('teamInternalId') !== $team->getSequence()) { + throw new Exception(Exception::TEAM_MEMBERSHIP_MISMATCH); + } + // Default should be "false", but existing projects already rely on this being "true" $membershipsPrivacy = [ 'userName' => $project->getAttribute('auths', [])['membershipsUserName'] ?? true, diff --git a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Update.php b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Update.php index 2198531e64..0d2f1e40b4 100644 --- a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Update.php +++ b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Update.php @@ -78,6 +78,10 @@ class Update extends Action throw new Exception(Exception::MEMBERSHIP_NOT_FOUND); } + if ($membership->getAttribute('teamInternalId') !== $team->getSequence()) { + throw new Exception(Exception::TEAM_MEMBERSHIP_MISMATCH); + } + $profile = $dbForProject->getDocument('users', $membership->getAttribute('userId')); if ($profile->isEmpty()) { throw new Exception(Exception::USER_NOT_FOUND); diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php index 27c4eacba3..cde39e9292 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php @@ -578,6 +578,14 @@ trait Deployment Span::add("{$logBase}.build.triggered", 'true'); //TODO: Add event? + } catch (Exception $e) { + Span::add("{$logBase}.error", $e->getMessage()); + Span::add("{$logBase}.error.type", $e->getType()); + if ($e->getCode() < 500) { + Console::warning("Skipping repository '{$repository->getId()}' ({$e->getType()}): {$e->getMessage()}"); + continue; + } + $errors[] = $e->getMessage(); } catch (\Throwable $e) { Span::add("{$logBase}.error", $e->getMessage()); $errors[] = $e->getMessage(); diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index c1a1891386..6153af6095 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -56,12 +56,15 @@ class ScheduleFunctions extends ScheduleBase foreach ($this->schedules as $key => $schedule) { try { $cron = new CronExpression($schedule['schedule']); + $nextDate = $cron->getNextRunDate(); } catch (\InvalidArgumentException) { // ignore invalid cron expressions continue; + } catch (\RuntimeException) { + // ignore impossible cron expressions + continue; } - $nextDate = $cron->getNextRunDate(); $next = DateTime::format($nextDate); $currentTick = $next < $timeFrame; diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 8a3cc65e60..0a066570d3 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -40,6 +40,7 @@ use function Swoole\Coroutine\batch; class Deletes extends Action { protected array $selects = ['$sequence', '$id', '$collection', '$permissions', '$updatedAt']; + public const PROCESSING_STUCK_RETENTION_SECONDS = 3 * 24 * 60 * 60; // 3 days public static function getName(): string { @@ -223,6 +224,7 @@ class Deletes extends Action $this->deleteExpiredTransactions($project, $getProjectDB); $this->deleteExpiredPresences($project, $getProjectDB, $publisherForUsage); $this->deleteOldDeployments($publisherForDeletes, $project, $getProjectDB); + $this->updateProcessingMigrations($project, $getProjectDB); break; case DELETE_TYPE_REPORT: $this->deleteReport($dbForPlatform, $project, $document); @@ -390,16 +392,59 @@ class Deletes extends Action */ private function deleteExpiredTargets(Document $project, callable $getProjectDB): void { + Console::info('Delete expired targets'); Targets::delete($getProjectDB($project), Query::equal('expired', [true])); } private function deleteSessionTargets(Document $project, callable $getProjectDB, Document $session): void { + Console::info('Delete session targets'); Targets::delete($getProjectDB($project), Query::equal('sessionInternalId', [$session->getSequence()])); } + private function updateProcessingMigrations(Document $project, callable $getProjectDB): void + { + if ($project->getId() === 'console') { + return; + } + + Console::info('Update processing migrations'); + + /** @var Database $dbForProject */ + $dbForProject = $getProjectDB($project); + + $date = DateTime::addSeconds(new \DateTime(), -self::PROCESSING_STUCK_RETENTION_SECONDS); + + $queries = [ + Query::select($this->selects), + Query::equal('status', ['processing']), + Query::lessThan('$updatedAt', $date), + ]; + + $this->listByGroup( + 'migrations', + $queries, + $dbForProject, + function (Document $migration) use ($dbForProject, $project) { + try { + $dbForProject->updateDocument('migrations', $migration->getId(), new Document([ + 'status' => 'failed' + ])); + } catch (Throwable $th) { + Console::error("Failed to update processing migration {$migration->getId()} for project {$project->getId()}: " . $th->getMessage()); + } + } + ); + } + private function deleteOldDeployments(DeletePublisher $publisherForDeletes, Document $project, callable $getProjectDB): void { + if ($project->getId() === 'console') { + return; + } + + Console::info('Delete old deployments'); + /** @var Database $dbForProject */ $dbForProject = $getProjectDB($project); @@ -545,6 +590,12 @@ class Deletes extends Action */ private function deleteUsageStats(Document $project, callable $getProjectDB, callable $getLogsDB, string $hourlyUsageRetentionDatetime): void { + if ($project->getId() === 'console') { + return; + } + + Console::info('Delete usage stats'); + /** @var Database $dbForProject */ $dbForProject = $getProjectDB($project); @@ -559,19 +610,17 @@ class Deletes extends Action Query::orderDesc(), ], $dbForProject); - if ($project->getId() !== 'console') { - /** @var Database $dbForLogs */ - $dbForLogs = call_user_func($getLogsDB, $project); + /** @var Database $dbForLogs */ + $dbForLogs = call_user_func($getLogsDB, $project); - // Delete Usage stats from logsDB - $this->deleteByGroup('stats', [ - Query::select($selects), - Query::equal('period', ['1h']), - Query::lessThan('time', $hourlyUsageRetentionDatetime), - Query::orderDesc('time'), - Query::orderDesc(), - ], $dbForLogs); - } + // Delete Usage stats from logsDB + $this->deleteByGroup('stats', [ + Query::select($selects), + Query::equal('period', ['1h']), + Query::lessThan('time', $hourlyUsageRetentionDatetime), + Query::orderDesc('time'), + Query::orderDesc(), + ], $dbForLogs); } /** @@ -981,6 +1030,12 @@ class Deletes extends Action */ private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime, ?int $executionsRetentionCount = 0): void { + if ($project->getId() === 'console') { + return; + } + + Console::info('Delete execution logs'); + /** @var Database $dbForProject */ $dbForProject = $getProjectDB($project); @@ -1012,7 +1067,7 @@ class Deletes extends Action ?string $resourceInternalId = null, ?string $resourceType = null ): void { - if ($executionsRetentionCount <= 0) { + if ($executionsRetentionCount <= 0 || $project->getId() === 'console') { return; } @@ -1077,6 +1132,8 @@ class Deletes extends Action */ private function deleteExpiredSessions(Document $project, callable $getProjectDB): void { + Console::info('Delete expired sessions'); + $dbForProject = $getProjectDB($project); $duration = $project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration); @@ -1150,6 +1207,8 @@ class Deletes extends Action */ private function deleteAuditLogs(Document $project, callable $getAudit, string $auditRetention): void { + Console::info('Delete audit logs'); + $projectId = $project->getId(); /** @var Audit $audit */ $audit = $getAudit($project); @@ -1520,6 +1579,11 @@ class Deletes extends Action ): void { $start = \microtime(true); + $message = 'collection:'.$database->getNamespace().'_'.$collection; + if ($database->getSharedTables()) { + $message .= ' Tenant:'.$database->getTenant(); + } + /** * deleteDocuments uses a cursor, we need to add a unique order by field or use default */ @@ -1530,13 +1594,13 @@ class Deletes extends Action onNext: $callback ); } catch (Throwable $th) { - $tenant = $database->getSharedTables() ? 'Tenant:' . $database->getTenant() : ''; - Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collection} {$tenant} :{$th->getMessage()}"); + $tenant = $database->getSharedTables() ? 'Tenant:'. $database->getTenant() : ''; + Console::error("Failed to delete documents for {$message} :{$th->getMessage()}"); return; } $end = \microtime(true); - Console::info("Deleted {$count} documents by group in " . ($end - $start) . " seconds"); + Console::info("Deleted {$count} documents by group in " . ($end - $start) . " seconds {$message}"); } /** @@ -1583,7 +1647,12 @@ class Deletes extends Action $end = \microtime(true); - Console::info("Listed {$count} documents by group in " . ($end - $start) . " seconds"); + $message = 'collection:'.$database->getNamespace().'_'.$collection; + if ($database->getSharedTables()) { + $message .= ' Tenant:'.$database->getTenant(); + } + + Console::info("Listed {$count} documents by group in " . ($end - $start) . " seconds {$message}"); } /** @@ -1723,6 +1792,12 @@ class Deletes extends Action private function deleteExpiredTransactions(Document $project, callable $getProjectDB): void { + if ($project->getId() === 'console') { + return; + } + + Console::info('Delete expired transactions'); + $dbForProject = $getProjectDB($project); $transactionInternalIds = []; @@ -1751,6 +1826,12 @@ class Deletes extends Action private function deleteExpiredPresences(Document $project, callable $getProjectDB, UsagePublisher $publisherForUsage): void { + if ($project->getId() === 'console') { + return; + } + + Console::info('Delete expired presences'); + $dbForProject = $getProjectDB($project); $now = DateTime::format(new \DateTime()); diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 7af079e914..ddd7edf641 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -16,11 +16,13 @@ use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\DSN\DSN; +use Utopia\Lock\Semaphore; use Utopia\Logger\Log; use Utopia\Messaging\Adapter\Email as EmailAdapter; use Utopia\Messaging\Adapter\Email\Mailgun; use Utopia\Messaging\Adapter\Email\Resend; use Utopia\Messaging\Adapter\Email\Sendgrid; +use Utopia\Messaging\Adapter\Email\SES; use Utopia\Messaging\Adapter\Email\SMTP; use Utopia\Messaging\Adapter\Push\APNS; use Utopia\Messaging\Adapter\Push as PushAdapter; @@ -52,8 +54,6 @@ use function Swoole\Coroutine\batch; class Messaging extends Action { - private ?Local $localDevice = null; - private ?SMSAdapter $adapter = null; public static function getName(): string @@ -130,69 +130,19 @@ class Messaging extends Action Document $project, UsagePublisher $publisherForUsage ): void { + $status = $message->getAttribute('status'); + + // Idempotency guard: queue redelivery can hand us a message that already finished. + if (\in_array($status, [MessageStatus::SENT, MessageStatus::FAILED], true)) { + Span::add('message.skipped', 'already_processed'); + return; + } + $topicIds = $message->getAttribute('topics', []); $targetIds = $message->getAttribute('targets', []); $userIds = $message->getAttribute('users', []); $providerType = $message->getAttribute('providerType'); - /** - * @var array $allTargets - */ - $allTargets = []; - - if (\count($topicIds) > 0) { - $topics = $dbForProject->find('topics', [ - Query::equal('$id', $topicIds), - Query::limit(\count($topicIds)), - ]); - foreach ($topics as $topic) { - $targets = \array_filter($topic->getAttribute('targets'), function (Document $target) use ($providerType) { - return $target->getAttribute('providerType') === $providerType; - }); - - \array_push($allTargets, ...$targets); - } - } - - if (\count($userIds) > 0) { - $limit = 1000; - $offset = 0; - - do { - $targets = $dbForProject->find('targets', [ - Query::equal('userId', $userIds), - Query::select(['providerId', 'identifier']), - Query::equal('providerType', [$providerType]), - Query::limit($limit), - Query::offset($offset), - ]); - - \array_push($allTargets, ...$targets); - $offset += \count($targets); - } while (\count($targets) === $limit); - } - - if (\count($targetIds) > 0) { - $targets = $dbForProject->find('targets', [ - Query::select(['providerId', 'identifier']), - Query::equal('$id', $targetIds), - Query::equal('providerType', [$providerType]), - Query::limit(\count($targetIds)), - ]); - - \array_push($allTargets, ...$targets); - } - - if (empty($allTargets)) { - $dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([ - 'status' => MessageStatus::FAILED, - 'deliveryErrors' => ['No valid recipients found.'] - ])); - - Span::add('message.skipped', 'no_valid_recipients'); - return; - } - $default = $dbForProject->findOne('providers', [ Query::equal('enabled', [true]), Query::equal('type', [$providerType]), @@ -209,55 +159,33 @@ class Messaging extends Action } /** - * @var array> $identifiers - */ - $identifiers = []; - - /** - * @var array $providers + * Resolved providers cached for the lifetime of this job, keyed by provider id. + * Seeded with the default provider so most sends never touch the providers collection. + * + * @var array $providers */ $providers = [ - $default->getId() => $default + $default->getId() => $default, ]; - foreach ($allTargets as $target) { - $providerId = $target->getAttribute('providerId'); + $semaphore = new Semaphore(MESSAGE_SEND_CONCURRENCY); - if (!$providerId) { - $providerId = $default->getId(); - } + $deliveredTotal = 0; + $failedTotal = 0; + $deliveryErrors = []; + $hasRecipients = false; - if ($providerId) { - if (!\array_key_exists($providerId, $identifiers)) { - $identifiers[$providerId] = []; - } - // Use null as value to avoid duplicate keys - $identifiers[$providerId][$target->getAttribute('identifier')] = null; - } - } + foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) { + /** + * @var array $tasks + */ + $tasks = []; - /** - * @var array $results - */ - $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - if (\array_key_exists($providerId, $providers)) { - $provider = $providers[$providerId]; - } else { - $provider = $dbForProject->getDocument('providers', $providerId); + foreach ($page as $providerId => $identifiers) { + $provider = $this->resolveProvider($dbForProject, $providerId, $providers, $default); + $resolvedProviderType = $provider->getAttribute('type'); - if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { - $provider = $default; - } else { - $providers[$providerId] = $provider; - } - } - - $identifiersForProvider = $identifiers[$providerId]; - - $providerType = $provider->getAttribute('type'); - - $adapter = match ($providerType) { + $adapter = match ($resolvedProviderType) { MESSAGE_TYPE_SMS => $this->getSmsAdapter($provider), MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider), MESSAGE_TYPE_EMAIL => $this->getEmailAdapter($provider), @@ -265,101 +193,74 @@ class Messaging extends Action }; $batches = \array_chunk( - \array_keys($identifiersForProvider), + \array_keys($identifiers), $adapter->getMaxMessagesPerRequest() ); - return batch(\array_map(function ($batch) use ($message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - return function () use ($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - $deliveredTotal = 0; - $deliveryErrors = []; - $messageData = clone $message; - $messageData->setAttribute('to', $batch); + foreach ($batches as $batch) { + $tasks[] = fn (): array => $semaphore->withLock( + fn (): array => $this->sendBatch( + $batch, + $message, + $provider, + $resolvedProviderType, + $adapter, + $dbForProject, + $deviceForFiles, + $project, + $publisherForUsage + ) + ); + } + } - $data = match ($providerType) { - MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), - MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), - MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), - }; + // A page can resolve to zero identifiers (e.g. subscribers whose targetInternalId matches no targets + // row), so an empty task list must not count as recipients or run batch() for nothing. + if (empty($tasks)) { + continue; + } - try { - $response = $adapter->send($data); - $deliveredTotal += (int) $response['deliveredTo']; - foreach ($response['results'] as $result) { - if ($result['status'] === 'failure') { - $deliveryErrors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}"; - } + $hasRecipients = true; - // Deleting push targets when token has expired. - if (($result['error'] ?? '') === 'Expired device token') { - $target = $dbForProject->findOne('targets', [ - Query::equal('identifier', [$result['recipient']]) - ]); + /** + * @var array}> $results + */ + $results = batch($tasks); - if (!$target->isEmpty()) { - $dbForProject->updateDocument( - 'targets', - $target->getId(), - $target->setAttribute('expired', true) - ); - } - } - } - } catch (\Throwable $e) { - $deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage(); - } finally { - $errorTotal = \count($deliveryErrors); - $usage = new UsageContext(); - $usage - ->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal)) - ->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal) - ->addMetric(METRIC_MESSAGES_FAILED, $errorTotal) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), ($deliveredTotal + $errorTotal)) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $deliveredTotal) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal)) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal); + foreach ($results as $result) { + $deliveredTotal += $result['delivered']; + $failedTotal += $result['recipients'] - $result['delivered']; - $publisherForUsage->enqueue(new Usage( - project: $project, - metrics: $usage->getMetrics(), - )); + foreach ($result['errors'] as $error) { + if (\count($deliveryErrors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) { + break; + } - return [ - 'deliveredTotal' => $deliveredTotal, - 'deliveryErrors' => $deliveryErrors, - ]; - } - }; - }, $batches)); - }; - }, \array_keys($identifiers))); + $deliveryErrors[] = $error; + } + } + } - $results = \array_merge(...$results); + if (!$hasRecipients) { + $dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([ + 'status' => MessageStatus::FAILED, + 'deliveryErrors' => ['No valid recipients found.'] + ])); - $deliveredTotal = 0; - $deliveryErrors = []; - - foreach ($results as $result) { - $deliveredTotal += $result['deliveredTotal']; - $deliveryErrors = \array_merge($deliveryErrors, $result['deliveryErrors']); + Span::add('message.skipped', 'no_valid_recipients'); + return; } if (empty($deliveryErrors) && $deliveredTotal === 0) { $deliveryErrors[] = 'Unknown error'; } + $hasFailures = $failedTotal > 0 || \count($deliveryErrors) > 0; + $message->setAttribute('status', $hasFailures ? MessageStatus::FAILED : MessageStatus::SENT); $message->setAttribute('deliveryErrors', $deliveryErrors); - if (\count($message->getAttribute('deliveryErrors')) > 0) { - $message->setAttribute('status', MessageStatus::FAILED); - } else { - $message->setAttribute('status', MessageStatus::SENT); - } - Span::add('message.delivered_total', $deliveredTotal); - Span::add('message.errors_total', \count($deliveryErrors)); + Span::add('message.errors_total', $failedTotal); $message->removeAttribute('to'); @@ -410,6 +311,461 @@ class Messaging extends Action } } + /** + * Stream a message's recipients in bounded pages, grouped by provider and deduplicated by identifier within each page. + * + * Peak memory is O(MESSAGE_RECIPIENTS_PAGE_SIZE), never O(topic size): topics are walked through the + * subscribers collection with cursor pagination rather than reading the topic's `targets` attribute, which + * triggers the subQueryTopicTargets filter and loads up to APP_LIMIT_SUBSCRIBERS_SUBQUERY rows at once. + * + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return \Generator>> + * @throws \Exception + */ + private function streamRecipients( + Database $dbForProject, + array $topicIds, + array $userIds, + array $targetIds, + string $providerType, + Document $default + ): \Generator { + if (\count($topicIds) > 0) { + $topics = $dbForProject->find('topics', [ + Query::select(['$sequence']), + Query::equal('$id', $topicIds), + Query::limit(\count($topicIds)), + ]); + + foreach ($topics as $topic) { + $cursor = null; + + do { + $queries = [ + Query::equal('topicInternalId', [$topic->getSequence()]), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'targetInternalId']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $subscribers = $dbForProject->getAuthorization()->skip( + fn () => $dbForProject->find('subscribers', $queries) + ); + + $count = \count($subscribers); + + if ($count === 0) { + break; + } + + $cursor = $subscribers[$count - 1]; + + $targetInternalIds = \array_map( + fn (Document $subscriber) => $subscriber->getAttribute('targetInternalId'), + $subscribers + ); + + $targets = $dbForProject->skipValidation( + fn () => $dbForProject->getAuthorization()->skip( + fn () => $dbForProject->find('targets', [ + Query::equal('$sequence', $targetInternalIds), + Query::select(['providerId', 'identifier']), + Query::limit(\count($targetInternalIds)), + ]) + ) + ); + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + } + + if (\count($userIds) > 0) { + $cursor = null; + + do { + $queries = [ + Query::equal('userId', $userIds), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'providerId', 'identifier']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $targets = $dbForProject->find('targets', $queries); + $count = \count($targets); + + if ($count === 0) { + break; + } + + $cursor = $targets[$count - 1]; + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + + if (\count($targetIds) > 0) { + $cursor = null; + + do { + $queries = [ + Query::equal('$id', $targetIds), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'providerId', 'identifier']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $targets = $dbForProject->find('targets', $queries); + $count = \count($targets); + + if ($count === 0) { + break; + } + + $cursor = $targets[$count - 1]; + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + } + + /** + * Group a page of target documents by provider id, deduplicating identifiers within the page. + * + * @param array $targets + * @return array> + */ + private function groupTargetsByProvider(array $targets, Document $default): array + { + /** + * @var array> $identifiers + */ + $identifiers = []; + + foreach ($targets as $target) { + $providerId = $target->getAttribute('providerId') ?: $default->getId(); + + if (!\array_key_exists($providerId, $identifiers)) { + $identifiers[$providerId] = []; + } + + // Null values keep identifiers unique without a second lookup structure. + $identifiers[$providerId][$target->getAttribute('identifier')] = null; + } + + return $identifiers; + } + + /** + * Resolve and cache a provider for the lifetime of a send job, falling back to the default provider. + * + * @param array $providers + */ + private function resolveProvider( + Database $dbForProject, + string $providerId, + array &$providers, + Document $default + ): Document { + if (\array_key_exists($providerId, $providers)) { + return $providers[$providerId]; + } + + $provider = $dbForProject->getDocument('providers', $providerId); + + if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { + // Cache the fallback under this id too, so a topic full of targets pointing at the same + // disabled/missing provider does not re-query the providers collection once per page. + $providers[$providerId] = $default; + + return $default; + } + + $providers[$providerId] = $provider; + + return $provider; + } + + /** + * Send a single adapter-sized batch and report delivery counts plus a bounded error list. + * + * Wraps the provider call in a backoff/retry loop that reacts to provider rate limiting and transient + * failures (see {@see retrySend()}). Accounting stays exact: only still-failing recipients are ever + * retried, so `delivered` is summed across attempts without double-counting, and `recipients` always + * reports the original batch size so the caller's `failed = recipients - delivered` holds. + * + * @param array $batch + * @return array{delivered: int, recipients: int, errors: array} + */ + private function sendBatch( + array $batch, + Document $message, + Document $provider, + string $providerType, + EmailAdapter|SMSAdapter|PushAdapter $adapter, + Database $dbForProject, + Device $deviceForFiles, + Document $project, + UsagePublisher $publisherForUsage + ): array { + $recipients = \count($batch); + + [ + 'delivered' => $delivered, + 'errors' => $errors, + ] = $this->retrySend($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project); + + $failed = $recipients - $delivered; + $usage = new UsageContext(); + $usage + ->addMetric(METRIC_MESSAGES, $recipients) + ->addMetric(METRIC_MESSAGES_SENT, $delivered) + ->addMetric(METRIC_MESSAGES_FAILED, $failed) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed); + + $publisherForUsage->enqueue(new Usage( + project: $project, + metrics: $usage->getMetrics(), + )); + + return [ + 'delivered' => $delivered, + 'recipients' => $recipients, + 'errors' => $errors, + ]; + } + + /** + * Drive a batch through provider sends with exponential backoff, reacting to throttling and transient + * errors by retrying only the recipients that are still failing. + * + * Partitioning per attempt: the provider's per-recipient results are split into delivered (summed into + * the running total, never retried) and failures. A failure whose error matches {@see isRetryableError()} + * stays pending for the next attempt; any other failure becomes a terminal error immediately. When + * `send()` throws, the throw is retryable only if its message matches; otherwise every still-pending + * recipient is recorded as terminal. The next attempt rebuilds the provider message with only the pending + * recipients in `to`, so an already-delivered recipient is never re-sent. After {@see MESSAGE_SEND_MAX_RETRIES} + * attempts, any recipients still pending are flushed to terminal errors. + * + * Note: provider `Retry-After` hints are not honored — the agnostic adapter {@see \Utopia\Messaging\Response} + * exposes only generic per-recipient strings, never a structured retry delay, so we keep the library + * adapter-agnostic and rely on exponential backoff alone. + * + * @param array $batch + * @return array{delivered: int, errors: array} + */ + private function retrySend( + array $batch, + Document $message, + Document $provider, + string $providerType, + EmailAdapter|SMSAdapter|PushAdapter $adapter, + Database $dbForProject, + Device $deviceForFiles, + Document $project + ): array { + $delivered = 0; + $errors = []; + + // Recipients still awaiting a successful (or terminal) outcome; shrinks to only the failing ones each attempt. + $pending = $batch; + + for ($attempt = 1; $attempt <= MESSAGE_SEND_MAX_RETRIES; $attempt++) { + $hasRetriesLeft = $attempt < MESSAGE_SEND_MAX_RETRIES; + + // Rebuild the provider message scoped to only the still-pending recipients so a partially-delivered + // batch never re-sends to recipients that already succeeded on an earlier attempt. + $data = $this->buildMessage($pending, $message, $provider, $providerType, $dbForProject, $deviceForFiles, $project); + + $retry = []; + + // The try/catch wraps ONLY the provider send. A whole-batch throw is retryable when transient, + // otherwise it records one representative terminal error. The previous behaviour of resetting + // $delivered to 0 on a throw is gone — the retry refactor sums delivered across attempts, and the + // expired-device-token cleanup below is isolated in its own try so a DB hiccup there can never be + // misattributed as a send failure. + try { + $response = $adapter->send($data); + } catch (\Throwable $e) { + if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) { + $retry = $pending; + } else { + $this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage()); + } + + $response = null; + } + + if ($response !== null) { + $delivered += (int) $response['deliveredTo']; + + foreach ($response['results'] as $result) { + if ($result['status'] !== 'failure') { + continue; + } + + $error = $result['error'] ?? null; + $recipient = $result['recipient']; + + // Best-effort: deleting push targets when the token has expired. Isolated so a transient DB + // error here never affects delivery accounting or the retry decision below. + if ($error === 'Expired device token') { + try { + $target = $dbForProject->findOne('targets', [ + Query::equal('identifier', [$recipient]) + ]); + + if (!$target->isEmpty()) { + $dbForProject->updateDocument( + 'targets', + $target->getId(), + $target->setAttribute('expired', true) + ); + } + } catch (\Throwable) { + // Best-effort; must not affect accounting or retries. + } + } + + if ($hasRetriesLeft && $this->isRetryableError($error)) { + $retry[] = $recipient; + continue; + } + + $this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}"); + } + } + + $pending = $retry; + + if (empty($pending)) { + break; + } + + // Exponential backoff with jitter; non-blocking under Swoole so sibling sends keep progressing. + // Skip a non-positive delay: Swoole\Coroutine::sleep() rejects 0/negative values (which tests + // produce by overriding the base delay to 0 for speed), and it would otherwise emit a warning. + $delay = $this->retryDelay() * (2 ** ($attempt - 1)); + $delay += $delay * (\random_int(0, 100) / 1000); + if ($delay > 0) { + \Swoole\Coroutine::sleep($delay); + } + } + + return [ + 'delivered' => $delivered, + 'errors' => $errors, + ]; + } + + /** + * Append a delivery error while keeping the retained list bounded by {@see MESSAGE_DELIVERY_ERRORS_LIMIT}. + * + * @param array $errors + */ + private function recordError(array &$errors, string $error): void + { + if (\count($errors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) { + return; + } + + $errors[] = $error; + } + + /** + * Conservatively classify a provider error string as retryable. The agnostic adapter + * {@see \Utopia\Messaging\Response} returns only free-form error strings, so string matching is the only + * provider-agnostic signal available. The pattern list is intentionally narrow — throttling, rate limits, + * quota, service-unavailable and timeout phrasing — so permanent failures (e.g. invalid recipients) are + * never retried. + */ + private function isRetryableError(?string $error): bool + { + if ($error === null || $error === '') { + return false; + } + + $patterns = [ + 'throttl', + 'rate exceeded', + 'rate limit', + 'too many requests', + '429', + 'quota', + 'service unavailable', + '503', + 'timed out', + 'timeout', + 'temporarily', + ]; + + $needle = \strtolower($error); + + foreach ($patterns as $pattern) { + if (\str_contains($needle, $pattern)) { + return true; + } + } + + return false; + } + + /** + * Base seconds for exponential backoff between send retries. Isolated so tests can override it to keep the + * suite instant without touching the production constant. + */ + protected function retryDelay(): float + { + return MESSAGE_SEND_RETRY_DELAY; + } + + /** + * Build the provider-specific message for a set of recipients. + * + * @param array $to + */ + private function buildMessage( + array $to, + Document $message, + Document $provider, + string $providerType, + Database $dbForProject, + Device $deviceForFiles, + Document $project + ): Email|SMS|Push { + $messageData = clone $message; + $messageData->setAttribute('to', $to); + + return match ($providerType) { + MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), + MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), + MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), + default => throw new \Exception('Provider with the requested ID is of the incorrect type') + }; + } + private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void { if ($this->adapter === null) { @@ -451,7 +807,7 @@ class Messaging extends Action } - private function getSmsAdapter(Document $provider): ?SMSAdapter + protected function getSmsAdapter(Document $provider): ?SMSAdapter { $credentials = $provider->getAttribute('credentials'); @@ -494,7 +850,7 @@ class Messaging extends Action }; } - private function getPushAdapter(Document $provider): ?PushAdapter + protected function getPushAdapter(Document $provider): ?PushAdapter { $credentials = $provider->getAttribute('credentials'); $options = $provider->getAttribute('options'); @@ -513,7 +869,7 @@ class Messaging extends Action }; } - private function getEmailAdapter(Document $provider): ?EmailAdapter + protected function getEmailAdapter(Document $provider): ?EmailAdapter { $credentials = $provider->getAttribute('credentials', []); $options = $provider->getAttribute('options', []); @@ -537,6 +893,12 @@ class Messaging extends Action ), 'sendgrid' => new Sendgrid($apiKey), 'resend' => new Resend($apiKey), + 'ses' => new SES( + $credentials['accessKey'] ?? '', + $credentials['secretKey'] ?? '', + $credentials['region'] ?? '', + $credentials['sessionToken'] ?? null, + ), default => null }; } @@ -709,11 +1071,9 @@ class Messaging extends Action private function getLocalDevice($project): Local { - if ($this->localDevice === null) { - $this->localDevice = new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); - } - - return $this->localDevice; + // Not cached: the path is project-scoped and the worker handles + // messages from many projects (and coroutines run them concurrently). + return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); } private function createInternalSMSAdapter(): ?SMSAdapter diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 2dd59c7b4c..6b0a33dfd7 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -49,6 +49,7 @@ use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Storage\Device; use Utopia\System\System; +use Utopia\Validator\Hostname; class Migrations extends Action { @@ -197,23 +198,58 @@ class Migrations extends Action /** @var Database|null $projectDB */ $projectDB = null; $useAppwriteApiSource = false; - if ($source === SourceAppwrite::getName() && empty($credentials['projectId'])) { + $isAppwriteSource = $source === SourceAppwrite::getName(); + $isAppwriteToAppwrite = $isAppwriteSource + && $destination === DestinationAppwrite::getName(); + + if ($isAppwriteSource && empty($credentials['projectId'])) { throw new Exception(Exception::MIGRATION_SOURCE_PROJECT_ID_REQUIRED); } - if (! empty($credentials['projectId'])) { + if ($isAppwriteSource) { $this->sourceProject = $this->dbForPlatform->getDocument('projects', $credentials['projectId']); - if ($this->sourceProject->isEmpty()) { - throw new Exception(Exception::MIGRATION_SOURCE_PROJECT_NOT_FOUND); + + // Trust DB fast path only when the source URL targets this cluster's host + // (env-configured or this project's verified custom API domain). + $sourceHost = parse_url($credentials['endpoint'] ?? '', PHP_URL_HOST); + $publicDomain = parse_url('http://' . System::getEnv('_APP_DOMAIN', ''), PHP_URL_HOST) ?: ''; + $internalHost = parse_url('http://' . System::getEnv('_APP_MIGRATION_HOST', ''), PHP_URL_HOST) ?: ''; + + $allowedHosts = array_filter([ + $publicDomain, + $publicDomain !== '' ? '*.' . $publicDomain : null, + $internalHost, + ]); + + if (is_string($sourceHost) && !$this->sourceProject->isEmpty()) { + $rule = $this->dbForPlatform->findOne('rules', [ + Query::equal('domain', [$sourceHost]), + Query::equal('type', ['api']), + Query::equal('status', [RULE_STATUS_VERIFIED]), + Query::equal('projectInternalId', [$this->sourceProject->getSequence()]), + ]); + if (!$rule->isEmpty()) { + $allowedHosts[] = $sourceHost; + } } + $isLocalEndpoint = is_string($sourceHost) + && !empty($allowedHosts) + && (new Hostname($allowedHosts))->isValid($sourceHost); + $sourceRegion = $this->sourceProject->getAttribute('region', 'default'); $destinationRegion = $this->project->getAttribute('region', 'default'); - $useAppwriteApiSource = $source === SourceAppwrite::getName() - && $destination === DestinationAppwrite::getName() - && $sourceRegion !== $destinationRegion; - if (! $useAppwriteApiSource) { + + $isLocalSource = !$this->sourceProject->isEmpty() + && $isLocalEndpoint + && (!$isAppwriteToAppwrite || $sourceRegion === $destinationRegion); + + if ($isLocalSource) { $projectDB = call_user_func($this->getProjectDB, $this->sourceProject); + } elseif ($isAppwriteToAppwrite) { + $useAppwriteApiSource = true; + } else { + throw new Exception(Exception::MIGRATION_SOURCE_PROJECT_NOT_FOUND); } } $getDatabasesDB = fn (Document $database): Database => @@ -577,11 +613,21 @@ class Migrations extends Action } if ($publish) { - call_user_func($this->logError, $th, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ + $extras = [ 'migrationId' => $migration->getId(), 'source' => $migration->getAttribute('source') ?? '', 'destination' => $migration->getAttribute('destination') ?? '', - ]); + ]; + + // Include source identifiers for Appwrite sources to make Sentry events + // self-debuggable. Never include the apiKey or any other secret. + if ($migration->getAttribute('source') === SourceAppwrite::getName()) { + $credentials = $migration->getAttribute('credentials', []) ?? []; + $extras['sourceProjectId'] = $credentials['projectId'] ?? ''; + $extras['sourceEndpoint'] = $credentials['endpoint'] ?? ''; + } + + call_user_func($this->logError, $th, 'appwrite-worker', 'appwrite-queue-' . self::getName(), $extras); } } finally { try { diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index 973e487de5..48f12d3ee9 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -6,6 +6,7 @@ use Appwrite\Event\Message\Mail as MailMessage; use Appwrite\Event\Message\Usage as UsageMessage; use Appwrite\Event\Publisher\Mail as MailPublisher; use Appwrite\Event\Publisher\Usage as UsagePublisher; +use Appwrite\Network\Validator\PublicHostname; use Appwrite\Template\Template; use Appwrite\Usage\Context as UsageContext; use Exception; @@ -101,6 +102,16 @@ class Webhooks extends Action } $url = \rawurldecode($webhook->getAttribute('url')); + + if (System::getEnv('_APP_ENV', 'development') === 'production') { + $host = \parse_url($url, PHP_URL_HOST) ?? ''; + $hostnameValidator = new PublicHostname(); + if (!$hostnameValidator->isValid($host)) { + $this->errors[] = 'Webhook target ' . $host . ' rejected: ' . $hostnameValidator->getDescription(); + return; + } + } + $signatureKey = $webhook->getAttribute('signatureKey'); $signature = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); $httpUser = $webhook->getAttribute('httpUser'); diff --git a/src/Appwrite/SDK/Specification/Format/OpenAPI3.php b/src/Appwrite/SDK/Specification/Format/OpenAPI3.php index b62397b2d0..8d07938bc8 100644 --- a/src/Appwrite/SDK/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/SDK/Specification/Format/OpenAPI3.php @@ -857,13 +857,6 @@ class OpenAPI3 extends Format if ($model->isAny()) { $output['components']['schemas'][$model->getType()]['additionalProperties'] = true; - - $additionalKey = \method_exists($model, 'getAdditionalPropertiesKey') - ? $model->getAdditionalPropertiesKey() - : null; - if ($additionalKey !== null) { - $output['components']['schemas'][$model->getType()]['x-additional-properties-key'] = $additionalKey; - } } if (!empty($required)) { diff --git a/src/Appwrite/SDK/Specification/Format/Swagger2.php b/src/Appwrite/SDK/Specification/Format/Swagger2.php index 1232ea5408..9265e1ec60 100644 --- a/src/Appwrite/SDK/Specification/Format/Swagger2.php +++ b/src/Appwrite/SDK/Specification/Format/Swagger2.php @@ -611,19 +611,15 @@ class Swagger2 extends Format } $enumKeys = []; - if (!empty($enum->map)) { - foreach ($enumValues as $enumValue) { - $enumKeys[] = $enum->map[$enumValue] ?? $enumValue; - } + foreach ($enumValues as $enumValue) { + $enumKeys[] = $enum->map[$enumValue] ?? $enumValue; } $node['items']['enum'] = $enumValues; if (!empty($enum->name)) { $node['items']['x-enum-name'] = $enum->name; } - if (!empty($enumKeys)) { - $node['items']['x-enum-keys'] = $enumKeys; - } + $node['items']['x-enum-keys'] = $enumKeys; } } if ($validator->getType() === 'integer') { @@ -652,19 +648,15 @@ class Swagger2 extends Format } $enumKeys = []; - if (!empty($enum->map)) { - foreach ($enumValues as $enumValue) { - $enumKeys[] = $enum->map[$enumValue] ?? $enumValue; - } + foreach ($enumValues as $enumValue) { + $enumKeys[] = $enum->map[$enumValue] ?? $enumValue; } $node['enum'] = $enumValues; if (!empty($enum->name)) { $node['x-enum-name'] = $enum->name; } - if (!empty($enumKeys)) { - $node['x-enum-keys'] = $enumKeys; - } + $node['x-enum-keys'] = $enumKeys; } } if ($validator->getType() === 'integer') { @@ -769,9 +761,7 @@ class Swagger2 extends Format /// If the enum flag is Set, add the enum values to the body $body['schema']['properties'][$name]['enum'] = $node['enum']; $body['schema']['properties'][$name]['x-enum-name'] = $node['x-enum-name'] ?? null; - if (isset($node['x-enum-keys'])) { - $body['schema']['properties'][$name]['x-enum-keys'] = $node['x-enum-keys']; - } + $body['schema']['properties'][$name]['x-enum-keys'] = $node['x-enum-keys']; } if ($parameter['nullable']) { @@ -830,13 +820,6 @@ class Swagger2 extends Format if ($model->isAny()) { $output['definitions'][$model->getType()]['additionalProperties'] = true; - - $additionalKey = \method_exists($model, 'getAdditionalPropertiesKey') - ? $model->getAdditionalPropertiesKey() - : null; - if ($additionalKey !== null) { - $output['definitions'][$model->getType()]['x-additional-properties-key'] = $additionalKey; - } } if (!empty($required)) { diff --git a/src/Appwrite/Task/Validator/Cron.php b/src/Appwrite/Task/Validator/Cron.php index 03bd1c5220..18f3333a57 100644 --- a/src/Appwrite/Task/Validator/Cron.php +++ b/src/Appwrite/Task/Validator/Cron.php @@ -38,7 +38,12 @@ class Cron extends Validator return false; } - return true; + try { + (new CronExpression($value))->getNextRunDate(); + return true; + } catch (\RuntimeException) { + return false; + } } /** diff --git a/src/Appwrite/URL/URL.php b/src/Appwrite/URL/URL.php index 49e95929e7..7f710f70b0 100644 --- a/src/Appwrite/URL/URL.php +++ b/src/Appwrite/URL/URL.php @@ -118,4 +118,151 @@ class URL { return \http_build_query($query); } + + /** + * Resolve a Location-header reference against an absolute base URL, + * following RFC 3986 §5.3 (Reference Resolution). Handles: + * - absolute references (scheme present) + * - protocol-relative (//host/path) + * - absolute-path (/foo) + * - relative-path (foo, ../foo) + * - query-only (?x=1) -- keeps base path + * - fragment-only (#frag) -- keeps base path AND base query + * - dot-segment normalisation -- /a/b/../c → /a/c + */ + public static function resolveLocation(string $base, string $reference): string + { + $reference = \trim($reference); + if ($reference === '') { + return $base; + } + + $ref = \parse_url($reference); + $bas = \parse_url($base); + if (!\is_array($ref) || !\is_array($bas)) { + return $reference; + } + + $target = []; + + if (isset($ref['scheme'])) { + $target['scheme'] = $ref['scheme']; + self::copyAuthority($ref, $target); + $target['path'] = self::removeDotSegments($ref['path'] ?? ''); + if (isset($ref['query'])) { + $target['query'] = $ref['query']; + } + } else { + $target['scheme'] = $bas['scheme'] ?? ''; + + if (isset($ref['host'])) { + self::copyAuthority($ref, $target); + $target['path'] = self::removeDotSegments($ref['path'] ?? ''); + if (isset($ref['query'])) { + $target['query'] = $ref['query']; + } + } else { + self::copyAuthority($bas, $target); + $refPath = $ref['path'] ?? ''; + + if ($refPath === '') { + $target['path'] = $bas['path'] ?? ''; + if (isset($ref['query'])) { + $target['query'] = $ref['query']; + } elseif (isset($bas['query'])) { + $target['query'] = $bas['query']; + } + } else { + if (\str_starts_with($refPath, '/')) { + $target['path'] = self::removeDotSegments($refPath); + } else { + $target['path'] = self::removeDotSegments(self::mergePaths($bas, $refPath)); + } + if (isset($ref['query'])) { + $target['query'] = $ref['query']; + } + } + } + } + + if (isset($ref['fragment'])) { + $target['fragment'] = $ref['fragment']; + } + + return self::unparse($target); + } + + /** + * @param array $source + * @param array $target + */ + private static function copyAuthority(array $source, array &$target): void + { + foreach (['user', 'pass', 'host', 'port'] as $key) { + if (isset($source[$key])) { + $target[$key] = $source[$key]; + } + } + } + + /** + * RFC 3986 §5.2.3 — merge base path with reference path. + */ + private static function mergePaths(array $base, string $referencePath): string + { + if (isset($base['host']) && empty($base['path'])) { + return '/' . $referencePath; + } + + $basePath = $base['path'] ?? ''; + $slash = \strrpos($basePath, '/'); + if ($slash === false) { + return $referencePath; + } + + return \substr($basePath, 0, $slash + 1) . $referencePath; + } + + /** + * RFC 3986 §5.2.4 — remove "." and ".." segments from a path. + */ + private static function removeDotSegments(string $path): string + { + $output = ''; + $input = $path; + + while ($input !== '') { + if (\str_starts_with($input, '../')) { + $input = \substr($input, 3); + } elseif (\str_starts_with($input, './')) { + $input = \substr($input, 2); + } elseif (\str_starts_with($input, '/./')) { + $input = '/' . \substr($input, 3); + } elseif ($input === '/.') { + $input = '/'; + } elseif (\str_starts_with($input, '/../')) { + $input = '/' . \substr($input, 4); + $slash = \strrpos($output, '/'); + $output = $slash === false ? '' : \substr($output, 0, $slash); + } elseif ($input === '/..') { + $input = '/'; + $slash = \strrpos($output, '/'); + $output = $slash === false ? '' : \substr($output, 0, $slash); + } elseif ($input === '.' || $input === '..') { + $input = ''; + } else { + $start = ($input[0] === '/') ? 1 : 0; + $next = \strpos($input, '/', $start); + if ($next === false) { + $output .= $input; + $input = ''; + } else { + $output .= \substr($input, 0, $next); + $input = \substr($input, $next); + } + } + } + + return $output; + } } diff --git a/src/Appwrite/Utopia/Response/Model/Any.php b/src/Appwrite/Utopia/Response/Model/Any.php index 0ae3040a1d..6863748ac8 100644 --- a/src/Appwrite/Utopia/Response/Model/Any.php +++ b/src/Appwrite/Utopia/Response/Model/Any.php @@ -12,26 +12,6 @@ class Any extends Model */ protected bool $any = true; - /** - * JSON wire-format key under which extra/dynamic attributes are exposed in - * generated SDK models (e.g. Document's `data` slot). Default null means - * SDK templates fall back to their hardcoded "data" key. Set this on - * subclasses (via setAdditionalPropertiesKey) to use a custom key like - * "metadata" while still benefiting from the generic `Model` mapping. - */ - protected ?string $additionalPropertiesKey = null; - - public function setAdditionalPropertiesKey(string $key): self - { - $this->additionalPropertiesKey = $key; - return $this; - } - - public function getAdditionalPropertiesKey(): ?string - { - return $this->additionalPropertiesKey; - } - /** * Get Name * diff --git a/src/Appwrite/Utopia/Response/Model/Embedding.php b/src/Appwrite/Utopia/Response/Model/Embedding.php index 9fce913723..1ac2abb002 100644 --- a/src/Appwrite/Utopia/Response/Model/Embedding.php +++ b/src/Appwrite/Utopia/Response/Model/Embedding.php @@ -22,7 +22,7 @@ class Embedding extends Model ->addRule('model', [ 'type' => self::TYPE_STRING, 'description' => 'Embedding model used to generate embeddings.', - 'example' => 'embeddinggemma' + 'example' => 'nomic-embed-text' ]) ->addRule('dimension', [ 'type' => self::TYPE_INTEGER, diff --git a/src/Appwrite/Utopia/Response/Model/PolicyList.php b/src/Appwrite/Utopia/Response/Model/PolicyList.php index 09548fedcf..11704cdbff 100644 --- a/src/Appwrite/Utopia/Response/Model/PolicyList.php +++ b/src/Appwrite/Utopia/Response/Model/PolicyList.php @@ -17,23 +17,31 @@ class PolicyList extends Model 'example' => 9, ]) ->addRule('policies', [ - 'type' => [ - Response::MODEL_POLICY_PASSWORD_DICTIONARY, - Response::MODEL_POLICY_PASSWORD_HISTORY, - Response::MODEL_POLICY_PASSWORD_PERSONAL_DATA, - Response::MODEL_POLICY_SESSION_ALERT, - Response::MODEL_POLICY_SESSION_DURATION, - Response::MODEL_POLICY_SESSION_INVALIDATION, - Response::MODEL_POLICY_SESSION_LIMIT, - Response::MODEL_POLICY_USER_LIMIT, - Response::MODEL_POLICY_MEMBERSHIP_PRIVACY, - ], + 'type' => $this->getPolicyModels(), 'description' => 'List of policies.', 'default' => [], 'array' => true, ]); } + /** + * @return array + */ + protected function getPolicyModels(): array + { + return [ + Response::MODEL_POLICY_PASSWORD_DICTIONARY, + Response::MODEL_POLICY_PASSWORD_HISTORY, + Response::MODEL_POLICY_PASSWORD_PERSONAL_DATA, + Response::MODEL_POLICY_SESSION_ALERT, + Response::MODEL_POLICY_SESSION_DURATION, + Response::MODEL_POLICY_SESSION_INVALIDATION, + Response::MODEL_POLICY_SESSION_LIMIT, + Response::MODEL_POLICY_USER_LIMIT, + Response::MODEL_POLICY_MEMBERSHIP_PRIVACY, + ]; + } + public function getName(): string { return 'Policies List'; diff --git a/src/Appwrite/Utopia/Response/Model/Presence.php b/src/Appwrite/Utopia/Response/Model/Presence.php index ecc3755bae..809018d29c 100644 --- a/src/Appwrite/Utopia/Response/Model/Presence.php +++ b/src/Appwrite/Utopia/Response/Model/Presence.php @@ -3,9 +3,10 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; +use Appwrite\Utopia\Response\Model; use Utopia\Database\Document as DatabaseDocument; -class Presence extends Any +class Presence extends Model { public function getName(): string { @@ -19,9 +20,6 @@ class Presence extends Any public function __construct() { - // Expose user-defined extras under "metadata" instead of the SDK default "data" - $this->setAdditionalPropertiesKey('metadata'); - $this ->addRule('$id', [ 'type' => self::TYPE_STRING, @@ -73,9 +71,14 @@ class Presence extends Any 'required' => false, 'default' => null, 'example' => self::TYPE_DATETIME_EXAMPLE, + ]) + ->addRule('metadata', [ + 'type' => self::TYPE_JSON, + 'description' => 'Presence metadata.', + 'required' => false, + 'default' => null, + 'example' => ['key' => 'value'], ]); - // User-defined extras flow through Any's generic mapping, surfaced under - // the "metadata" key declared via setAdditionalPropertiesKey() above. } public function filter(DatabaseDocument $document): DatabaseDocument diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index dddcb14491..017131bfbe 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -2035,7 +2035,7 @@ class UsageTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'], ], $this->getHeaders()), [ - 'model' => 'embeddinggemma', + 'model' => 'nomic-embed-text', 'texts' => ['usage test warm-up ' . $callCount], ] ); @@ -2059,7 +2059,7 @@ class UsageTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'], ], $this->getHeaders()), [ - 'model' => 'embeddinggemma', + 'model' => 'nomic-embed-text', 'texts' => ['usage test text ' . $i], ] ); diff --git a/tests/e2e/Services/Databases/VectorsDB/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/VectorsDB/DatabasesCustomServerTest.php index 9564b76079..b34aaef99e 100644 --- a/tests/e2e/Services/Databases/VectorsDB/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/VectorsDB/DatabasesCustomServerTest.php @@ -582,7 +582,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'model' => 'embeddinggemma', + 'model' => 'nomic-embed-text', 'texts' => [ 'hello world', 'second sentence', @@ -616,7 +616,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'model' => 'embeddinggemma', + 'model' => 'nomic-embed-text', 'texts' => [ 'valid text', 123, // invalid, not a string diff --git a/tests/e2e/Services/Databases/VectorsDBCustomServerTest.php b/tests/e2e/Services/Databases/VectorsDBCustomServerTest.php index ceb672443e..12fef2ddd2 100644 --- a/tests/e2e/Services/Databases/VectorsDBCustomServerTest.php +++ b/tests/e2e/Services/Databases/VectorsDBCustomServerTest.php @@ -581,7 +581,7 @@ class VectorsDBCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'model' => 'embeddinggemma', + 'model' => 'nomic-embed-text', 'texts' => [ 'hello world', 'second sentence', @@ -615,7 +615,7 @@ class VectorsDBCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'model' => 'embeddinggemma', + 'model' => 'nomic-embed-text', 'texts' => [ 'valid text', 123, // invalid, not a string diff --git a/tests/e2e/Services/Messaging/MessagingBase.php b/tests/e2e/Services/Messaging/MessagingBase.php index 43dc58d44c..c291dd94c9 100644 --- a/tests/e2e/Services/Messaging/MessagingBase.php +++ b/tests/e2e/Services/Messaging/MessagingBase.php @@ -1047,6 +1047,95 @@ trait MessagingBase } } + public function testSesProvider(): void + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + // Create with full credentials but no fromEmail, so the provider stays disabled. + $response = $this->client->call(Client::METHOD_POST, '/messaging/providers/ses', $headers, [ + 'providerId' => ID::unique(), + 'name' => 'SES1', + 'accessKey' => 'my-access-key', + 'secretKey' => 'my-secret-key', + 'region' => 'us-east-1', + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals('SES1', $response['body']['name']); + $this->assertEquals('ses', $response['body']['provider']); + $this->assertEquals('email', $response['body']['type']); + $this->assertEquals(false, $response['body']['enabled']); + $this->assertEquals('my-access-key', $response['body']['credentials']['accessKey']); + $this->assertEquals('my-secret-key', $response['body']['credentials']['secretKey']); + $this->assertEquals('us-east-1', $response['body']['credentials']['region']); + $this->assertArrayHasKey('fromEmail', $response['body']['options']); + $this->assertArrayNotHasKey('accessKey', $response['body']['options']); + + $providerId = $response['body']['$id']; + + // Create enabled: all credentials plus fromEmail present. + $enabledResponse = $this->client->call(Client::METHOD_POST, '/messaging/providers/ses', $headers, [ + 'providerId' => ID::unique(), + 'name' => 'SES-enabled', + 'accessKey' => 'my-access-key', + 'secretKey' => 'my-secret-key', + 'region' => 'us-east-1', + 'fromName' => 'Sender Name', + 'fromEmail' => 'sender-email@my-domain.com', + 'enabled' => true, + ]); + + $this->assertEquals(201, $enabledResponse['headers']['status-code']); + $this->assertEquals(true, $enabledResponse['body']['enabled']); + $this->assertEquals('sender-email@my-domain.com', $enabledResponse['body']['options']['fromEmail']); + + // Sparse update: change only the region, credentials must be preserved. + $updateResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ + 'name' => 'SES2', + 'region' => 'eu-west-1', + ]); + + $this->assertEquals(200, $updateResponse['headers']['status-code']); + $this->assertEquals('SES2', $updateResponse['body']['name']); + $this->assertEquals('eu-west-1', $updateResponse['body']['credentials']['region']); + $this->assertEquals('my-access-key', $updateResponse['body']['credentials']['accessKey']); + $this->assertEquals('my-secret-key', $updateResponse['body']['credentials']['secretKey']); + + // Regression: enabling while fromEmail is still empty must be rejected, not silently enabled. + $enableWithoutFromEmail = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ + 'enabled' => true, + ]); + + $this->assertEquals(400, $enableWithoutFromEmail['headers']['status-code']); + $this->assertEquals('provider_missing_credentials', $enableWithoutFromEmail['body']['type']); + + // Enable the first provider once fromEmail is supplied. + $enableResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ + 'fromEmail' => 'sender-email@my-domain.com', + 'enabled' => true, + ]); + + $this->assertEquals(200, $enableResponse['headers']['status-code']); + $this->assertEquals(true, $enableResponse['body']['enabled']); + + // Updating an SES provider through a different provider route is rejected. + $wrongTypeResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/sendgrid/' . $providerId, $headers, [ + 'name' => 'Wrong', + ]); + + $this->assertEquals(400, $wrongTypeResponse['headers']['status-code']); + + $deleteResponse = $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, $headers); + $this->assertEquals(204, $deleteResponse['headers']['status-code']); + + $deleteEnabledResponse = $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $enabledResponse['body']['$id'], $headers); + $this->assertEquals(204, $deleteEnabledResponse['headers']['status-code']); + } + public function testCreateTopic(): void { $response1 = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ diff --git a/tests/e2e/Services/Presences/PresenceConsoleClientTest.php b/tests/e2e/Services/Presences/PresenceConsoleClientTest.php index 1f1c52a234..63cb509a37 100644 --- a/tests/e2e/Services/Presences/PresenceConsoleClientTest.php +++ b/tests/e2e/Services/Presences/PresenceConsoleClientTest.php @@ -6,6 +6,11 @@ use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideConsole; +use Utopia\Database\Helpers\ID; +use Utopia\Database\Helpers\Permission; +use Utopia\Database\Helpers\Role; +use WebSocket\Client as WebSocketClient; +use WebSocket\TimeoutException; class PresenceConsoleClientTest extends Scope { @@ -59,4 +64,146 @@ class PresenceConsoleClientTest extends Scope $this->assertIsNumeric($response['body']['usersOnlineTotal']); $this->assertIsArray($response['body']['presences']); } + + public function testConsolePresenceUpsertAndUpdateBroadcastRealtime(): void + { + $user = $this->getUser(); + $presenceId = ID::unique(); + + $client = $this->openConsolePresenceSocket($user, $presenceId); + $needsCleanup = false; + + try { + $upsertMetadata = ['testRunId' => ID::unique(), 'case' => 'console-upsert']; + $upsert = $this->client->call( + Client::METHOD_PUT, + '/presences/' . $presenceId, + \array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + ], $this->getHeaders()), + [ + 'status' => 'online', + 'metadata' => $upsertMetadata, + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ] + ); + $this->assertSame(200, $upsert['headers']['status-code']); + $needsCleanup = true; + + $upsertEvent = $this->receivePresenceFrame($client, $presenceId, 'upsert'); + $this->assertSame('online', $upsertEvent['data']['payload']['status'] ?? null); + $this->assertSame($upsertMetadata, $upsertEvent['data']['payload']['metadata'] ?? null); + $this->assertSame($user['$id'], $upsertEvent['data']['payload']['userId'] ?? null); + // Internal fields must not leak through the realtime broadcast. + $this->assertArrayNotHasKey('userInternalId', $upsertEvent['data']['payload']); + $this->assertArrayNotHasKey('permissionsHash', $upsertEvent['data']['payload']); + $this->assertArrayNotHasKey('hostname', $upsertEvent['data']['payload']); + + $updateMetadata = ['testRunId' => $upsertMetadata['testRunId'], 'case' => 'console-update']; + $update = $this->client->call( + Client::METHOD_PATCH, + '/presences/' . $presenceId, + \array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + ], $this->getHeaders()), + [ + 'status' => 'away', + 'metadata' => $updateMetadata, + ] + ); + $this->assertSame(200, $update['headers']['status-code']); + + $updateEvent = $this->receivePresenceFrame($client, $presenceId, 'update'); + $this->assertSame('away', $updateEvent['data']['payload']['status'] ?? null); + $this->assertSame($updateMetadata, $updateEvent['data']['payload']['metadata'] ?? null); + $this->assertSame($user['$id'], $updateEvent['data']['payload']['userId'] ?? null); + } finally { + $client->close(); + + if ($needsCleanup) { + // Drop the row so reruns / parallel users don't accumulate orphaned presences. + $this->client->call( + Client::METHOD_DELETE, + '/presences/' . $presenceId, + \array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + ], $this->getHeaders()) + ); + } + } + } + + private function openConsolePresenceSocket(array $user, string $presenceId): WebSocketClient + { + $queryString = \http_build_query([ + 'project' => 'console', + 'channels' => ['presences', 'presences.' . $presenceId], + ]); + + $client = new WebSocketClient( + 'ws://appwrite.test/v1/realtime?' . $queryString, + [ + 'headers' => [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_console=' . $user['session'], + ], + 'timeout' => 1, + ] + ); + + $connected = \json_decode($client->receive(), true); + $this->assertSame('connected', $connected['type'] ?? null); + + return $client; + } + + private function receivePresenceFrame( + WebSocketClient $client, + string $presenceId, + string $action, + int $timeoutMs = 3000 + ): array { + $deadline = \microtime(true) + ($timeoutMs / 1000); + $lastFrame = []; + + while (\microtime(true) < $deadline) { + try { + $raw = $client->receive(); + } catch (TimeoutException) { + continue; + } + + $frame = \json_decode($raw, true); + if (!\is_array($frame)) { + continue; + } + + $lastFrame = $frame; + if ( + ($frame['type'] ?? null) === 'event' + && ($frame['data']['payload']['$id'] ?? null) === $presenceId + && \in_array( + 'presences.' . $presenceId . '.' . $action, + $frame['data']['events'] ?? [], + true + ) + ) { + $this->assertContains('presences', $frame['data']['channels'] ?? []); + $this->assertContains('presences.' . $presenceId, $frame['data']['channels'] ?? []); + return $frame; + } + } + + $this->fail( + 'Timed out waiting for presences.' . $presenceId . '.' . $action + . ' frame on console. Last frame: ' . \json_encode($lastFrame) + ); + } } diff --git a/tests/e2e/Services/Project/PoliciesBase.php b/tests/e2e/Services/Project/PoliciesBase.php index 43c09b55c3..71fc85919f 100644 --- a/tests/e2e/Services/Project/PoliciesBase.php +++ b/tests/e2e/Services/Project/PoliciesBase.php @@ -7,13 +7,12 @@ use Utopia\Database\Query; trait PoliciesBase { - // ========================================================================= - // Get Policy - // ========================================================================= - - public function testGetPolicy(): void + /** + * @return array> + */ + protected function getExpectedPolicies(): array { - $expectedFields = [ + return [ 'password-dictionary' => ['enabled'], 'password-history' => ['total'], 'password-personal-data' => ['enabled'], @@ -24,8 +23,15 @@ trait PoliciesBase 'user-limit' => ['total'], 'membership-privacy' => ['userId', 'userEmail', 'userPhone', 'userName', 'userMFA'], ]; + } - foreach ($expectedFields as $policyId => $fields) { + // ========================================================================= + // Get Policy + // ========================================================================= + + public function testGetPolicy(): void + { + foreach ($this->getExpectedPolicies() as $policyId => $fields) { $response = $this->getPolicy($policyId); $this->assertSame(200, $response['headers']['status-code']); @@ -125,25 +131,25 @@ trait PoliciesBase { $response = $this->listPolicies(); + $expected = $this->getExpectedPolicies(); + $this->assertSame(200, $response['headers']['status-code']); $this->assertArrayHasKey('policies', $response['body']); $this->assertArrayHasKey('total', $response['body']); $this->assertIsArray($response['body']['policies']); $this->assertIsInt($response['body']['total']); - $this->assertSame(9, $response['body']['total']); - $this->assertCount(9, $response['body']['policies']); + + // Unpaginated list: total reflects exactly what was returned. + $this->assertSame($response['body']['total'], \count($response['body']['policies'])); + + // At least the known policies are present; downstream may add more. + $this->assertGreaterThanOrEqual(\count($expected), $response['body']['total']); $policyIds = \array_column($response['body']['policies'], '$id'); - $this->assertContains('password-dictionary', $policyIds); - $this->assertContains('password-history', $policyIds); - $this->assertContains('password-personal-data', $policyIds); - $this->assertContains('session-alert', $policyIds); - $this->assertContains('session-duration', $policyIds); - $this->assertContains('session-invalidation', $policyIds); - $this->assertContains('session-limit', $policyIds); - $this->assertContains('user-limit', $policyIds); - $this->assertContains('membership-privacy', $policyIds); + foreach (\array_keys($expected) as $policyId) { + $this->assertContains($policyId, $policyIds); + } } public function testListPoliciesResponseModel(): void @@ -161,19 +167,13 @@ trait PoliciesBase $byId[$policy['$id']] = $policy; } - $this->assertArrayHasKey('enabled', $byId['password-dictionary']); - $this->assertArrayHasKey('total', $byId['password-history']); - $this->assertArrayHasKey('enabled', $byId['password-personal-data']); - $this->assertArrayHasKey('enabled', $byId['session-alert']); - $this->assertArrayHasKey('duration', $byId['session-duration']); - $this->assertArrayHasKey('enabled', $byId['session-invalidation']); - $this->assertArrayHasKey('total', $byId['session-limit']); - $this->assertArrayHasKey('total', $byId['user-limit']); - $this->assertArrayHasKey('userId', $byId['membership-privacy']); - $this->assertArrayHasKey('userEmail', $byId['membership-privacy']); - $this->assertArrayHasKey('userPhone', $byId['membership-privacy']); - $this->assertArrayHasKey('userName', $byId['membership-privacy']); - $this->assertArrayHasKey('userMFA', $byId['membership-privacy']); + foreach ($this->getExpectedPolicies() as $policyId => $fields) { + $this->assertArrayHasKey($policyId, $byId); + + foreach ($fields as $field) { + $this->assertArrayHasKey($field, $byId[$policyId]); + } + } } public function testListPoliciesReflectsUpdates(): void @@ -226,7 +226,15 @@ trait PoliciesBase $this->assertSame(200, $response['headers']['status-code']); $this->assertSame(0, $response['body']['total']); - $this->assertCount(9, $response['body']['policies']); + + // total=false skips the count but still returns every policy; downstream may add more. + $this->assertGreaterThanOrEqual(\count($this->getExpectedPolicies()), \count($response['body']['policies'])); + + $policyIds = \array_column($response['body']['policies'], '$id'); + + foreach (\array_keys($this->getExpectedPolicies()) as $policyId) { + $this->assertContains($policyId, $policyIds); + } } public function testListPoliciesWithLimit(): void @@ -237,7 +245,9 @@ trait PoliciesBase $this->assertSame(200, $response['headers']['status-code']); $this->assertCount(1, $response['body']['policies']); - $this->assertSame(9, $response['body']['total']); + + // limit caps the page, not the total; downstream may add more policies. + $this->assertGreaterThanOrEqual(\count($this->getExpectedPolicies()), $response['body']['total']); } public function testListPoliciesWithOffset(): void diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 375e526fcf..f654083d80 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -1289,6 +1289,100 @@ trait StorageBase $this->assertEquals(204, $deleteBucketResponse['headers']['status-code']); } + public function testCreateBucketFileChunkedUploadWithoutReadPermission(): void + { + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket Chunked Upload No Read Permission', + 'fileSecurity' => true, + 'permissions' => [ + Permission::create(Role::any()), + ], + ]); + + $this->assertEquals(201, $bucket['headers']['status-code']); + + $bucketId = $bucket['body']['$id']; + $source = __DIR__ . "/../../../resources/disk-a/large-file.mp4"; + $totalSize = \filesize($source); + $chunkSize = 5 * 1024 * 1024; + $mimeType = mime_content_type($source); + $fileId = ID::unique(); + $uploadedFileId = null; + $chunksTotal = (int) ceil($totalSize / $chunkSize); + + $this->assertGreaterThan($chunkSize, $totalSize, 'Test file must span at least 2 chunks'); + + try { + $handle = fopen($source, 'rb'); + $this->assertNotFalse($handle, "Could not open test resource: $source"); + + $permissions = [ + Permission::delete(Role::user($this->getUser()['$id'])), + ]; + $upload = null; + + for ($chunk = 0; $chunk < $chunksTotal; $chunk++) { + $start = $chunk * $chunkSize; + $chunkData = fread($handle, min($chunkSize, $totalSize - $start)); + $end = $start + strlen($chunkData) - 1; + + $headers = [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'content-range' => 'bytes ' . $start . '-' . $end . '/' . $totalSize, + ]; + + if (!empty($uploadedFileId)) { + $headers['x-appwrite-id'] = $uploadedFileId; + } + + $upload = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge($headers, $this->getHeaders()), [ + 'fileId' => $fileId, + 'file' => new CURLFile('data://' . $mimeType . ';base64,' . base64_encode($chunkData), $mimeType, 'large-file.mp4'), + 'permissions' => $permissions, + ]); + + $this->assertEquals(201, $upload['headers']['status-code'], $upload['body']['message'] ?? ''); + $this->assertEquals($chunk + 1, $upload['body']['chunksUploaded']); + + $uploadedFileId ??= $upload['body']['$id']; + } + + fclose($handle); + + if ($upload === null) { + $this->fail('Expected at least one chunk upload response.'); + } + + $this->assertNotContains(Permission::read(Role::user($this->getUser()['$id'])), $upload['body']['$permissions']); + $this->assertEquals($chunksTotal, $upload['body']['chunksTotal']); + $this->assertEquals($chunksTotal, $upload['body']['chunksUploaded']); + } finally { + if (isset($handle) && \is_resource($handle)) { + fclose($handle); + } + + if (!empty($uploadedFileId)) { + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $uploadedFileId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + } + public function testCreateBucketFileOutOfOrder(): void { // Create a bucket for this test diff --git a/tests/e2e/Services/TablesDB/DatabasesStringTypesTest.php b/tests/e2e/Services/TablesDB/DatabasesStringTypesTest.php index b7fd982a6d..cb25e8a305 100644 --- a/tests/e2e/Services/TablesDB/DatabasesStringTypesTest.php +++ b/tests/e2e/Services/TablesDB/DatabasesStringTypesTest.php @@ -795,7 +795,9 @@ class DatabasesStringTypesTest extends Scope $this->assertEquals(204, $deleteVarchar['headers']['status-code']); - // Poll until async deletion completes + // Poll until async deletion completes. Timeout raised to 60s to match + // the migration-polling and schema-polling defaults used elsewhere — + // shared-mode CI delete-worker contention can exceed the original 30s. $this->assertEventually(function () use ($databaseId, $tableId) { $response = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/varchar_min', [ 'content-type' => 'application/json', @@ -803,6 +805,6 @@ class DatabasesStringTypesTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'], ]); $this->assertEquals(404, $response['headers']['status-code']); - }, 30000, 250); + }, 60000, 250); } } diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php new file mode 100644 index 0000000000..02b1ee55d4 --- /dev/null +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -0,0 +1,808 @@ + 'provider1', + '$sequence' => '1', + 'enabled' => true, + 'type' => MESSAGE_TYPE_EMAIL, + 'provider' => 'mock', + 'name' => 'Mock Email', + 'credentials' => [], + 'options' => [ + 'fromName' => 'Appwrite', + 'fromEmail' => 'noreply@example.com', + ], + ]); + } + + private function message(): Document + { + return new Document([ + '$id' => 'message1', + '$sequence' => '1', + 'providerType' => MESSAGE_TYPE_EMAIL, + 'topics' => ['topic1'], + 'targets' => [], + 'users' => [], + 'data' => [ + 'subject' => 'Hello', + 'content' => 'World', + 'html' => false, + ], + ]); + } + + /** + * Invoke the worker's private {@see Messaging::retrySend()} for an email batch. + * + * @param array $batch + * @return array{delivered: int, errors: array} + */ + private function retrySend(array $batch, EmailAdapter $adapter, Database $database): array + { + $worker = new TestableMessaging(); + $method = new \ReflectionMethod(Messaging::class, 'retrySend'); + + $device = new Local(\sys_get_temp_dir()); + $project = new Document(['$id' => 'project1', '$sequence' => '1']); + + /** @var array{delivered: int, errors: array} $result */ + $result = $method->invoke( + $worker, + $batch, + $this->message(), + $this->provider(), + MESSAGE_TYPE_EMAIL, + $adapter, + $database, + $device, + $project + ); + + return $result; + } + + public function testIsRetryableErrorMatchesTransientFailures(): void + { + $worker = new TestableMessaging(); + $method = new \ReflectionMethod(Messaging::class, 'isRetryableError'); + + $retryable = [ + 'Throttling', + 'rate exceeded', + 'Rate limit reached', + 'Too Many Requests', + '429 Too Many Requests', + 'quota exceeded', + 'Service Unavailable', + '503 Service Unavailable', + 'Connection timed out', + 'Request timeout', + 'Service temporarily unavailable', + ]; + + foreach ($retryable as $error) { + $this->assertTrue( + $method->invoke($worker, $error), + "Expected '{$error}' to be classified as retryable" + ); + } + + $permanent = [ + 'invalid recipient', + 'Mailbox does not exist', + 'Authentication failed', + 'Expired device token', + ]; + + foreach ($permanent as $error) { + $this->assertFalse( + $method->invoke($worker, $error), + "Expected '{$error}' to be classified as permanent" + ); + } + + $this->assertFalse($method->invoke($worker, null), 'A null error must never be retryable'); + $this->assertFalse($method->invoke($worker, ''), 'An empty error must never be retryable'); + } + + public function testThrottledBatchIsRetriedThenFullyDelivered(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com', 'user4@example.com', 'user5@example.com']; + + // First attempt throttles every recipient; the script then runs out so the second attempt succeeds. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): string => 'Rate limit exceeded, please retry', + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // Retried exactly once, then everyone delivered with no double-counting and no terminal errors. + $this->assertSame(2, $adapter->sendCalls, 'Throttled batch must be retried once'); + $this->assertSame(\count($batch), $result['delivered']); + $this->assertEmpty($result['errors']); + + // Both attempts targeted the full recipient set (none had succeeded on the first attempt). + $this->assertSame($batch, $adapter->recipientsPerCall[0]); + $this->assertSame($batch, $adapter->recipientsPerCall[1]); + } + + public function testRetryTargetsOnlyThrottledRecipients(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com', 'user4@example.com', 'user5@example.com']; + + // user2@ and user4@ throttle on the first attempt; everyone else succeeds. The second attempt has no + // script step, so the two retried recipients then succeed. + $throttled = ['user2@example.com', 'user4@example.com']; + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => \in_array($recipient, $throttled, true) + ? 'Too Many Requests' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + $this->assertSame(2, $adapter->sendCalls, 'A partially throttled batch must be retried once'); + + // The retry must re-send to EXACTLY the throttled recipients — never the ones already delivered. + $this->assertSame($throttled, $adapter->recipientsPerCall[1]); + + // Every recipient is ultimately delivered and counted exactly once. + $this->assertSame(\count($batch), $result['delivered']); + $this->assertEmpty($result['errors']); + } + + public function testNonRetryableFailureIsNotRetried(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com']; + + // user1@ permanently fails with a non-retryable error; the rest succeed. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => $recipient === 'user1@example.com' + ? 'Invalid recipient address' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // A permanent failure is recorded immediately, never retried. + $this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry'); + $this->assertSame(\count($batch) - 1, $result['delivered']); + $this->assertCount(1, $result['errors']); + $this->assertStringContainsString('user1@example.com', $result['errors'][0]); + $this->assertStringContainsString('Invalid recipient address', $result['errors'][0]); + } + + public function testExhaustedRetriesRecordTerminalFailure(): void + { + $batch = ['user1@example.com', 'user2@example.com']; + + // Every attempt throttles, for more steps than MESSAGE_SEND_MAX_RETRIES, so retries are exhausted. + $script = \array_fill( + 0, + MESSAGE_SEND_MAX_RETRIES + 2, + static fn (string $recipient): string => 'Service unavailable, throttled' + ); + $adapter = new ScriptedEmailAdapter($script); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // The batch is attempted exactly MESSAGE_SEND_MAX_RETRIES times, then the still-throttled recipients + // become terminal failures with nothing delivered. + $this->assertSame(MESSAGE_SEND_MAX_RETRIES, $adapter->sendCalls, 'Attempts must be capped at MESSAGE_SEND_MAX_RETRIES'); + $this->assertSame(0, $result['delivered']); + $this->assertCount(\count($batch), $result['errors']); + } + + public function testDeliveryErrorsAreCappedWhileDeliveredCountIsExact(): void + { + // More failing recipients than the retained-error cap, plus a known number of successes, so the + // delivered tally stays exact even though the error list is truncated. + $failingCount = MESSAGE_DELIVERY_ERRORS_LIMIT + 20; + $succeedingCount = 100; + + $batch = []; + for ($i = 1; $i <= $failingCount; $i++) { + $batch[] = "fail{$i}@example.com"; + } + for ($i = 1; $i <= $succeedingCount; $i++) { + $batch[] = "ok{$i}@example.com"; + } + + // Permanent (non-retryable) failure for every "fail*" recipient; everyone else is delivered. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => \str_starts_with($recipient, 'fail') + ? 'Invalid recipient address' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // Errors are capped, but the delivered count reflects every success exactly. + $this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $result['errors'], 'Delivery errors must be capped'); + $this->assertSame($succeedingCount, $result['delivered'], 'Delivered count stays exact despite the error cap'); + $this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry'); + } + + public function testExpiredTokenCleanupFailureDoesNotCorruptAccounting(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com']; + + // The cleanup path always throws, but the worker must isolate it: the send result is what's accounted. + $database = new ExpiredTokenCleanupFailingDatabase(); + + // user1@ comes back with an expired device token (triggering cleanup, which then throws); the other + // two recipients are delivered. The expired-token error is terminal, never retryable. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => $recipient === 'user1@example.com' + ? 'Expired device token' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, $database); + + // The worker reached the cleanup (so the isolation is genuinely exercised) and it threw. + $this->assertGreaterThan(0, $database->targetUpdateAttempts, 'Expired-token cleanup must have been attempted'); + + // The send was attempted exactly once: a thrown cleanup must not look like a transient send failure + // and must not trigger a retry. + $this->assertSame(1, $adapter->sendCalls, 'Cleanup failure must not trigger a send retry'); + + // Accounting reflects the send result, not the cleanup throw: two delivered, one terminal failure. + $this->assertSame(\count($batch) - 1, $result['delivered']); + + // The recorded error is the provider's expired-token failure for the real recipient — proving the + // database exception was swallowed rather than misattributed as the send error. + $this->assertCount(1, $result['errors']); + $this->assertStringContainsString('user1@example.com', $result['errors'][0]); + $this->assertStringContainsString('Expired device token', $result['errors'][0]); + $this->assertStringNotContainsString('Transient database error', $result['errors'][0]); + } + + public function testRecipientsArePaginatedWithoutSubqueryFilter(): void + { + // Always span several pages regardless of the configured page size, so cursor pagination is exercised. + $count = MESSAGE_RECIPIENTS_PAGE_SIZE * 2 + 500; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets']); + + $identifiers = $this->collectStream($database, topicIds: ['topic1']); + + $topicReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics')); + $subscriberReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'subscribers')); + + // The topic's `targets` attribute (the subQueryTopicTargets filter) is never materialized: topics are + // only read with a $sequence projection, and recipients come from the subscribers collection instead. + $this->assertNotEmpty($topicReads); + foreach ($topicReads as $call) { + $this->assertNotContains('targets', $this->selectFields($call['queries']), 'Topic read must not select the targets subquery attribute'); + } + + // Subscribers are walked in bounded pages via cursorAfter; no query ever requests a runaway page size. + $expectedPages = (int) \ceil($count / MESSAGE_RECIPIENTS_PAGE_SIZE); + $this->assertCount($expectedPages, $subscriberReads); + + foreach ($subscriberReads as $index => $call) { + $this->assertSame(MESSAGE_RECIPIENTS_PAGE_SIZE, $this->limitOf($call['queries'])); + $this->assertSame('100', $this->topicInternalIdOf($call['queries']), 'Subscribers must be filtered by the topic internal id'); + + if ($index === 0) { + $this->assertFalse($this->hasCursor($call['queries']), 'First page must not carry a cursor'); + } else { + $this->assertTrue($this->hasCursor($call['queries']), 'Subsequent pages must paginate with cursorAfter'); + } + } + + // No query in the whole stream issues a runaway (>= 1,000,000) limit. + foreach ($database->findCalls as $call) { + $this->assertLessThan(1_000_000, $this->limitOf($call['queries']), 'No query may request a million-row page'); + } + + // The duplicate identifier collapses: one row per distinct subscriber identifier resolved. + $this->assertCount($count, $identifiers); + for ($i = 1; $i <= $count; $i++) { + $this->assertArrayHasKey("user{$i}@example.com", $identifiers); + } + } + + public function testStreamDeduplicatesIdentifiersWithinPage(): void + { + // Two subscribers in a single page resolve to targets sharing one identifier; the page must collapse + // them to a single recipient entry. + $subscribers = [ + new Document(['$id' => 'sub1', '$sequence' => '1', 'topicInternalId' => '100', 'targetInternalId' => '1', 'providerType' => MESSAGE_TYPE_EMAIL]), + new Document(['$id' => 'sub2', '$sequence' => '2', 'topicInternalId' => '100', 'targetInternalId' => '2', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $targets = [ + '1' => new Document(['$id' => 'target1', '$sequence' => '1', 'providerId' => null, 'identifier' => 'dup@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + '2' => new Document(['$id' => 'target2', '$sequence' => '2', 'providerId' => null, 'identifier' => 'dup@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $database = new RecordingDatabase($subscribers, $targets); + + $pages = $this->collectPages($database, topicIds: ['topic1']); + + $this->assertCount(1, $pages, 'A single subscriber page yields a single grouped result'); + $providerId = $this->provider()->getId(); + $this->assertArrayHasKey($providerId, $pages[0]); + $this->assertSame(['dup@example.com'], \array_keys($pages[0][$providerId]), 'Duplicate identifiers must collapse to one'); + } + + public function testStreamResolvesUsersAndDirectTargets(): void + { + $userTargets = [ + new Document(['$id' => 'utarget1', '$sequence' => '10', 'providerId' => null, 'identifier' => 'byuser@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $directTargets = [ + new Document(['$id' => 'dtarget1', '$sequence' => '20', 'providerId' => null, 'identifier' => 'bytarget@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $database = new RecordingDatabase([], [], $userTargets, $directTargets); + + $identifiers = $this->collectStream($database, userIds: ['user-a'], targetIds: ['target-b']); + + // No topics in play: only the user and direct-target queries run, and both identifiers resolve. + $this->assertSame([], \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics'))); + $this->assertArrayHasKey('byuser@example.com', $identifiers); + $this->assertArrayHasKey('bytarget@example.com', $identifiers); + $this->assertCount(2, $identifiers); + } + + /** + * Build a topic-backed dataset: $count subscribers, each pointing at one email target. A single identifier + * is duplicated so per-page dedup is exercised when the dataset fits in one page. + * + * @return array{subscribers: array, targets: array} + */ + private function topicDataset(int $count): array + { + $subscribers = []; + $targets = []; + + for ($i = 1; $i <= $count; $i++) { + $sequence = (string) $i; + $subscribers[] = new Document([ + '$id' => 'sub' . $i, + '$sequence' => $sequence, + 'topicInternalId' => '100', + 'targetInternalId' => $sequence, + 'providerType' => MESSAGE_TYPE_EMAIL, + ]); + $targets[$sequence] = new Document([ + '$id' => 'target' . $i, + '$sequence' => $sequence, + 'providerId' => null, + 'identifier' => "user{$i}@example.com", + 'providerType' => MESSAGE_TYPE_EMAIL, + ]); + } + + return ['subscribers' => $subscribers, 'targets' => $targets]; + } + + /** + * Drive {@see Messaging::streamRecipients()} and merge every yielded page into a single identifier map. + * + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return array + */ + private function collectStream( + RecordingDatabase $database, + array $topicIds = [], + array $userIds = [], + array $targetIds = [] + ): array { + $identifiers = []; + + foreach ($this->collectPages($database, $topicIds, $userIds, $targetIds) as $page) { + foreach ($page as $perProvider) { + foreach ($perProvider as $identifier => $_) { + $identifiers[$identifier] = null; + } + } + } + + return $identifiers; + } + + /** + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return array>> + */ + private function collectPages( + RecordingDatabase $database, + array $topicIds = [], + array $userIds = [], + array $targetIds = [] + ): array { + $method = new \ReflectionMethod(Messaging::class, 'streamRecipients'); + $worker = new TestableMessaging(); + + /** @var \Generator>> $generator */ + $generator = $method->invoke( + $worker, + $database, + $topicIds, + $userIds, + $targetIds, + MESSAGE_TYPE_EMAIL, + $this->provider() + ); + + return \iterator_to_array($generator, false); + } + + /** + * @param array $queries + * @return array + */ + private function selectFields(array $queries): array + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_SELECT) { + return $query->getValues(); + } + } + + return []; + } + + /** + * @param array $queries + */ + private function limitOf(array $queries): int + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_LIMIT) { + return (int) ($query->getValues()[0] ?? 0); + } + } + + return 0; + } + + /** + * @param array $queries + */ + private function hasCursor(array $queries): bool + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) { + return true; + } + } + + return false; + } + + /** + * @param array $queries + */ + private function topicInternalIdOf(array $queries): ?string + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'topicInternalId') { + return (string) ($query->getValues()[0] ?? null); + } + } + + return null; + } +} + +/** + * Worker subclass that zeroes the retry backoff so the throttle/retry tests stay instant. The worker already + * skips a non-positive delay, so no coroutine sleep is ever reached. + */ +class TestableMessaging extends Messaging +{ + protected function retryDelay(): float + { + return 0.0; + } +} + +/** + * Email adapter double whose per-attempt outcome is scripted. Each {@see process()} call records the exact + * `to` recipients it received and consults the next scripted step, which either throws (simulating a + * connection-level provider error) or returns a per-recipient success/failure verdict. When the script runs + * out, every recipient succeeds — modelling a provider that has stopped throttling. + */ +class ScriptedEmailAdapter extends EmailAdapter +{ + public int $sendCalls = 0; + + /** + * @var array> Recipient email lists captured per {@see process()} call, in call order. + */ + public array $recipientsPerCall = []; + + /** + * @param array<\Throwable|(callable(string): ?string)> $script One step per attempt. A Throwable is thrown + * for that attempt; a callable returns an error string for a failing recipient or null for success. + */ + public function __construct(private readonly array $script = []) + { + } + + public function getName(): string + { + return 'ScriptedEmail'; + } + + public function getMaxMessagesPerRequest(): int + { + return 1000; + } + + /** + * @param EmailMessage $message + * @return array{deliveredTo: int, type: string, results: array>} + */ + protected function process(EmailMessage $message): array + { + $recipients = \array_map(static fn (array $entry): string => $entry['email'], $message->getTo()); + $this->recipientsPerCall[] = $recipients; + + $step = $this->script[$this->sendCalls] ?? null; + $this->sendCalls++; + + if ($step instanceof \Throwable) { + throw $step; + } + + $response = new Response($this->getType()); + $delivered = 0; + + foreach ($recipients as $recipient) { + $error = \is_callable($step) ? $step($recipient) : null; + + if ($error === null) { + $delivered++; + $response->addResult($recipient); + } else { + $response->addResult($recipient, $error); + } + } + + $response->setDeliveredTo($delivered); + + return $response->toArray(); + } +} + +/** + * Minimal in-memory {@see Database} double for the retry tests, which only build email messages with no + * attachments and never touch the expired-token cleanup path. Every read resolves empty. + */ +class InMemoryDatabase extends Database +{ + private readonly Authorization $authorization; + + public function __construct() + { + $this->authorization = new Authorization(); + } + + public function getAuthorization(): Authorization + { + return $this->authorization; + } + + public function skipValidation(callable $callback): mixed + { + return $callback(); + } + + public function findOne(string $collection, array $queries = []): Document + { + return new Document(); + } + + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document + { + return new Document(); + } + + public function updateDocument(string $collection, string $id, Document $document): Document + { + return $document; + } + + public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array + { + return []; + } +} + +/** + * {@see InMemoryDatabase} variant whose expired-device-token cleanup always fails: `findOne('targets')` + * resolves to a real target so the worker attempts the update, and `updateDocument('targets')` throws a + * transient error. Used to prove that a DB hiccup during best-effort cleanup never leaks into delivery + * accounting or the retry decision. + */ +class ExpiredTokenCleanupFailingDatabase extends InMemoryDatabase +{ + public int $targetUpdateAttempts = 0; + + public function findOne(string $collection, array $queries = []): Document + { + if ($collection === 'targets') { + return new Document(['$id' => 'target-expired', '$sequence' => '1']); + } + + return parent::findOne($collection, $queries); + } + + public function updateDocument(string $collection, string $id, Document $document): Document + { + if ($collection === 'targets') { + $this->targetUpdateAttempts++; + + throw new \RuntimeException('Transient database error during target cleanup'); + } + + return parent::updateDocument($collection, $id, $document); + } +} + +/** + * In-memory {@see Database} double for the streaming tests. Serves subscribers/targets through cursor + * pagination and records every query issued so the test can assert the streaming contract (cursorAfter + + * bounded page size, topic reads never selecting the 1M-row `targets` subquery, no runaway limits). + */ +class RecordingDatabase extends Database +{ + /** + * @var array}> + */ + public array $findCalls = []; + + private readonly Authorization $authorization; + + /** + * @param array $subscribers Subscriber rows (carry $sequence + targetInternalId + providerType). + * @param array $targets Target rows keyed by $sequence (resolved from a subscriber page). + * @param array $userTargets Targets addressed by userId. + * @param array $directTargets Targets addressed directly by id. + */ + public function __construct( + private readonly array $subscribers = [], + private readonly array $targets = [], + private readonly array $userTargets = [], + private readonly array $directTargets = [] + ) { + $this->authorization = new Authorization(); + } + + public function getAuthorization(): Authorization + { + return $this->authorization; + } + + public function skipValidation(callable $callback): mixed + { + return $callback(); + } + + public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array + { + $this->findCalls[] = ['collection' => $collection, 'queries' => $queries]; + + return match ($collection) { + 'topics' => [new Document(['$id' => 'topic1', '$sequence' => '100'])], + 'subscribers' => $this->paginate($this->subscribers, $queries), + 'targets' => $this->resolveTargets($queries), + default => [], + }; + } + + /** + * @param array $rows + * @param array $queries + * @return array + */ + private function paginate(array $rows, array $queries): array + { + $limit = $this->limitOf($queries); + $cursor = $this->cursorOf($queries); + + if ($cursor !== null) { + $rows = \array_values(\array_filter( + $rows, + fn (Document $row) => $row->getSequence() > $cursor->getSequence() + )); + } + + return \array_slice($rows, 0, $limit); + } + + /** + * Either resolve a subscriber page's targets by $sequence, or paginate user/direct targets. + * + * @param array $queries + * @return array + */ + private function resolveTargets(array $queries): array + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === '$sequence') { + $sequences = $query->getValues(); + + return \array_values(\array_map( + fn ($sequence) => $this->targets[$sequence], + \array_filter($sequences, fn ($sequence) => isset($this->targets[$sequence])) + )); + } + + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'userId') { + return $this->paginate($this->userTargets, $queries); + } + } + + return $this->paginate($this->directTargets, $queries); + } + + /** + * @param array $queries + */ + private function limitOf(array $queries): int + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_LIMIT) { + return (int) ($query->getValues()[0] ?? 0); + } + } + + return 0; + } + + /** + * @param array $queries + */ + private function cursorOf(array $queries): ?Document + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) { + $value = $query->getValues()[0] ?? null; + + return $value instanceof Document ? $value : null; + } + } + + return null; + } +} diff --git a/tests/unit/Network/FetcherTest.php b/tests/unit/Network/FetcherTest.php new file mode 100644 index 0000000000..48c49551bb --- /dev/null +++ b/tests/unit/Network/FetcherTest.php @@ -0,0 +1,250 @@ +expectNotToPerformAssertions(); + } + + #[DataProvider('unsafeUrls')] + public function testAssertSafeRejectsUnsafeUrls(string $url, string $reasonFragment): void + { + try { + TestableGet::assertSafe($url); + $this->fail("Expected Exception for {$url}"); + } catch (Exception $e) { + $this->assertSame(Exception::AVATAR_REMOTE_URL_FAILED, $e->getType()); + $this->assertStringContainsString( + $reasonFragment, + $e->getMessage(), + "Wrong rejection reason for {$url}" + ); + } + } + + public static function unsafeUrls(): array + { + return [ + 'loopback v4' => ['http://127.0.0.1/iam', 'private or reserved'], + 'loopback v6' => ['http://[::1]/iam', 'private or reserved'], + 'link-local imds' => ['http://169.254.169.254/latest/', 'private or reserved'], + 'private rfc1918 a' => ['http://10.0.0.5/secret', 'private or reserved'], + 'private rfc1918 b' => ['http://192.168.1.1/admin', 'private or reserved'], + 'private rfc1918 c' => ['http://172.16.0.1/x', 'private or reserved'], + 'cgnat' => ['http://100.64.0.1/x', 'private or reserved'], + 'ipv4-mapped loopback' => ['http://[::ffff:127.0.0.1]/x', 'private or reserved'], + '6to4 imds' => ['http://[2002:a9fe:a9fe::]/x', 'private or reserved'], + 'file scheme' => ['file:///etc/passwd', "Scheme 'file' is not allowed"], + 'gopher scheme' => ['gopher://attacker/x', "Scheme 'gopher' is not allowed"], + 'ftp scheme' => ['ftp://internal/file', "Scheme 'ftp' is not allowed"], + 'no scheme' => ['example.com/path', "Scheme '' is not allowed"], + 'malformed no host' => ['http:///path', 'Malformed URL'], + 'unknown psl' => ['http://not-a-real-tld.notreal/x', 'not a known public domain'], + ]; + } + + public function testFetchFollowsPublicRedirect(): void + { + $adapter = $this->scriptedAdapter([ + $this->redirect('https://1.1.1.1/final'), + $this->ok('FINAL_BODY'), + ]); + + $fetcher = new TestableGet(); + $response = $fetcher->fetchForTest('http://8.8.8.8/start', $adapter); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame('FINAL_BODY', $response->getBody()); + $this->assertSame(2, $adapter->callCount); + } + + public function testFetchBlocksRedirectToPrivateIp(): void + { + // This is the exact SSRF chain from the report: + // public attacker page → 302 → 169.254.169.254 (AWS IMDS) + $adapter = $this->scriptedAdapter([ + $this->redirect('http://169.254.169.254/latest/meta-data/iam/security-credentials/'), + // This second response must NEVER be served — the fetcher should + // reject the redirect target before calling send() again. + $this->ok('SHOULD_NEVER_LEAK'), + ]); + + $fetcher = new TestableGet(); + + try { + $fetcher->fetchForTest('http://8.8.8.8/attacker-page', $adapter); + $this->fail('Expected Exception'); + } catch (Exception $e) { + $this->assertSame(Exception::AVATAR_REMOTE_URL_FAILED, $e->getType()); + $this->assertStringContainsString('169.254.169.254', $e->getMessage()); + $this->assertSame( + 1, + $adapter->callCount, + 'Redirect target should not have been fetched — only the initial request should hit the adapter' + ); + } + } + + public function testFetchBlocksRedirectToLoopback(): void + { + $adapter = $this->scriptedAdapter([ + $this->redirect('http://127.0.0.1:9999/secrets'), + $this->ok('SHOULD_NEVER_LEAK'), + ]); + + $fetcher = new TestableGet(); + + try { + $fetcher->fetchForTest('http://8.8.8.8/attacker-page', $adapter); + $this->fail('Expected Exception'); + } catch (Exception $e) { + $this->assertSame(Exception::AVATAR_REMOTE_URL_FAILED, $e->getType()); + $this->assertStringContainsString('127.0.0.1', $e->getMessage()); + $this->assertSame(1, $adapter->callCount); + } + } + + public function testFetchBlocksRedirectToRelativePathOnPrivateBase(): void + { + // Edge case: a server on a public IP returns a relative redirect. + // We resolve it against the (still public) base and continue — safe. + $adapter = $this->scriptedAdapter([ + $this->redirect('/different-path'), + $this->ok('FINAL'), + ]); + + $fetcher = new TestableGet(); + $response = $fetcher->fetchForTest('http://8.8.8.8/start', $adapter); + + $this->assertSame('FINAL', $response->getBody()); + } + + public function testFetchRejectsRedirectChainOverLimit(): void + { + // Six redirects all to public targets; with maxRedirects=5 the sixth + // hop trips the limit. + $adapter = $this->scriptedAdapter([ + $this->redirect('https://1.1.1.1/2'), + $this->redirect('https://1.0.0.1/3'), + $this->redirect('https://8.8.8.8/4'), + $this->redirect('https://8.8.4.4/5'), + $this->redirect('https://9.9.9.9/6'), + $this->redirect('https://208.67.222.222/7'), + ]); + + $fetcher = new TestableGet(); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('Too many redirects'); + $fetcher->fetchForTest('http://8.8.8.8/0', $adapter); + } + + public function testFetchBlocksNonHttpRedirect(): void + { + $adapter = $this->scriptedAdapter([ + $this->redirect('file:///etc/passwd'), + $this->ok('SHOULD_NEVER_LEAK'), + ]); + + $fetcher = new TestableGet(); + + try { + $fetcher->fetchForTest('http://8.8.8.8/page', $adapter); + $this->fail('Expected Exception'); + } catch (Exception $e) { + $this->assertSame(Exception::AVATAR_REMOTE_URL_FAILED, $e->getType()); + $this->assertStringContainsString("Scheme 'file'", $e->getMessage()); + $this->assertSame(1, $adapter->callCount); + } + } + + private function redirect(string $location): Response + { + // Mix the header key case on purpose. Fetcher must normalise per + // RFC 7230 §3.2; if it ever drops to a naive lowercase lookup the + // entire redirect-following suite fails — that's the regression + // signal we want. + return new Response(302, '', ['Location' => $location]); + } + + private function ok(string $body): Response + { + return new Response(200, $body, []); + } + + /** + * @param Response[] $responses + */ + private function scriptedAdapter(array $responses): ScriptedAdapter + { + return new ScriptedAdapter($responses); + } +} + +/** + * Test-only adapter that returns a pre-scripted list of responses in order. + * Exposed as a named class so PHPStan can see the callCount property. + */ +class ScriptedAdapter implements Adapter +{ + public int $callCount = 0; + + /** @param Response[] $responses */ + public function __construct(private array $responses) + { + } + + public function send( + string $url, + string $method, + mixed $body, + array $headers, + RequestOptions $options, + ?callable $chunkCallback = null + ): Response { + $response = $this->responses[$this->callCount] ?? null; + $this->callCount++; + if ($response === null) { + throw new \RuntimeException("Adapter ran out of scripted responses (call {$this->callCount})"); + } + return $response; + } +} + +class TestableGet extends Get +{ + /** + * Skip parent constructor. Action::__construct chains into + * Appwrite\SDK\Method, which loads Swoole at file-load time — + * Swoole is a production-only extension, so building the real + * Action in a unit test fails. The protected method bridges + * below are all the surface the redirect-chain tests need. + */ + public function __construct() + { + } + + public static function assertSafe(string $url): void + { + parent::assertSafeUrl($url); + } + + public function fetchForTest(string $url, Adapter $adapter): Response + { + return $this->safeFetch($url, 'test', $adapter); + } +} diff --git a/tests/unit/Network/Validators/PublicHostnameTest.php b/tests/unit/Network/Validators/PublicHostnameTest.php new file mode 100644 index 0000000000..81c5708769 --- /dev/null +++ b/tests/unit/Network/Validators/PublicHostnameTest.php @@ -0,0 +1,162 @@ +assertFalse($validator->isValid('')); + $this->assertFalse($validator->isValid(null)); + $this->assertFalse($validator->isValid(123)); + $this->assertFalse($validator->isValid([])); + } + + #[DataProvider('privateIpv4Addresses')] + public function testRejectsPrivateIpv4Literals(string $ip): void + { + $validator = new PublicHostname(); + + $this->assertFalse( + $validator->isValid($ip), + "Expected {$ip} to be rejected as private/reserved" + ); + $this->assertFalse(PublicHostname::isPublicIp($ip)); + } + + public static function privateIpv4Addresses(): array + { + return [ + 'unspecified' => ['0.0.0.0'], + 'private 10/8' => ['10.0.0.1'], + 'private 172.16/12' => ['172.16.5.10'], + 'private 192.168/16' => ['192.168.1.1'], + 'cgnat' => ['100.64.0.1'], + 'loopback' => ['127.0.0.1'], + 'link-local imds' => ['169.254.169.254'], + 'gcp metadata' => ['169.254.169.254'], + 'multicast' => ['224.0.0.1'], + 'reserved 240/4' => ['240.0.0.1'], + 'broadcast' => ['255.255.255.255'], + 'test-net-1' => ['192.0.2.1'], + 'test-net-2' => ['198.51.100.1'], + 'test-net-3' => ['203.0.113.1'], + 'benchmark' => ['198.18.0.1'], + ]; + } + + #[DataProvider('privateIpv6Addresses')] + public function testRejectsPrivateIpv6Literals(string $ip): void + { + $validator = new PublicHostname(); + + $this->assertFalse( + $validator->isValid($ip), + "Expected {$ip} to be rejected as private/reserved" + ); + $this->assertFalse(PublicHostname::isPublicIp($ip)); + } + + public static function privateIpv6Addresses(): array + { + return [ + 'loopback' => ['::1'], + 'unspecified' => ['::'], + 'link-local' => ['fe80::1'], + 'unique-local' => ['fc00::1'], + 'unique-local fd' => ['fd12:3456:789a::1'], + 'multicast' => ['ff02::1'], + 'ipv4-mapped loopback' => ['::ffff:127.0.0.1'], + 'ipv4-mapped private' => ['::ffff:10.0.0.1'], + 'ipv4-mapped imds' => ['::ffff:169.254.169.254'], + '6to4 loopback' => ['2002:7f00:1::'], + '6to4 imds' => ['2002:a9fe:a9fe::'], + 'teredo' => ['2001:0:1::1'], + 'documentation' => ['2001:db8::1'], + ]; + } + + #[DataProvider('publicIpAddresses')] + public function testAcceptsPublicIpLiterals(string $ip): void + { + $validator = new PublicHostname(); + + $this->assertTrue( + $validator->isValid($ip), + "Expected {$ip} to be accepted as public" + ); + $this->assertTrue(PublicHostname::isPublicIp($ip)); + } + + public static function publicIpAddresses(): array + { + return [ + 'google dns' => ['8.8.8.8'], + 'cloudflare' => ['1.1.1.1'], + 'opendns' => ['208.67.222.222'], + 'public ipv6' => ['2606:4700:4700::1111'], + ]; + } + + public function testRejectsMalformedInput(): void + { + $validator = new PublicHostname(); + + $this->assertFalse($validator->isValid('not a hostname at all')); + $this->assertFalse($validator->isValid('http://example.com')); + $this->assertFalse($validator->isValid('999.999.999.999')); + } + + public function testRejectsNonResolvingHostname(): void + { + $validator = new PublicHostname(); + + // RFC 2606 reserves this for testing — guaranteed not to resolve. + $this->assertFalse($validator->isValid('a-hostname-that-does-not-exist.invalid')); + } + + public function testReasonIsPopulatedOnFailure(): void + { + $validator = new PublicHostname(); + $validator->isValid('127.0.0.1'); + + $this->assertStringContainsString('127.0.0.1', $validator->getDescription()); + $this->assertStringContainsString('private', $validator->getDescription()); + } + + public function testReasonResetsBetweenCalls(): void + { + $validator = new PublicHostname(); + + $validator->isValid(''); + $this->assertSame('Hostname is empty.', $validator->getDescription()); + + $validator->isValid('127.0.0.1'); + $this->assertStringContainsString('127.0.0.1', $validator->getDescription()); + } + + public function testIpInCidrEdgesViaPublicIp(): void + { + // Boundary checks: first and last address of common ranges. + $this->assertFalse(PublicHostname::isPublicIp('10.0.0.0')); + $this->assertFalse(PublicHostname::isPublicIp('10.255.255.255')); + $this->assertTrue(PublicHostname::isPublicIp('11.0.0.0')); + $this->assertTrue(PublicHostname::isPublicIp('9.255.255.255')); + + $this->assertFalse(PublicHostname::isPublicIp('169.254.0.0')); + $this->assertFalse(PublicHostname::isPublicIp('169.254.255.255')); + $this->assertTrue(PublicHostname::isPublicIp('169.253.255.255')); + $this->assertTrue(PublicHostname::isPublicIp('169.255.0.0')); + + $this->assertFalse(PublicHostname::isPublicIp('100.64.0.0')); + $this->assertFalse(PublicHostname::isPublicIp('100.127.255.255')); + $this->assertTrue(PublicHostname::isPublicIp('100.128.0.0')); + $this->assertTrue(PublicHostname::isPublicIp('100.63.255.255')); + } +} diff --git a/tests/unit/Task/Validator/CronTest.php b/tests/unit/Task/Validator/CronTest.php index 4f9d8ede24..3809c825b5 100644 --- a/tests/unit/Task/Validator/CronTest.php +++ b/tests/unit/Task/Validator/CronTest.php @@ -28,12 +28,14 @@ class CronTest extends TestCase $this->assertEquals($this->object->isValid('* * * * *'), true); // execute on every minutes // $this->assertEquals($this->object->isValid('0 17 * * sun'), true); // execute on every Sunday at 5 PM $this->assertEquals($this->object->isValid('*/10 * * * *'), true); // execute on every 10 minutes + $this->assertEquals($this->object->isValid('*/5 22-23,0-3 * * *'), true); // execute every 5 minutes from 10pm to 3am // $this->assertEquals($this->object->isValid('* * * jan,may,aug *'), true); // execute on selected months // $this->assertEquals($this->object->isValid('0 17 * * sun,fri'), true); // execute on selected days // $this->assertEquals($this->object->isValid('0 2 * * sun'), true); // execute on first sunday of every month $this->assertEquals($this->object->isValid('0 */4 * * *'), true); // execute on every four hours // $this->assertEquals($this->object->isValid('0 4,17 * * sun,mon'), true); // execute twice on every Sunday and Monday $this->assertEquals($this->object->isValid('bad expression'), false); + $this->assertEquals($this->object->isValid('*/5 22-3 * * *'), false); $this->assertEquals(null, false); $this->assertEquals('', false); } diff --git a/tests/unit/URL/URLTest.php b/tests/unit/URL/URLTest.php index 597d77f74c..6558acc66c 100644 --- a/tests/unit/URL/URLTest.php +++ b/tests/unit/URL/URLTest.php @@ -114,4 +114,156 @@ class URLTest extends TestCase $this->assertEquals('param1=value1¶m2=value2', $result); } + + /** + * RFC 3986 §5.4 reference resolution test cases. + */ + public function testResolveLocationAbsolute(): void + { + // Absolute URL wins over the base + $this->assertSame( + 'https://other.example/path', + URL::resolveLocation('https://base.example/a/b', 'https://other.example/path') + ); + + // Different scheme too + $this->assertSame( + 'http://other.example/', + URL::resolveLocation('https://base.example/a/b', 'http://other.example/') + ); + } + + public function testResolveLocationProtocolRelative(): void + { + // Inherits scheme from base + $this->assertSame( + 'https://other.example/path', + URL::resolveLocation('https://base.example/a/b', '//other.example/path') + ); + + // Including with a port + $this->assertSame( + 'https://other.example:8080/path', + URL::resolveLocation('https://base.example/', '//other.example:8080/path') + ); + } + + public function testResolveLocationAbsolutePath(): void + { + $this->assertSame( + 'https://base.example/new', + URL::resolveLocation('https://base.example/a/b', '/new') + ); + + // Preserves port + $this->assertSame( + 'https://base.example:9000/new', + URL::resolveLocation('https://base.example:9000/a/b', '/new') + ); + } + + public function testResolveLocationRelativePath(): void + { + $this->assertSame( + 'https://base.example/a/c', + URL::resolveLocation('https://base.example/a/b', 'c') + ); + + $this->assertSame( + 'https://base.example/a/b/c', + URL::resolveLocation('https://base.example/a/b/', 'c') + ); + } + + public function testResolveLocationQueryOnly(): void + { + // The bug Copilot called out: query-only must keep the base path + $this->assertSame( + 'https://base.example/a/b?x=1', + URL::resolveLocation('https://base.example/a/b', '?x=1') + ); + + // Replaces any base query + $this->assertSame( + 'https://base.example/a/b?x=1', + URL::resolveLocation('https://base.example/a/b?old=value', '?x=1') + ); + } + + public function testResolveLocationFragmentOnly(): void + { + // Fragment-only must keep base path AND base query + $this->assertSame( + 'https://base.example/a/b#section', + URL::resolveLocation('https://base.example/a/b', '#section') + ); + + $this->assertSame( + 'https://base.example/a/b?x=1#section', + URL::resolveLocation('https://base.example/a/b?x=1', '#section') + ); + } + + public function testResolveLocationDotSegments(): void + { + // RFC 3986 §5.2.4 normalisation. `/c/d/..` resolves to `/c/` + // (the trailing slash is significant — it's the directory of `d`). + $this->assertSame( + 'https://base.example/c/', + URL::resolveLocation('https://base.example/a/b', '/c/d/..') + ); + + $this->assertSame( + 'https://base.example/c/', + URL::resolveLocation('https://base.example/a/b', '/c/d/../') + ); + + // From inside the directory `/a/b/`, `../c` goes up one level → `/a/c` + $this->assertSame( + 'https://base.example/a/c', + URL::resolveLocation('https://base.example/a/b/', '../c') + ); + + // From file `/a/b`, `../c` goes up to root → `/c` + $this->assertSame( + 'https://base.example/c', + URL::resolveLocation('https://base.example/a/b', '../c') + ); + + $this->assertSame( + 'https://base.example/a/b/c', + URL::resolveLocation('https://base.example/a/b/', './c') + ); + } + + public function testResolveLocationEmpty(): void + { + // Empty reference returns base unchanged + $this->assertSame( + 'https://base.example/a/b', + URL::resolveLocation('https://base.example/a/b', '') + ); + + // Whitespace-only reference is treated as empty + $this->assertSame( + 'https://base.example/a/b', + URL::resolveLocation('https://base.example/a/b', ' ') + ); + } + + public function testResolveLocationPreservesAuthorityFromBase(): void + { + // user/pass survive relative resolution + $this->assertSame( + 'https://user:pass@base.example/c', + URL::resolveLocation('https://user:pass@base.example/a/b', '/c') + ); + + // port survives, and `c` relative to `/a/b` resolves to `/a/c` + // (RFC 3986 §5.2.3 merge replaces the last segment of the base path). + $this->assertSame( + 'http://base.example:8080/a/c', + URL::resolveLocation('http://base.example:8080/a/b', 'c') + ); + } }