From 19ad2088a094b60e6aee4e5eb622b11316afbc89 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 10:08:07 +0200 Subject: [PATCH 01/54] update region --- app/controllers/api/functions.php | 2 +- app/controllers/api/projects.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 4 ++-- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 583468f6c1..0e2d8a0103 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -295,7 +295,7 @@ App::post('/v1/functions') $schedule = Authorization::skip( fn () => $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), // Todo replace with projects region + 'region' => $project->getAttribute('region'), 'resourceType' => 'function', 'resourceId' => $function->getId(), 'resourceInternalId' => $function->getInternalId(), diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 48d20cd17f..5099393e68 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -81,7 +81,7 @@ App::post('/v1/projects') ->param('projectId', '', new ProjectId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, and hyphen. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Project name. Max length: 128 chars.') ->param('teamId', '', new UID(), 'Team unique ID.') - ->param('region', System::getEnv('_APP_REGION', 'default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) + ->param('region', System::getEnv('_APP_REGION'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) ->param('description', '', new Text(256), 'Project description. Max length: 256 chars.', true) ->param('logo', '', new Text(1024), 'Project logo.', true) ->param('url', '', new URL(), 'Project URL.', true) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index dad2db0d9a..b62b95b066 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -104,7 +104,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', [System::getEnv('_APP_REGION')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); @@ -154,7 +154,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', [System::getEnv('_APP_REGION')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index e11181d199..9cb8f21a49 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -183,7 +183,7 @@ class Deletes extends Action $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', [System::getEnv('_APP_REGION')]), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], From 2a5b3317bf05bed6d57e2278d6dd2904ede59feb Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 4 Mar 2025 11:36:45 +0200 Subject: [PATCH 02/54] update region --- .env | 1 + app/config/variables.php | 9 +++++++++ app/views/install/compose.phtml | 22 +++++++++++++++++++++- docker-compose.yml | 21 +++++++++++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 1893e023ba..9b4aef4ddd 100644 --- a/.env +++ b/.env @@ -1,6 +1,7 @@ _APP_ENV=development _APP_EDITION=self-hosted _APP_LOCALE=en +_APP_REGION=fra _APP_WORKER_PER_CORE=6 _APP_COMPRESSION_MIN_SIZE_BYTES=1024 _APP_CONSOLE_WHITELIST_ROOT=disabled diff --git a/app/config/variables.php b/app/config/variables.php index 98dd9ffec1..2f7a10221d 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -7,6 +7,15 @@ return [ 'category' => 'General', 'description' => '', 'variables' => [ + [ + 'name' => '_APP_REGION', + 'description' => 'Set your server running geo region. By default, the var is set to \'fra\'.', + 'introduction' => '', + 'default' => 'fra', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_ENV', 'description' => 'Set your server running environment. By default, the var is set to \'development\'. When deploying to production, change it to: \'production\'.', diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 62fcd03624..c85fbcafcf 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -164,7 +164,7 @@ $image = $this->getParam('image', ''); - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_ASSISTANT_OPENAI_API_KEY - + - _APP_REGION appwrite-console: <<: *x-logging container_name: appwrite-console @@ -229,6 +229,7 @@ $image = $this->getParam('image', ''); - _APP_DB_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-audits: image: /: @@ -255,6 +256,7 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-webhooks: image: /: @@ -283,6 +285,7 @@ $image = $this->getParam('image', ''); - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-deletes: image: /: @@ -342,6 +345,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_EXECUTION + - _APP_REGION appwrite-worker-databases: image: /: @@ -368,6 +372,7 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-builds: image: /: @@ -432,6 +437,7 @@ $image = $this->getParam('image', ''); - _APP_STORAGE_WASABI_SECRET - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET + - _APP_REGION appwrite-worker-certificates: image: /: @@ -465,6 +471,7 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-functions: image: /: @@ -503,6 +510,7 @@ $image = $this->getParam('image', ''); - _APP_DOCKER_HUB_USERNAME - _APP_DOCKER_HUB_PASSWORD - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-mails: image: /: @@ -537,6 +545,7 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_DOMAIN - _APP_OPTIONS_FORCE_HTTPS + - _APP_REGION appwrite-worker-messaging: image: /: @@ -588,6 +597,7 @@ $image = $this->getParam('image', ''); - _APP_STORAGE_WASABI_SECRET - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET + - _APP_REGION appwrite-worker-migrations: image: /: @@ -618,6 +628,7 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET + - _APP_REGION appwrite-task-maintenance: image: /: @@ -652,6 +663,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES + - _APP_REGION appwrite-task-stats-resources: image: /: @@ -683,6 +695,7 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL + - _APP_REGION appwrite-worker-stats-resources: image: /: @@ -711,6 +724,7 @@ $image = $this->getParam('image', ''); - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_STATS_RESOURCES_INTERVAL + - _APP_REGION appwrite-worker-stats-usage: image: /: @@ -739,6 +753,7 @@ $image = $this->getParam('image', ''); - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_REGION appwrite-worker-stats-usage-dump: image: /: @@ -767,6 +782,7 @@ $image = $this->getParam('image', ''); - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_REGION appwrite-task-scheduler-functions: image: /: @@ -792,6 +808,7 @@ $image = $this->getParam('image', ''); - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REGION appwrite-task-scheduler-executions: image: /: @@ -817,6 +834,7 @@ $image = $this->getParam('image', ''); - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REGION appwrite-task-scheduler-messages: image: /: @@ -842,6 +860,7 @@ $image = $this->getParam('image', ''); - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REGION appwrite-assistant: image: appwrite/assistant:0.4.0 @@ -877,6 +896,7 @@ $image = $this->getParam('image', ''); - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_ENV=$_APP_ENV + - OPR_EXECUTOR_REGION=$_APP_REGION - OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG diff --git a/docker-compose.yml b/docker-compose.yml index facf0e6db9..51689fd872 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -268,6 +268,7 @@ services: - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-audits: entrypoint: worker-audits @@ -297,6 +298,7 @@ services: - _APP_DB_PASS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-webhooks: entrypoint: worker-webhooks @@ -329,6 +331,7 @@ services: - _APP_LOGGING_CONFIG - _APP_WEBHOOK_MAX_FAILED_ATTEMPTS - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-deletes: entrypoint: worker-deletes @@ -389,6 +392,7 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES + - _APP_REGION appwrite-worker-databases: entrypoint: worker-databases @@ -420,6 +424,7 @@ services: - _APP_WORKERS_NUM - _APP_QUEUE_NAME - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-builds: entrypoint: worker-builds @@ -486,6 +491,7 @@ services: - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-certificates: entrypoint: worker-certificates @@ -521,6 +527,7 @@ services: - _APP_DB_PASS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-functions: entrypoint: worker-functions @@ -563,6 +570,7 @@ services: - _APP_LOGGING_CONFIG - _APP_LOGGING_PROVIDER - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-mails: entrypoint: worker-mails @@ -597,6 +605,7 @@ services: - _APP_DOMAIN - _APP_OPTIONS_FORCE_HTTPS - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-messaging: entrypoint: worker-messaging @@ -652,6 +661,7 @@ services: - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-migrations: entrypoint: worker-migrations @@ -687,6 +697,7 @@ services: - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-task-maintenance: entrypoint: maintenance @@ -725,6 +736,7 @@ services: - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_MAINTENANCE_DELAY - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-task-stats-resources: container_name: appwrite-task-stats-resources @@ -756,6 +768,7 @@ services: - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL + - _APP_REGION appwrite-worker-stats-resources: entrypoint: worker-stats-resources @@ -787,6 +800,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-stats-usage: entrypoint: worker-stats-usage @@ -818,6 +832,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-stats-usage-dump: entrypoint: worker-stats-usage-dump @@ -850,6 +865,7 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - _APP_STATS_USAGE_DUAL_WRITING_DBS + - _APP_REGION appwrite-task-scheduler-functions: entrypoint: schedule-functions @@ -878,6 +894,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-task-scheduler-executions: entrypoint: schedule-executions @@ -905,6 +922,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REGION appwrite-task-scheduler-messages: entrypoint: schedule-messages @@ -933,6 +951,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-assistant: container_name: appwrite-assistant @@ -966,6 +985,7 @@ services: - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_ENV=$_APP_ENV + - OPR_EXECUTOR_REGION=$_APP_REGION - OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - OPR_EXECUTOR_RUNTIME_VERSIONS=v2,v4 @@ -1005,6 +1025,7 @@ services: environment: - OPR_PROXY_WORKER_PER_CORE=$_APP_WORKER_PER_CORE - OPR_PROXY_ENV=$_APP_ENV + - OPR_PROXY_REGION=$_APP_REGION - OPR_PROXY_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - OPR_PROXY_SECRET=$_APP_EXECUTOR_SECRET - OPR_PROXY_LOGGING_CONFIG=$_APP_LOGGING_CONFIG From 4eece99768a78ea1f8c64cd76711aae6cab63397 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 16 Mar 2025 02:35:10 +0000 Subject: [PATCH 03/54] Feat: domains count --- app/init/constants.php | 1 + src/Appwrite/Platform/Workers/StatsResources.php | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/app/init/constants.php b/app/init/constants.php index 3383b28f57..5e4edfd97d 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -221,6 +221,7 @@ const METRIC_TOPICS = 'topics'; const METRIC_TARGETS = 'targets'; const METRIC_PROVIDER_TYPE_TARGETS = '{providerType}.targets'; const METRIC_KEYS = 'keys'; +const METRIC_DOMAINS = 'domains'; const METRIC_RESOURCE_TYPE_ID_BUILDS = '{resourceType}.{resourceInternalId}.builds'; const METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE = '{resourceType}.{resourceInternalId}.builds.storage'; const METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments'; diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index 1140698342..3c0e772bd4 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -114,6 +114,13 @@ class StatsResources extends Action $keys = $dbForPlatform->count('keys', [ Query::equal('projectInternalId', [$project->getInternalId()]) ]); + + $domains = $dbForPlatform->count('rules', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::equal('owner', ['']), + ]); + + $databases = $dbForProject->count('databases'); $buckets = $dbForProject->count('buckets'); $users = $dbForProject->count('users'); @@ -162,6 +169,7 @@ class StatsResources extends Action METRIC_PROVIDERS => $providers, METRIC_TOPICS => $topics, METRIC_KEYS => $keys, + METRIC_DOMAINS => $domains, METRIC_TARGETS => $targets, str_replace('{providerType}', MESSAGE_TYPE_EMAIL, METRIC_PROVIDER_TYPE_TARGETS) => $emailTargets, str_replace('{providerType}', MESSAGE_TYPE_PUSH, METRIC_PROVIDER_TYPE_TARGETS) => $pushTargets, From 04711d8c02a341ca9e18e3b820f31a27a19f6c61 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 19:21:31 +1300 Subject: [PATCH 04/54] Add doc --- src/Appwrite/Platform/Workers/Migrations.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index f21a846a0d..4939dc8143 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -34,6 +34,9 @@ class Migrations extends Action protected Document $project; + /** + * @var callable + */ protected $logError; public static function getName(): string From 9cb860d4467f8e087bc1e16e1f6f515c10ad7cee Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 17 Mar 2025 13:52:17 +0200 Subject: [PATCH 05/54] _app_region --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 1cd9df243d..ea796682ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,6 +93,7 @@ services: - app/http.php environment: - _APP_ENV + - _APP_REGION - _APP_EDITION - _APP_WORKER_PER_CORE - _APP_LOCALE From bbfd22ce2e68c86ee862ff450ee294d442554dec Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 17 Mar 2025 17:05:40 +0200 Subject: [PATCH 06/54] Index check for delete worker --- src/Appwrite/Platform/Workers/Deletes.php | 147 +++++++++++++++++----- 1 file changed, 113 insertions(+), 34 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9b0590181a..3c8bcdd635 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -248,7 +248,8 @@ class Deletes extends Action $this->deleteByGroup( 'subscribers', [ - Query::equal('topicInternalId', [$topic->getInternalId()]) + Query::equal('topicInternalId', [$topic->getInternalId()]), + Query::orderAsc(), ], $getProjectDB($project) ); @@ -269,7 +270,8 @@ class Deletes extends Action $this->deleteByGroup( 'subscribers', [ - Query::equal('targetInternalId', [$target->getInternalId()]) + Query::equal('targetInternalId', [$target->getInternalId()]), + Query::orderAsc(), ], $dbForProject, function (Document $subscriber) use ($dbForProject, $target) { @@ -303,10 +305,14 @@ class Deletes extends Action */ private function deleteExpiredTargets(Document $project, callable $getProjectDB): void { + /** + * todo: No index found for `expired` attribute + */ $this->deleteByGroup( 'targets', [ - Query::equal('expired', [true]) + Query::equal('expired', [true]), + Query::orderAsc(), ], $getProjectDB($project), function (Document $target) use ($getProjectDB, $project) { @@ -317,10 +323,14 @@ class Deletes extends Action private function deleteSessionTargets(Document $project, callable $getProjectDB, Document $session): void { + /** + * todo: No index found for `sessionInternalId` attribute + */ $this->deleteByGroup( 'targets', [ - Query::equal('sessionInternalId', [$session->getInternalId()]) + Query::equal('sessionInternalId', [$session->getInternalId()]), + Query::orderAsc(), ], $getProjectDB($project), function (Document $target) use ($getProjectDB, $project) { @@ -348,10 +358,17 @@ class Deletes extends Action ); $query[] = Query::equal('resource', [$resource]); + if (!empty($resourceType)) { $query[] = Query::equal('resourceType', [$resourceType]); } + $query[] = Query::orderAsc(); + + /** + * todo: missing index on `resource`, `resourceType` + */ + $this->deleteByGroup( 'cache', $query, @@ -388,7 +405,7 @@ class Deletes extends Action $query = [ Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), - Query::orderDesc('$internalId'), + Query::orderDesc(), ]; $this->deleteByGroup( @@ -421,10 +438,10 @@ class Deletes extends Action // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ + Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc('$internalId'), - Query::equal('period', ['1h']), + Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC ], $dbForProject); if ($project->getId() !== 'console') { @@ -433,10 +450,10 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ + Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc('$internalId'), - Query::equal('period', ['1h']), + Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC ], $dbForLogs); } } @@ -457,7 +474,8 @@ class Deletes extends Action $this->deleteByGroup( 'memberships', [ - Query::equal('teamInternalId', [$teamInternalId]) + Query::equal('teamInternalId', [$teamInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $membership) use ($dbForProject) { @@ -547,7 +565,13 @@ class Deletes extends Action if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) { $dbForProject->deleteCollection($collection->getId()); } else { - $this->deleteByGroup($collection->getId(), [], database: $dbForProject); + $this->deleteByGroup( + $collection->getId(), + [ + Query::orderAsc() + ], + database: $dbForProject + ); } } catch (Throwable $e) { Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); @@ -567,58 +591,82 @@ class Deletes extends Action // Delete Platforms $this->deleteByGroup('platforms', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete project and function rules $this->deleteByGroup('rules', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($dbForPlatform, $certificates) { $this->deleteRule($dbForPlatform, $document, $certificates); }); // Delete Keys $this->deleteByGroup('keys', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete Webhooks $this->deleteByGroup('webhooks', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete VCS Installations $this->deleteByGroup('installations', [ - Query::equal('projectInternalId', [$projectInternalId]) + Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete VCS Repositories $this->deleteByGroup('repositories', [ Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); // Delete VCS comments $this->deleteByGroup('vcsComments', [ Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); - // Delete Schedules (No projectInternalId in this collection) + /** + * No projectInternalId in this collection + * todo: No index for `projectId` attribute + */ + // Delete Schedules () $this->deleteByGroup('schedules', [ Query::equal('projectId', [$projectId]), + Query::orderAsc() ], $dbForPlatform); // Delete metadata table if ($projectTables) { $dbForProject->deleteCollection(Database::METADATA); } elseif ($sharedTablesV1) { - $this->deleteByGroup(Database::METADATA, [], $dbForProject); + $this->deleteByGroup( + Database::METADATA, + [ + Query::orderAsc() + ], + $dbForProject + ); } elseif ($sharedTablesV2) { $queries = \array_map( fn ($id) => Query::notEqual('$id', $id), $projectCollectionIds ); - $this->deleteByGroup(Database::METADATA, $queries, $dbForProject); + $queries[] = Query::orderAsc(); + + $this->deleteByGroup( + Database::METADATA, + $queries, + $dbForProject + ); } // Delete all storage directories @@ -643,14 +691,16 @@ class Deletes extends Action // Delete all sessions of this user from the sessions table and update the sessions field of the user record $this->deleteByGroup('sessions', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject); $dbForProject->purgeCachedDocument('users', $userId); // Delete Memberships and decrement team membership counts $this->deleteByGroup('memberships', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($dbForProject) { if ($document->getAttribute('confirm')) { // Count only confirmed members $teamId = $document->getAttribute('teamId'); @@ -663,19 +713,25 @@ class Deletes extends Action // Delete tokens $this->deleteByGroup('tokens', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject); // Delete identities + /** + * todo: Remove Duplication index `_key_userInternalId` + */ $this->deleteByGroup('identities', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject); // Delete targets $this->deleteByGroup( 'targets', [ - Query::equal('userInternalId', [$userInternalId]) + Query::equal('userInternalId', [$userInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $target) use ($getProjectDB, $project) { @@ -699,7 +755,7 @@ class Deletes extends Action $this->deleteByGroup('executions', [ Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), - Query::orderDesc('$internalId'), + Query::orderDesc(), ], $dbForProject); } @@ -719,7 +775,7 @@ class Deletes extends Action $this->deleteByGroup('sessions', [ Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), - Query::orderDesc('$internalId'), + Query::orderDesc(), ], $dbForProject); } @@ -735,7 +791,7 @@ class Deletes extends Action $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime), Query::orderDesc('timestamp'), - Query::orderDesc('$internalId'), + Query::orderAsc(), // KEY "_key_timestamp" ("timestamp" DESC), ], $dbForPlatform); } @@ -755,7 +811,7 @@ class Deletes extends Action $this->deleteByGroup(Audit::COLLECTION, [ Query::lessThan('time', $auditRetention), Query::orderDesc('time'), - Query::orderDesc('$internalId'), + Query::orderAsc(), // KEY "index-time" ("time" DESC) ], $dbForProject); } catch (DatabaseException $e) { Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); @@ -780,33 +836,39 @@ class Deletes extends Action /** * Delete rules + * todo: No index for this query */ Console::info("Deleting rules for function " . $functionId); $this->deleteByGroup('rules', [ Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), - Query::equal('projectInternalId', [$project->getInternalId()]) + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($project, $dbForPlatform, $certificates) { $this->deleteRule($dbForPlatform, $document, $certificates); }); /** * Delete Variables + * todo: No index for this query */ Console::info("Deleting variables for function " . $functionId); $this->deleteByGroup('variables', [ Query::equal('resourceType', ['function']), - Query::equal('resourceInternalId', [$functionInternalId]) + Query::equal('resourceInternalId', [$functionInternalId]), + Query::orderAsc() ], $dbForProject); /** * Delete Deployments + * todo: No index for `resourceInternalId`, perhaps use resourceId until fixed */ Console::info("Deleting deployments for function " . $functionId); $deploymentInternalIds = []; $this->deleteByGroup('deployments', [ - Query::equal('resourceInternalId', [$functionInternalId]) + Query::equal('resourceInternalId', [$functionInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($deviceForFunctions, &$deploymentInternalIds) { $deploymentInternalIds[] = $document->getInternalId(); $this->deleteDeploymentFiles($deviceForFunctions, $document); @@ -814,12 +876,14 @@ class Deletes extends Action /** * Delete builds + * todo: No index for `deploymentInternalId`, perhaps use deploymentId until fixed */ Console::info("Deleting builds for function " . $functionId); foreach ($deploymentInternalIds as $deploymentInternalId) { $this->deleteByGroup('builds', [ - Query::equal('deploymentInternalId', [$deploymentInternalId]) + Query::equal('deploymentInternalId', [$deploymentInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($deviceForBuilds) { $this->deleteBuildFiles($deviceForBuilds, $document); }); @@ -827,26 +891,35 @@ class Deletes extends Action /** * Delete Executions + * todo: No index for `functionInternalId` , perhaps use functionId until fixed */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ - Query::equal('functionInternalId', [$functionInternalId]) + Query::equal('functionInternalId', [$functionInternalId]), + Query::orderAsc() ], $dbForProject); /** * Delete VCS Repositories and VCS Comments + * todo: no index for this query */ Console::info("Deleting VCS repositories and comments linked to function " . $functionId); $this->deleteByGroup('repositories', [ Query::equal('projectInternalId', [$project->getInternalId()]), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('resourceType', ['function']), + Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($dbForPlatform) { $providerRepositoryId = $document->getAttribute('providerRepositoryId', ''); $projectInternalId = $document->getAttribute('projectInternalId', ''); + + /** + * todo: add index to this query + */ $this->deleteByGroup('vcsComments', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::equal('projectInternalId', [$projectInternalId]), + Query::orderAsc() ], $dbForPlatform); }); @@ -942,11 +1015,13 @@ class Deletes extends Action /** * Delete builds + * todo: no index for `deploymentInternalId` change to temporary to deploymentId? */ Console::info("Deleting builds for deployment " . $deploymentId); $this->deleteByGroup('builds', [ - Query::equal('deploymentInternalId', [$deploymentInternalId]) + Query::equal('deploymentInternalId', [$deploymentInternalId]), + Query::orderAsc() ], $dbForProject, function (Document $document) use ($deviceForBuilds) { $this->deleteBuildFiles($deviceForBuilds, $document); }); @@ -974,6 +1049,10 @@ class Deletes extends Action ): void { $start = \microtime(true); + /** + * deleteDocuments uses a cursor, we need to add a unique order by field or use default + */ + try { $documents = $database->deleteDocuments($collection, $queries); } catch (Throwable $th) { From 8184bc8c78df84dc04a8c0514edda828a26f061d Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 17 Mar 2025 17:31:24 +0200 Subject: [PATCH 07/54] _app_region --- .env | 2 +- app/config/variables.php | 4 ++-- app/controllers/api/projects.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 4 ++-- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 9b4aef4ddd..ae4522c898 100644 --- a/.env +++ b/.env @@ -1,7 +1,7 @@ _APP_ENV=development _APP_EDITION=self-hosted _APP_LOCALE=en -_APP_REGION=fra +_APP_REGION=default _APP_WORKER_PER_CORE=6 _APP_COMPRESSION_MIN_SIZE_BYTES=1024 _APP_CONSOLE_WHITELIST_ROOT=disabled diff --git a/app/config/variables.php b/app/config/variables.php index 2f7a10221d..592d398062 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -9,9 +9,9 @@ return [ 'variables' => [ [ 'name' => '_APP_REGION', - 'description' => 'Set your server running geo region. By default, the var is set to \'fra\'.', + 'description' => 'Set your server running geo region. By default, the var is set to \'default\'.', 'introduction' => '', - 'default' => 'fra', + 'default' => 'default', 'required' => false, 'question' => '', 'filter' => '' diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 5099393e68..2f46c3c338 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -81,7 +81,7 @@ App::post('/v1/projects') ->param('projectId', '', new ProjectId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, and hyphen. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Project name. Max length: 128 chars.') ->param('teamId', '', new UID(), 'Team unique ID.') - ->param('region', System::getEnv('_APP_REGION'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) + ->param('region', System::getEnv('_APP_REGION','default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) ->param('description', '', new Text(256), 'Project description. Max length: 256 chars.', true) ->param('logo', '', new Text(1024), 'Project logo.', true) ->param('url', '', new URL(), 'Project URL.', true) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index b62b95b066..b96fd5622f 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -104,7 +104,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION')]), + Query::equal('region', [System::getEnv('_APP_REGION','default')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); @@ -154,7 +154,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION')]), + Query::equal('region', [System::getEnv('_APP_REGION','default')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index e8195d175d..624948e2f5 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -183,7 +183,7 @@ class Deletes extends Action $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION')]), + Query::equal('region', [System::getEnv('_APP_REGION','default')]), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], From 59f613f9bbda26a3019feb283b6de0f27b88af93 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 17 Mar 2025 17:33:48 +0200 Subject: [PATCH 08/54] _app_region --- app/controllers/api/projects.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 4 ++-- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 2f46c3c338..48d20cd17f 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -81,7 +81,7 @@ App::post('/v1/projects') ->param('projectId', '', new ProjectId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, and hyphen. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Project name. Max length: 128 chars.') ->param('teamId', '', new UID(), 'Team unique ID.') - ->param('region', System::getEnv('_APP_REGION','default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) + ->param('region', System::getEnv('_APP_REGION', 'default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) ->param('description', '', new Text(256), 'Project description. Max length: 256 chars.', true) ->param('logo', '', new Text(1024), 'Project logo.', true) ->param('url', '', new URL(), 'Project URL.', true) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index b96fd5622f..dad2db0d9a 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -104,7 +104,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION','default')]), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); @@ -154,7 +154,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION','default')]), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 624948e2f5..9b0590181a 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -183,7 +183,7 @@ class Deletes extends Action $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION','default')]), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], From 7149deeacb498429a756987d1a5990ca82a164d6 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 17 Mar 2025 18:11:49 +0200 Subject: [PATCH 09/54] _app_region --- .env | 2 +- app/config/regions.php | 7 ------ app/config/variables.php | 2 +- app/controllers/api/functions.php | 2 +- app/controllers/api/messaging.php | 12 +++++----- app/controllers/api/projects.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 4 ++-- src/Appwrite/Platform/Workers/Deletes.php | 2 +- .../Platform/Workers/StatsUsageDump.php | 4 ++-- tests/e2e/Scopes/ProjectCustom.php | 2 +- .../Projects/ProjectsConsoleClientTest.php | 22 +++++++++---------- 11 files changed, 27 insertions(+), 34 deletions(-) diff --git a/.env b/.env index ae4522c898..9b4aef4ddd 100644 --- a/.env +++ b/.env @@ -1,7 +1,7 @@ _APP_ENV=development _APP_EDITION=self-hosted _APP_LOCALE=en -_APP_REGION=default +_APP_REGION=fra _APP_WORKER_PER_CORE=6 _APP_COMPRESSION_MIN_SIZE_BYTES=1024 _APP_CONSOLE_WHITELIST_ROOT=disabled diff --git a/app/config/regions.php b/app/config/regions.php index 61029a9996..0b32d2ba4b 100644 --- a/app/config/regions.php +++ b/app/config/regions.php @@ -1,13 +1,6 @@ [ - '$id' => 'default', - 'name' => 'Frankfurt', - 'disabled' => false, - 'flag' => 'de', - 'default' => true, - ], 'fra' => [ '$id' => 'fra', 'name' => 'Frankfurt', diff --git a/app/config/variables.php b/app/config/variables.php index 592d398062..7c774e8d6a 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -9,7 +9,7 @@ return [ 'variables' => [ [ 'name' => '_APP_REGION', - 'description' => 'Set your server running geo region. By default, the var is set to \'default\'.', + 'description' => 'Set your server running geo region. By default, the var is set to \'fra\'.', 'introduction' => '', 'default' => 'default', 'required' => false, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 1483c3bf03..37932e4165 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -2079,7 +2079,7 @@ App::post('/v1/functions/:functionId/executions') ]; $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => ScheduleExecutions::getSupportedResource(), 'resourceId' => $execution->getId(), 'resourceInternalId' => $execution->getInternalId(), diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 9393f1fbfe..4812fab0f3 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -2936,7 +2936,7 @@ App::post('/v1/messaging/messages/email') break; case MessageStatus::SCHEDULED: $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -3058,7 +3058,7 @@ App::post('/v1/messaging/messages/sms') break; case MessageStatus::SCHEDULED: $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => System::getEnv('_APP_REGION'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -3275,7 +3275,7 @@ App::post('/v1/messaging/messages/push') break; case MessageStatus::SCHEDULED: $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -3661,7 +3661,7 @@ App::patch('/v1/messaging/messages/email/:messageId') if (\is_null($currentScheduledAt) && !\is_null($scheduledAt)) { $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -3862,7 +3862,7 @@ App::patch('/v1/messaging/messages/sms/:messageId') if (\is_null($currentScheduledAt) && !\is_null($scheduledAt)) { $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -4035,7 +4035,7 @@ App::patch('/v1/messaging/messages/push/:messageId') if (\is_null($currentScheduledAt) && !\is_null($scheduledAt)) { $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 48d20cd17f..5099393e68 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -81,7 +81,7 @@ App::post('/v1/projects') ->param('projectId', '', new ProjectId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, and hyphen. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Project name. Max length: 128 chars.') ->param('teamId', '', new UID(), 'Team unique ID.') - ->param('region', System::getEnv('_APP_REGION', 'default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) + ->param('region', System::getEnv('_APP_REGION'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) ->param('description', '', new Text(256), 'Project description. Max length: 256 chars.', true) ->param('logo', '', new Text(1024), 'Project logo.', true) ->param('url', '', new URL(), 'Project URL.', true) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index dad2db0d9a..b62b95b066 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -104,7 +104,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', [System::getEnv('_APP_REGION')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); @@ -154,7 +154,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', [System::getEnv('_APP_REGION')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9b0590181a..e8195d175d 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -183,7 +183,7 @@ class Deletes extends Action $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', [System::getEnv('_APP_REGION')]), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 119a9e7288..3100a39dea 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -157,7 +157,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => System::getEnv('_APP_REGION'), ]); $documentClone = new Document($document->getArrayCopy()); @@ -191,7 +191,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => System::getEnv('_APP_REGION'), ]); $documentClone = new Document($document->getArrayCopy()); $dbForProject->createOrUpdateDocumentsWithIncrease( diff --git a/tests/e2e/Scopes/ProjectCustom.php b/tests/e2e/Scopes/ProjectCustom.php index 7f84ace6f2..2799e29772 100644 --- a/tests/e2e/Scopes/ProjectCustom.php +++ b/tests/e2e/Scopes/ProjectCustom.php @@ -42,7 +42,7 @@ trait ProjectCustom 'x-appwrite-project' => 'console', ], [ 'projectId' => ID::unique(), - 'region' => 'default', + 'region' => 'fra', 'name' => 'Demo Project', 'teamId' => $team['body']['$id'], 'description' => 'Demo Project Description', diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index a6498b8a4e..16a18d27c1 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -49,7 +49,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default', + 'region' => 'fra', ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -89,7 +89,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => '', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(400, $response['headers']['status-code']); @@ -100,7 +100,7 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders()), [ 'projectId' => ID::unique(), 'name' => 'Project Test', - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(401, $response['headers']['status-code']); @@ -129,7 +129,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => $projectId, 'name' => 'Project Duplicate', 'teamId' => $teamId, - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(409, $response['headers']['status-code']); @@ -178,7 +178,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Team 1 Project', 'teamId' => $team1, - 'region' => 'default', + 'region' => 'fra', ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -277,7 +277,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test 2', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -2042,7 +2042,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -2135,7 +2135,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3749,7 +3749,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project', 'teamId' => $teamId, - 'region' => 'default' + 'region' => 'fra' ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3820,7 +3820,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 1', 'teamId' => $teamId, - 'region' => 'default' + 'region' => 'fra' ]); $project2 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ @@ -3830,7 +3830,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 2', 'teamId' => $teamId, - 'region' => 'default' + 'region' => 'fra' ]); $project1Id = $project1['body']['$id']; From 60912cb0845bb5787126171608a54bcb11619049 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 17 Mar 2025 18:14:58 +0200 Subject: [PATCH 10/54] _app_region --- app/controllers/api/messaging.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 4812fab0f3..178266db60 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -3058,7 +3058,7 @@ App::post('/v1/messaging/messages/sms') break; case MessageStatus::SCHEDULED: $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), From e5261e7d70d62ff0ef398facb58f65f1a64247d1 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 17 Mar 2025 18:31:14 +0200 Subject: [PATCH 11/54] _app_region --- src/Appwrite/Migration/Version/V20.php | 2 +- tests/e2e/Services/Functions/FunctionsCustomClientTest.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Version/V20.php b/src/Appwrite/Migration/Version/V20.php index 5a0807cedf..cca0d9244b 100644 --- a/src/Appwrite/Migration/Version/V20.php +++ b/src/Appwrite/Migration/Version/V20.php @@ -426,7 +426,7 @@ class V20 extends Migration 'period' => 'inf', 'value' => $value, 'time' => null, - 'region' => 'default', + 'region' => 'fra', ])); } catch (Duplicate $th) { Console::warning("Error while creating inf metric: duplicate id {$metric} {$id}"); diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 914a255663..5e24dc4d89 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -111,7 +111,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('default', $output['APPWRITE_REGION']); + $this->assertEquals('fra', $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); @@ -221,7 +221,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('default', $output['APPWRITE_REGION']); + $this->assertEquals('fra', $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); From 5f7ceecb30e64f11fa6bb0a5cb746007cc983b8d Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Mar 2025 03:46:14 +0000 Subject: [PATCH 12/54] chore: stop tests on failure --- phpunit.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpunit.xml b/phpunit.xml index 4c4e55ea4e..598b730908 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -6,7 +6,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false" + stopOnFailure="true" > From a7165e6a32a0c731e0a10939935b863fcb824573 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 16:51:00 +1300 Subject: [PATCH 13/54] Disable PDO persistence since we manage our own pool --- app/init/registers.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init/registers.php b/app/init/registers.php index 9c90eb54d2..1ebcbc1691 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -217,7 +217,7 @@ $register->set('pools', function () { return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, + PDO::ATTR_PERSISTENT => false, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_EMULATE_PREPARES => true, PDO::ATTR_STRINGIFY_FETCHES => true From e43a3109eb98b46400c19f119d81220f6ea8b81e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 20:05:12 +1300 Subject: [PATCH 14/54] Update migration lib --- composer.json | 2 +- composer.lock | 40 ++++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/composer.json b/composer.json index 53c2a6d482..b604d6d1b8 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.16.*", - "utopia-php/migration": "0.6.*", + "utopia-php/migration": "0.8.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.5.*", diff --git a/composer.lock b/composer.lock index ed19d10202..2e5e6738e6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "44c6436ced36b0b026139edba252052e", + "content-hash": "ef36941f461409608974a804999f6787", "packages": [ { "name": "adhocore/jwt", @@ -4159,16 +4159,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.22", + "version": "0.8.0", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "a0269746bd318ff0993f5aa008675b971689d5b5" + "reference": "d8245090a7c55ea3c1fc411815b06bad0d5bcdd8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/a0269746bd318ff0993f5aa008675b971689d5b5", - "reference": "a0269746bd318ff0993f5aa008675b971689d5b5", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/d8245090a7c55ea3c1fc411815b06bad0d5bcdd8", + "reference": "d8245090a7c55ea3c1fc411815b06bad0d5bcdd8", "shasum": "" }, "require": { @@ -4209,9 +4209,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.22" + "source": "https://github.com/utopia-php/migration/tree/0.8.0" }, - "time": "2025-03-13T07:35:55+00:00" + "time": "2025-03-18T06:52:33+00:00" }, { "name": "utopia-php/mongo", @@ -4760,16 +4760,16 @@ }, { "name": "utopia-php/telemetry", - "version": "0.1.0", + "version": "0.1.1", "source": { "type": "git", "url": "https://github.com/utopia-php/telemetry.git", - "reference": "d35f2f0632f4ee0be63fb7ace6a94a6adda71a80" + "reference": "437f0021777f0e575dfb9e8a1a081b3aed75e33f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/d35f2f0632f4ee0be63fb7ace6a94a6adda71a80", - "reference": "d35f2f0632f4ee0be63fb7ace6a94a6adda71a80", + "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/437f0021777f0e575dfb9e8a1a081b3aed75e33f", + "reference": "437f0021777f0e575dfb9e8a1a081b3aed75e33f", "shasum": "" }, "require": { @@ -4790,7 +4790,7 @@ "type": "library", "autoload": { "psr-4": { - "Utopia\\": "src/" + "Utopia\\Telemetry\\": "src/Telemetry" } }, "notification-url": "https://packagist.org/downloads/", @@ -4804,9 +4804,9 @@ ], "support": { "issues": "https://github.com/utopia-php/telemetry/issues", - "source": "https://github.com/utopia-php/telemetry/tree/0.1.0" + "source": "https://github.com/utopia-php/telemetry/tree/0.1.1" }, - "time": "2024-11-13T10:29:53+00:00" + "time": "2025-03-17T11:57:52+00:00" }, { "name": "utopia-php/vcs", @@ -5043,16 +5043,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.7", + "version": "0.40.9", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "9e89b0bc4d8e6c81817d27096629f34a149fa873" + "reference": "dbb45a5db22cdc3368fe2573c07ba6088f188fa4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/9e89b0bc4d8e6c81817d27096629f34a149fa873", - "reference": "9e89b0bc4d8e6c81817d27096629f34a149fa873", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/dbb45a5db22cdc3368fe2573c07ba6088f188fa4", + "reference": "dbb45a5db22cdc3368fe2573c07ba6088f188fa4", "shasum": "" }, "require": { @@ -5088,9 +5088,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.7" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.9" }, - "time": "2025-03-12T08:43:55+00:00" + "time": "2025-03-17T18:39:14+00:00" }, { "name": "doctrine/annotations", From fe308b4f1f86ad97bdb6ef0e3f46becb6b4c5bf2 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 20:49:19 +1300 Subject: [PATCH 15/54] Update lib --- composer.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/composer.lock b/composer.lock index 2e5e6738e6..07e77a9037 100644 --- a/composer.lock +++ b/composer.lock @@ -4159,16 +4159,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.0", + "version": "0.8.1", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "d8245090a7c55ea3c1fc411815b06bad0d5bcdd8" + "reference": "36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/d8245090a7c55ea3c1fc411815b06bad0d5bcdd8", - "reference": "d8245090a7c55ea3c1fc411815b06bad0d5bcdd8", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab", + "reference": "36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab", "shasum": "" }, "require": { @@ -4209,9 +4209,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.0" + "source": "https://github.com/utopia-php/migration/tree/0.8.1" }, - "time": "2025-03-18T06:52:33+00:00" + "time": "2025-03-18T07:48:08+00:00" }, { "name": "utopia-php/mongo", From aa5ea7f3a10f632d87c1dbbf334104c2c38d4135 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 18 Mar 2025 10:01:06 +0100 Subject: [PATCH 16/54] feat: add pool telemetry --- composer.json | 2 +- composer.lock | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/composer.json b/composer.json index b604d6d1b8..d7b8505b5c 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/migration": "0.8.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "0.5.*", + "utopia-php/pools": "0.7.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.9.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index 07e77a9037..cdba756c72 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ef36941f461409608974a804999f6787", + "content-hash": "e0d7f21b681e4591144fec16c4f0d6aa", "packages": [ { "name": "adhocore/jwt", @@ -4375,25 +4375,26 @@ }, { "name": "utopia-php/pools", - "version": "0.5.0", + "version": "0.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "6f716a213a08db95eda1b5dddfa90983c1834817" + "reference": "ad64d45afda08ec8b29e2642a8d18075964d40bf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/6f716a213a08db95eda1b5dddfa90983c1834817", - "reference": "6f716a213a08db95eda1b5dddfa90983c1834817", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/ad64d45afda08ec8b29e2642a8d18075964d40bf", + "reference": "ad64d45afda08ec8b29e2642a8d18075964d40bf", "shasum": "" }, "require": { - "php": ">=8.0" + "php": ">=8.3", + "utopia-php/telemetry": "0.1.*" }, "require-dev": { - "laravel/pint": "1.2.*", - "phpstan/phpstan": "1.8.*", - "phpunit/phpunit": "^9.3" + "laravel/pint": "1.*", + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "11.*" }, "type": "library", "autoload": { @@ -4420,9 +4421,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.5.0" + "source": "https://github.com/utopia-php/pools/tree/0.7.0" }, - "time": "2024-04-19T11:11:54+00:00" + "time": "2025-03-18T03:55:33+00:00" }, { "name": "utopia-php/preloader", @@ -8402,7 +8403,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 70e27748be399a05cacf1bc222b49dac1198731e Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Mar 2025 12:52:41 +0000 Subject: [PATCH 17/54] chore: set min operations to 1 for reads and writes --- app/controllers/api/databases.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index d8d496089c..2a1ec4cba4 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3355,7 +3355,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, min($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection @@ -3520,7 +3520,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, min($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -3661,7 +3661,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, min($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -3959,7 +3959,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, min($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); $response->addHeader('X-Debug-Operations', $operations); From aab8bca91e98fcdc42fc854853e6d0725fee86ce Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 18 Mar 2025 14:20:25 +0000 Subject: [PATCH 18/54] fix: logic --- app/controllers/api/databases.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 2a1ec4cba4..0c37e1a765 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3355,7 +3355,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, min($operations, 1)) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection @@ -3520,7 +3520,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, min($operations, 1)) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -3661,7 +3661,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, min($operations, 1)) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -3959,7 +3959,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, min($operations, 1)) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); $response->addHeader('X-Debug-Operations', $operations); From 795928a037644ca8aca6a3ad6fd07b2f6ffffef5 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 18 Mar 2025 16:59:05 +0200 Subject: [PATCH 19/54] Fix $id --- src/Appwrite/Platform/Workers/Deletes.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 3c8bcdd635..a14e54caab 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -1045,7 +1045,8 @@ class Deletes extends Action string $collection, array $queries, Database $database, - ?callable $callback = null + ?callable $callback = null, + bool $shortSelect = false ): void { $start = \microtime(true); @@ -1053,6 +1054,12 @@ class Deletes extends Action * deleteDocuments uses a cursor, we need to add a unique order by field or use default */ + if (!\is_callable($callback) && $shortSelect) { + $queries = array_merge($queries, [ + Query::select(['$internalId', '$id', '$permissions', '$updatedAt']) + ]); + } + try { $documents = $database->deleteDocuments($collection, $queries); } catch (Throwable $th) { From e9841e53b9995091d64b665addcd1d2389740d25 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 18 Mar 2025 21:40:46 +0200 Subject: [PATCH 20/54] revert --- .env | 1 - app/config/regions.php | 63 +------------------ app/config/variables.php | 9 --- app/controllers/api/projects.php | 2 +- app/views/install/compose.phtml | 37 +++++------ docker-compose.yml | 40 ++++++------ src/Appwrite/Migration/Version/V20.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 4 +- src/Appwrite/Platform/Workers/Deletes.php | 2 +- .../Platform/Workers/StatsUsageDump.php | 4 +- tests/e2e/Scopes/ProjectCustom.php | 2 +- .../Functions/FunctionsCustomClientTest.php | 4 +- .../Projects/ProjectsConsoleClientTest.php | 22 +++---- 13 files changed, 60 insertions(+), 132 deletions(-) diff --git a/.env b/.env index 9b4aef4ddd..1893e023ba 100644 --- a/.env +++ b/.env @@ -1,7 +1,6 @@ _APP_ENV=development _APP_EDITION=self-hosted _APP_LOCALE=en -_APP_REGION=fra _APP_WORKER_PER_CORE=6 _APP_COMPRESSION_MIN_SIZE_BYTES=1024 _APP_CONSOLE_WHITELIST_ROOT=disabled diff --git a/app/config/regions.php b/app/config/regions.php index 0b32d2ba4b..05e04930fe 100644 --- a/app/config/regions.php +++ b/app/config/regions.php @@ -1,67 +1,10 @@ [ - '$id' => 'fra', - 'name' => 'Frankfurt', + 'default' => [ + '$id' => 'default', + 'name' => 'default', 'disabled' => false, - 'flag' => 'de', - 'default' => true, - ], - 'nyc' => [ - '$id' => 'nyc', - 'name' => 'New York', - 'disabled' => true, - 'flag' => 'us', - 'default' => true, - ], - 'sfo' => [ - '$id' => 'sfo', - 'name' => 'San Francisco', - 'disabled' => true, - 'flag' => 'us', - 'default' => true, - ], - 'blr' => [ - '$id' => 'blr', - 'name' => 'Bengaluru', - 'disabled' => true, - 'flag' => 'in', - 'default' => true, - ], - 'lon' => [ - '$id' => 'lon', - 'name' => 'London', - 'disabled' => true, - 'flag' => 'gb', - 'default' => true, - ], - 'ams' => [ - '$id' => 'ams', - 'name' => 'Amsterdam', - 'disabled' => true, - 'flag' => 'nl', - 'default' => true, - ], - 'sgp' => [ - '$id' => 'sgp', - 'name' => 'Singapore', - 'disabled' => true, - 'flag' => 'sg', - 'default' => true, - ], - 'tor' => [ - '$id' => 'tor', - 'name' => 'Toronto', - 'disabled' => true, - 'flag' => 'ca', - 'default' => true, - ], - 'syd' => [ - '$id' => 'syd', - 'name' => 'Sydney', - 'disabled' => true, - 'flag' => 'au', 'default' => true, ], ]; diff --git a/app/config/variables.php b/app/config/variables.php index 7c774e8d6a..98dd9ffec1 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -7,15 +7,6 @@ return [ 'category' => 'General', 'description' => '', 'variables' => [ - [ - 'name' => '_APP_REGION', - 'description' => 'Set your server running geo region. By default, the var is set to \'fra\'.', - 'introduction' => '', - 'default' => 'default', - 'required' => false, - 'question' => '', - 'filter' => '' - ], [ 'name' => '_APP_ENV', 'description' => 'Set your server running environment. By default, the var is set to \'development\'. When deploying to production, change it to: \'production\'.', diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 5099393e68..48d20cd17f 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -81,7 +81,7 @@ App::post('/v1/projects') ->param('projectId', '', new ProjectId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, and hyphen. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Project name. Max length: 128 chars.') ->param('teamId', '', new UID(), 'Team unique ID.') - ->param('region', System::getEnv('_APP_REGION'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) + ->param('region', System::getEnv('_APP_REGION', 'default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true) ->param('description', '', new Text(256), 'Project description. Max length: 256 chars.', true) ->param('logo', '', new Text(1024), 'Project logo.', true) ->param('url', '', new URL(), 'Project URL.', true) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 344a433103..77cf21efd2 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -164,7 +164,6 @@ $image = $this->getParam('image', ''); - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_ASSISTANT_OPENAI_API_KEY - - _APP_REGION appwrite-console: <<: *x-logging container_name: appwrite-console @@ -229,7 +228,6 @@ $image = $this->getParam('image', ''); - _APP_DB_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_REGION appwrite-worker-audits: image: /: @@ -256,7 +254,6 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - _APP_LOGGING_CONFIG - - _APP_REGION appwrite-worker-webhooks: image: /: @@ -285,7 +282,6 @@ $image = $this->getParam('image', ''); - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_LOGGING_CONFIG - - _APP_REGION appwrite-worker-deletes: image: /: @@ -347,7 +343,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_EMAIL_CERTIFICATES - - _APP_REGION + appwrite-worker-databases: image: /: @@ -374,7 +370,7 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - _APP_LOGGING_CONFIG - - _APP_REGION + appwrite-worker-builds: image: /: @@ -439,7 +435,7 @@ $image = $this->getParam('image', ''); - _APP_STORAGE_WASABI_SECRET - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - - _APP_REGION + appwrite-worker-certificates: image: /: @@ -473,7 +469,7 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - _APP_LOGGING_CONFIG - - _APP_REGION + appwrite-worker-functions: image: /: @@ -512,7 +508,7 @@ $image = $this->getParam('image', ''); - _APP_DOCKER_HUB_USERNAME - _APP_DOCKER_HUB_PASSWORD - _APP_LOGGING_CONFIG - - _APP_REGION + appwrite-worker-mails: image: /: @@ -547,7 +543,7 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_DOMAIN - _APP_OPTIONS_FORCE_HTTPS - - _APP_REGION + appwrite-worker-messaging: image: /: @@ -599,7 +595,7 @@ $image = $this->getParam('image', ''); - _APP_STORAGE_WASABI_SECRET - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - - _APP_REGION + appwrite-worker-migrations: image: /: @@ -630,7 +626,7 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - - _APP_REGION + appwrite-task-maintenance: image: /: @@ -665,7 +661,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - - _APP_REGION + appwrite-task-stats-resources: image: /: @@ -697,7 +693,7 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL - - _APP_REGION + appwrite-worker-stats-resources: image: /: @@ -726,7 +722,7 @@ $image = $this->getParam('image', ''); - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_STATS_RESOURCES_INTERVAL - - _APP_REGION + appwrite-worker-stats-usage: image: /: @@ -755,7 +751,7 @@ $image = $this->getParam('image', ''); - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - - _APP_REGION + appwrite-worker-stats-usage-dump: image: /: @@ -784,7 +780,7 @@ $image = $this->getParam('image', ''); - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - - _APP_REGION + appwrite-task-scheduler-functions: image: /: @@ -810,7 +806,7 @@ $image = $this->getParam('image', ''); - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_REGION + appwrite-task-scheduler-executions: image: /: @@ -836,7 +832,7 @@ $image = $this->getParam('image', ''); - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_REGION + appwrite-task-scheduler-messages: image: /: @@ -862,7 +858,7 @@ $image = $this->getParam('image', ''); - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_REGION + appwrite-assistant: image: appwrite/assistant:0.4.0 @@ -898,7 +894,6 @@ $image = $this->getParam('image', ''); - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_ENV=$_APP_ENV - - OPR_EXECUTOR_REGION=$_APP_REGION - OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG diff --git a/docker-compose.yml b/docker-compose.yml index ea796682ba..f1451426e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,7 +93,7 @@ services: - app/http.php environment: - _APP_ENV - - _APP_REGION + - _APP_EDITION - _APP_WORKER_PER_CORE - _APP_LOCALE @@ -269,7 +269,7 @@ services: - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-audits: entrypoint: worker-audits @@ -299,7 +299,7 @@ services: - _APP_DB_PASS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-webhooks: entrypoint: worker-webhooks @@ -332,7 +332,7 @@ services: - _APP_LOGGING_CONFIG - _APP_WEBHOOK_MAX_FAILED_ATTEMPTS - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-deletes: entrypoint: worker-deletes @@ -393,7 +393,7 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES - - _APP_REGION + appwrite-worker-databases: entrypoint: worker-databases @@ -425,7 +425,7 @@ services: - _APP_WORKERS_NUM - _APP_QUEUE_NAME - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-builds: entrypoint: worker-builds @@ -492,7 +492,7 @@ services: - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-certificates: entrypoint: worker-certificates @@ -528,7 +528,7 @@ services: - _APP_DB_PASS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-functions: entrypoint: worker-functions @@ -571,7 +571,7 @@ services: - _APP_LOGGING_CONFIG - _APP_LOGGING_PROVIDER - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-mails: entrypoint: worker-mails @@ -606,7 +606,7 @@ services: - _APP_DOMAIN - _APP_OPTIONS_FORCE_HTTPS - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-messaging: entrypoint: worker-messaging @@ -662,7 +662,7 @@ services: - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-migrations: entrypoint: worker-migrations @@ -698,7 +698,7 @@ services: - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-task-maintenance: entrypoint: maintenance @@ -737,7 +737,7 @@ services: - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_MAINTENANCE_DELAY - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-task-stats-resources: container_name: appwrite-task-stats-resources @@ -769,7 +769,7 @@ services: - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL - - _APP_REGION + appwrite-worker-stats-resources: entrypoint: worker-stats-resources @@ -801,7 +801,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-stats-usage: entrypoint: worker-stats-usage @@ -833,7 +833,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-worker-stats-usage-dump: entrypoint: worker-stats-usage-dump @@ -866,7 +866,7 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - _APP_STATS_USAGE_DUAL_WRITING_DBS - - _APP_REGION + appwrite-task-scheduler-functions: entrypoint: schedule-functions @@ -895,7 +895,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-task-scheduler-executions: entrypoint: schedule-executions @@ -923,7 +923,7 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_REGION + appwrite-task-scheduler-messages: entrypoint: schedule-messages @@ -952,7 +952,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - - _APP_REGION + appwrite-assistant: container_name: appwrite-assistant diff --git a/src/Appwrite/Migration/Version/V20.php b/src/Appwrite/Migration/Version/V20.php index cca0d9244b..5a0807cedf 100644 --- a/src/Appwrite/Migration/Version/V20.php +++ b/src/Appwrite/Migration/Version/V20.php @@ -426,7 +426,7 @@ class V20 extends Migration 'period' => 'inf', 'value' => $value, 'time' => null, - 'region' => 'fra', + 'region' => 'default', ])); } catch (Duplicate $th) { Console::warning("Error while creating inf metric: duplicate id {$metric} {$id}"); diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index b62b95b066..dad2db0d9a 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -104,7 +104,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION')]), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); @@ -154,7 +154,7 @@ abstract class ScheduleBase extends Action } $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION')]), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index e8195d175d..9b0590181a 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -183,7 +183,7 @@ class Deletes extends Action $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION')]), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 3100a39dea..c0ff93e9b9 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -157,7 +157,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION'), + 'region' => System::getEnv('_APP_REGION', 'default') ]); $documentClone = new Document($document->getArrayCopy()); @@ -191,7 +191,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION'), + 'region' => System::getEnv('_APP_REGION', 'default'), ]); $documentClone = new Document($document->getArrayCopy()); $dbForProject->createOrUpdateDocumentsWithIncrease( diff --git a/tests/e2e/Scopes/ProjectCustom.php b/tests/e2e/Scopes/ProjectCustom.php index 2799e29772..7f84ace6f2 100644 --- a/tests/e2e/Scopes/ProjectCustom.php +++ b/tests/e2e/Scopes/ProjectCustom.php @@ -42,7 +42,7 @@ trait ProjectCustom 'x-appwrite-project' => 'console', ], [ 'projectId' => ID::unique(), - 'region' => 'fra', + 'region' => 'default', 'name' => 'Demo Project', 'teamId' => $team['body']['$id'], 'description' => 'Demo Project Description', diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 5e24dc4d89..914a255663 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -111,7 +111,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('fra', $output['APPWRITE_REGION']); + $this->assertEquals('default', $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); @@ -221,7 +221,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('fra', $output['APPWRITE_REGION']); + $this->assertEquals('default', $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 16a18d27c1..a6498b8a4e 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -49,7 +49,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'fra', + 'region' => 'default', ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -89,7 +89,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => '', 'teamId' => $team['body']['$id'], - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(400, $response['headers']['status-code']); @@ -100,7 +100,7 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders()), [ 'projectId' => ID::unique(), 'name' => 'Project Test', - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(401, $response['headers']['status-code']); @@ -129,7 +129,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => $projectId, 'name' => 'Project Duplicate', 'teamId' => $teamId, - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(409, $response['headers']['status-code']); @@ -178,7 +178,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Team 1 Project', 'teamId' => $team1, - 'region' => 'fra', + 'region' => 'default', ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -277,7 +277,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test 2', 'teamId' => $team['body']['$id'], - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -2042,7 +2042,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -2135,7 +2135,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3749,7 +3749,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project', 'teamId' => $teamId, - 'region' => 'fra' + 'region' => 'default' ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3820,7 +3820,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 1', 'teamId' => $teamId, - 'region' => 'fra' + 'region' => 'default' ]); $project2 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ @@ -3830,7 +3830,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 2', 'teamId' => $teamId, - 'region' => 'fra' + 'region' => 'default' ]); $project1Id = $project1['body']['$id']; From e6e7c5c8161ca02f1c9fefb23285c98ad10eb7b9 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 18 Mar 2025 21:44:34 +0200 Subject: [PATCH 21/54] revert --- app/views/install/compose.phtml | 16 ---------------- docker-compose.yml | 19 ------------------- 2 files changed, 35 deletions(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 77cf21efd2..fd05d2a0b6 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -344,7 +344,6 @@ $image = $this->getParam('image', ''); - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_EMAIL_CERTIFICATES - appwrite-worker-databases: image: /: entrypoint: worker-databases @@ -371,7 +370,6 @@ $image = $this->getParam('image', ''); - _APP_DB_PASS - _APP_LOGGING_CONFIG - appwrite-worker-builds: image: /: entrypoint: worker-builds @@ -436,7 +434,6 @@ $image = $this->getParam('image', ''); - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - appwrite-worker-certificates: image: /: entrypoint: worker-certificates @@ -470,7 +467,6 @@ $image = $this->getParam('image', ''); - _APP_DB_PASS - _APP_LOGGING_CONFIG - appwrite-worker-functions: image: /: entrypoint: worker-functions @@ -509,7 +505,6 @@ $image = $this->getParam('image', ''); - _APP_DOCKER_HUB_PASSWORD - _APP_LOGGING_CONFIG - appwrite-worker-mails: image: /: entrypoint: worker-mails @@ -544,7 +539,6 @@ $image = $this->getParam('image', ''); - _APP_DOMAIN - _APP_OPTIONS_FORCE_HTTPS - appwrite-worker-messaging: image: /: entrypoint: worker-messaging @@ -596,7 +590,6 @@ $image = $this->getParam('image', ''); - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - appwrite-worker-migrations: image: /: entrypoint: worker-migrations @@ -627,7 +620,6 @@ $image = $this->getParam('image', ''); - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - appwrite-task-maintenance: image: /: entrypoint: maintenance @@ -662,7 +654,6 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - appwrite-task-stats-resources: image: /: container_name: appwrite-task-stats-resources @@ -694,7 +685,6 @@ $image = $this->getParam('image', ''); - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL - appwrite-worker-stats-resources: image: /: entrypoint: worker-stats-resources @@ -723,7 +713,6 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_STATS_RESOURCES_INTERVAL - appwrite-worker-stats-usage: image: /: entrypoint: worker-stats-usage @@ -752,7 +741,6 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - appwrite-worker-stats-usage-dump: image: /: entrypoint: worker-stats-usage-dump @@ -781,7 +769,6 @@ $image = $this->getParam('image', ''); - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - appwrite-task-scheduler-functions: image: /: entrypoint: schedule-functions @@ -807,7 +794,6 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - appwrite-task-scheduler-executions: image: /: entrypoint: schedule-executions @@ -833,7 +819,6 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - appwrite-task-scheduler-messages: image: /: entrypoint: schedule-messages @@ -859,7 +844,6 @@ $image = $this->getParam('image', ''); - _APP_DB_USER - _APP_DB_PASS - appwrite-assistant: image: appwrite/assistant:0.4.0 container_name: appwrite-assistant diff --git a/docker-compose.yml b/docker-compose.yml index f1451426e8..5933e37611 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -270,7 +270,6 @@ services: - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - appwrite-worker-audits: entrypoint: worker-audits <<: *x-logging @@ -300,7 +299,6 @@ services: - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - appwrite-worker-webhooks: entrypoint: worker-webhooks <<: *x-logging @@ -333,7 +331,6 @@ services: - _APP_WEBHOOK_MAX_FAILED_ATTEMPTS - _APP_DATABASE_SHARED_TABLES - appwrite-worker-deletes: entrypoint: worker-deletes <<: *x-logging @@ -394,7 +391,6 @@ services: - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES - appwrite-worker-databases: entrypoint: worker-databases <<: *x-logging @@ -426,7 +422,6 @@ services: - _APP_QUEUE_NAME - _APP_DATABASE_SHARED_TABLES - appwrite-worker-builds: entrypoint: worker-builds <<: *x-logging @@ -493,7 +488,6 @@ services: - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES - appwrite-worker-certificates: entrypoint: worker-certificates <<: *x-logging @@ -529,7 +523,6 @@ services: - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - appwrite-worker-functions: entrypoint: worker-functions <<: *x-logging @@ -572,7 +565,6 @@ services: - _APP_LOGGING_PROVIDER - _APP_DATABASE_SHARED_TABLES - appwrite-worker-mails: entrypoint: worker-mails <<: *x-logging @@ -607,7 +599,6 @@ services: - _APP_OPTIONS_FORCE_HTTPS - _APP_DATABASE_SHARED_TABLES - appwrite-worker-messaging: entrypoint: worker-messaging <<: *x-logging @@ -663,7 +654,6 @@ services: - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES - appwrite-worker-migrations: entrypoint: worker-migrations <<: *x-logging @@ -699,7 +689,6 @@ services: - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_DATABASE_SHARED_TABLES - appwrite-task-maintenance: entrypoint: maintenance <<: *x-logging @@ -738,7 +727,6 @@ services: - _APP_MAINTENANCE_DELAY - _APP_DATABASE_SHARED_TABLES - appwrite-task-stats-resources: container_name: appwrite-task-stats-resources entrypoint: stats-resources @@ -770,7 +758,6 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL - appwrite-worker-stats-resources: entrypoint: worker-stats-resources <<: *x-logging @@ -802,7 +789,6 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - appwrite-worker-stats-usage: entrypoint: worker-stats-usage <<: *x-logging @@ -834,7 +820,6 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - appwrite-worker-stats-usage-dump: entrypoint: worker-stats-usage-dump <<: *x-logging @@ -867,7 +852,6 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_STATS_USAGE_DUAL_WRITING_DBS - appwrite-task-scheduler-functions: entrypoint: schedule-functions <<: *x-logging @@ -896,7 +880,6 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - appwrite-task-scheduler-executions: entrypoint: schedule-executions <<: *x-logging @@ -924,7 +907,6 @@ services: - _APP_DB_USER - _APP_DB_PASS - appwrite-task-scheduler-messages: entrypoint: schedule-messages <<: *x-logging @@ -953,7 +935,6 @@ services: - _APP_DB_PASS - _APP_DATABASE_SHARED_TABLES - appwrite-assistant: container_name: appwrite-assistant image: appwrite/assistant:0.7.0 From ee09e2c91b18c280b2ebbf082df06a9e9351fa83 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 18 Mar 2025 21:47:54 +0200 Subject: [PATCH 22/54] revert --- docker-compose.yml | 5 +---- src/Appwrite/Platform/Workers/StatsUsageDump.php | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5933e37611..3f9f54bc1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,7 +93,6 @@ services: - app/http.php environment: - _APP_ENV - - _APP_EDITION - _APP_WORKER_PER_CORE - _APP_LOCALE @@ -788,7 +787,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - + - appwrite-worker-stats-usage: entrypoint: worker-stats-usage <<: *x-logging @@ -967,7 +966,6 @@ services: - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_ENV=$_APP_ENV - - OPR_EXECUTOR_REGION=$_APP_REGION - OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - OPR_EXECUTOR_RUNTIME_VERSIONS=v2,v4 @@ -1007,7 +1005,6 @@ services: environment: - OPR_PROXY_WORKER_PER_CORE=$_APP_WORKER_PER_CORE - OPR_PROXY_ENV=$_APP_ENV - - OPR_PROXY_REGION=$_APP_REGION - OPR_PROXY_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - OPR_PROXY_SECRET=$_APP_EXECUTOR_SECRET - OPR_PROXY_LOGGING_CONFIG=$_APP_LOGGING_CONFIG diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index c0ff93e9b9..747e6fdbfd 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -191,7 +191,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => System::getEnv('_APP_REGION', 'default') ]); $documentClone = new Document($document->getArrayCopy()); $dbForProject->createOrUpdateDocumentsWithIncrease( From eb4340cf4c3f3a445ced961f2e75f0957035d11b Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 18 Mar 2025 21:49:28 +0200 Subject: [PATCH 23/54] revert --- docker-compose.yml | 2 +- src/Appwrite/Platform/Workers/StatsUsageDump.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3f9f54bc1e..b6dc80df6a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -787,7 +787,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - - + appwrite-worker-stats-usage: entrypoint: worker-stats-usage <<: *x-logging diff --git a/src/Appwrite/Platform/Workers/StatsUsageDump.php b/src/Appwrite/Platform/Workers/StatsUsageDump.php index 747e6fdbfd..119a9e7288 100644 --- a/src/Appwrite/Platform/Workers/StatsUsageDump.php +++ b/src/Appwrite/Platform/Workers/StatsUsageDump.php @@ -157,7 +157,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default') + 'region' => System::getEnv('_APP_REGION', 'default'), ]); $documentClone = new Document($document->getArrayCopy()); @@ -191,7 +191,7 @@ class StatsUsageDump extends Action 'time' => $time, 'metric' => $key, 'value' => $value, - 'region' => System::getEnv('_APP_REGION', 'default') + 'region' => System::getEnv('_APP_REGION', 'default'), ]); $documentClone = new Document($document->getArrayCopy()); $dbForProject->createOrUpdateDocumentsWithIncrease( From faed019d921847e05ff9ff5f7d2b42684dc8424e Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 19 Mar 2025 09:32:48 +0200 Subject: [PATCH 24/54] revert --- tests/e2e/Scopes/ProjectCustom.php | 3 ++- .../Projects/ProjectsConsoleClientTest.php | 23 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/e2e/Scopes/ProjectCustom.php b/tests/e2e/Scopes/ProjectCustom.php index 7f84ace6f2..533fccd87d 100644 --- a/tests/e2e/Scopes/ProjectCustom.php +++ b/tests/e2e/Scopes/ProjectCustom.php @@ -4,6 +4,7 @@ namespace Tests\E2E\Scopes; use Tests\E2E\Client; use Utopia\Database\Helpers\ID; +use Utopia\System\System; trait ProjectCustom { @@ -42,7 +43,7 @@ trait ProjectCustom 'x-appwrite-project' => 'console', ], [ 'projectId' => ID::unique(), - 'region' => 'default', + 'region' => System::getEnv('_APP_REGION', 'default'), 'name' => 'Demo Project', 'teamId' => $team['body']['$id'], 'description' => 'Demo Project Description', diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index a6498b8a4e..8e2ab03880 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -14,6 +14,7 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; +use Utopia\System\System; class ProjectsConsoleClientTest extends Scope { @@ -49,7 +50,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default', + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -89,7 +90,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => '', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(400, $response['headers']['status-code']); @@ -100,7 +101,7 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders()), [ 'projectId' => ID::unique(), 'name' => 'Project Test', - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(401, $response['headers']['status-code']); @@ -129,7 +130,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => $projectId, 'name' => 'Project Duplicate', 'teamId' => $teamId, - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(409, $response['headers']['status-code']); @@ -178,7 +179,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Team 1 Project', 'teamId' => $team1, - 'region' => 'default', + 'region' => System::getEnv('_APP_REGION', 'default'), ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -277,7 +278,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test 2', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -2042,7 +2043,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -2135,7 +2136,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3749,7 +3750,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project', 'teamId' => $teamId, - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3820,7 +3821,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 1', 'teamId' => $teamId, - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $project2 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ @@ -3830,7 +3831,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 2', 'teamId' => $teamId, - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $project1Id = $project1['body']['$id']; From 88d6150db2f67ffaf1a4918f0427d444fca770d1 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 19 Mar 2025 10:29:18 +0200 Subject: [PATCH 25/54] revert --- tests/e2e/Services/Functions/FunctionsCustomClientTest.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 914a255663..55db081720 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -8,6 +8,7 @@ use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideClient; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Role; +use Utopia\System\System; class FunctionsCustomClientTest extends Scope { @@ -111,7 +112,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('default', $output['APPWRITE_REGION']); + $this->assertEquals(System::getEnv('_APP_REGION', 'default'), $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); @@ -221,7 +222,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('default', $output['APPWRITE_REGION']); + $this->assertEquals(System::getEnv('_APP_REGION', 'default'), $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); From cc0a97b10e30290fce85fc1a84cf5f9948c8f391 Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 19 Mar 2025 16:37:15 +0200 Subject: [PATCH 26/54] Fixes --- app/config/collections/common.php | 6 +++--- src/Appwrite/Platform/Workers/Deletes.php | 24 ++++++++++++----------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/app/config/collections/common.php b/app/config/collections/common.php index f68400e226..3f6245dfdd 100644 --- a/app/config/collections/common.php +++ b/app/config/collections/common.php @@ -1038,7 +1038,7 @@ return [ '$id' => ID::custom('providerUid'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 2048, + 'size' => 2048, // Decrease to 128 as in index length? 'signed' => true, 'required' => false, 'default' => null, @@ -1107,14 +1107,14 @@ return [ '$id' => ID::custom('_key_userInternalId_provider_providerUid'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['userInternalId', 'provider', 'providerUid'], - 'lengths' => [11, 128, 128], + 'lengths' => [11, 128, 128], // providerUid is length 2000! 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ '$id' => ID::custom('_key_provider_providerUid'), 'type' => Database::INDEX_UNIQUE, 'attributes' => ['provider', 'providerUid'], - 'lengths' => [128, 128], + 'lengths' => [128, 128], // providerUid is length 2000! 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], [ diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index a14e54caab..27ff27fe31 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -357,21 +357,23 @@ class Deletes extends Action new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) ); - $query[] = Query::equal('resource', [$resource]); + $queries = [ + Query::equal('resource', [$resource]) + ]; if (!empty($resourceType)) { - $query[] = Query::equal('resourceType', [$resourceType]); + $queries[] = Query::equal('resourceType', [$resourceType]); } - $query[] = Query::orderAsc(); + $queries[] = Query::orderAsc(); /** - * todo: missing index on `resource`, `resourceType` + * todo: No index on `resource`, `resourceType`, But it is fine since resource index is a strong index! */ $this->deleteByGroup( 'cache', - $query, + $queries, $dbForProject, function (Document $document) use ($cache, $projectId) { $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); @@ -719,7 +721,7 @@ class Deletes extends Action // Delete identities /** - * todo: Remove Duplication index `_key_userInternalId` + * todo: Remove Duplication index `_key_userInternalId` , But lets leave because of index length issue in unique index */ $this->deleteByGroup('identities', [ Query::equal('userInternalId', [$userInternalId]), @@ -836,13 +838,13 @@ class Deletes extends Action /** * Delete rules - * todo: No index for this query + * todo: No index for this query, drop _key_projectInternalId and create _key_projectInternalId, resourceInternalId, resourceType */ Console::info("Deleting rules for function " . $functionId); $this->deleteByGroup('rules', [ - Query::equal('resourceType', ['function']), - Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('projectInternalId', [$project->getInternalId()]), + Query::equal('resourceInternalId', [$functionInternalId]), + Query::equal('resourceType', ['function']), Query::orderAsc() ], $dbForPlatform, function (Document $document) use ($project, $dbForPlatform, $certificates) { $this->deleteRule($dbForPlatform, $document, $certificates); @@ -850,12 +852,12 @@ class Deletes extends Action /** * Delete Variables - * todo: No index for this query + * todo: No index for this query , drop _key_resourceInternalId and create new one with {resourceInternalId, resourceType} */ Console::info("Deleting variables for function " . $functionId); $this->deleteByGroup('variables', [ - Query::equal('resourceType', ['function']), Query::equal('resourceInternalId', [$functionInternalId]), + Query::equal('resourceType', ['function']), Query::orderAsc() ], $dbForProject); From 31ca35782baa46d73a2ad75cdb209f6c08d7bc97 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 20 Mar 2025 15:21:24 +1300 Subject: [PATCH 27/54] Use cursor pagination with bigger limit for maintenance project loop --- src/Appwrite/Platform/Tasks/Maintenance.php | 33 +++------------------ 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 98a3f4d295..b9e312a3fb 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -47,13 +47,15 @@ class Maintenance extends Action Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); - $this->foreachProject($dbForPlatform, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) { + $dbForPlatform->foreach('projects', function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) { $queueForDeletes ->setType(DELETE_TYPE_MAINTENANCE) ->setProject($project) ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) ->trigger(); - }); + }, [ + Query::limit(100), + ]); $queueForDeletes ->setType(DELETE_TYPE_MAINTENANCE) @@ -68,33 +70,6 @@ class Maintenance extends Action }, $interval, $delay); } - protected function foreachProject(Database $dbForPlatform, callable $callback): void - { - // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document - $count = 0; - $chunk = 0; - $limit = 50; - $sum = $limit; - $executionStart = \microtime(true); - - while ($sum === $limit) { - $projects = $dbForPlatform->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]); - - $chunk++; - - /** @var string[] $projectIds */ - $sum = count($projects); - - foreach ($projects as $project) { - $callback($project); - $count++; - } - } - - $executionEnd = \microtime(true); - Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds"); - } - private function notifyDeleteConnections(Delete $queueForDeletes): void { $queueForDeletes From 9c54cce5bd8c61d26a549273dfe30b6bcfd94751 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 20 Mar 2025 09:07:32 +0200 Subject: [PATCH 28/54] Queries fixes --- src/Appwrite/Platform/Workers/Deletes.php | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 27ff27fe31..b7fce0405d 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -365,6 +365,7 @@ class Deletes extends Action $queries[] = Query::equal('resourceType', [$resourceType]); } + $queries[] = Query::select(['$internalId', '$id', '$updatedAt']); $queries[] = Query::orderAsc(); /** @@ -404,7 +405,8 @@ class Deletes extends Action new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) ); - $query = [ + $queries = [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), Query::orderDesc(), @@ -412,7 +414,7 @@ class Deletes extends Action $this->deleteByGroup( 'cache', - $query, + $queries, $dbForProject, function (Document $document) use ($cache, $projectId) { $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); @@ -440,6 +442,7 @@ class Deletes extends Action // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -452,6 +455,7 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -755,6 +759,7 @@ class Deletes extends Action // Delete Executions $this->deleteByGroup('executions', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -775,6 +780,7 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -811,6 +817,7 @@ class Deletes extends Action try { $this->deleteByGroup(Audit::COLLECTION, [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderAsc(), // KEY "index-time" ("time" DESC) @@ -897,6 +904,7 @@ class Deletes extends Action */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ + Query::select(['$internalId', '$id', '$updatedAt']), Query::equal('functionInternalId', [$functionInternalId]), Query::orderAsc() ], $dbForProject); @@ -1017,7 +1025,7 @@ class Deletes extends Action /** * Delete builds - * todo: no index for `deploymentInternalId` change to temporary to deploymentId? + * todo: no index for `deploymentInternalId` Same as above index , no need to handle again... */ Console::info("Deleting builds for deployment " . $deploymentId); @@ -1047,8 +1055,7 @@ class Deletes extends Action string $collection, array $queries, Database $database, - ?callable $callback = null, - bool $shortSelect = false + ?callable $callback = null ): void { $start = \microtime(true); @@ -1056,12 +1063,6 @@ class Deletes extends Action * deleteDocuments uses a cursor, we need to add a unique order by field or use default */ - if (!\is_callable($callback) && $shortSelect) { - $queries = array_merge($queries, [ - Query::select(['$internalId', '$id', '$permissions', '$updatedAt']) - ]); - } - try { $documents = $database->deleteDocuments($collection, $queries); } catch (Throwable $th) { From 5eb7c36f50e2713aee76beaff1bb207eca130116 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 10:32:36 +0000 Subject: [PATCH 29/54] chore: add console audit retention --- .env | 2 +- app/worker.php | 2 +- docker-compose.yml | 1 + src/Appwrite/Platform/Workers/Deletes.php | 9 ++++++++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 1893e023ba..c68f0a0d52 100644 --- a/.env +++ b/.env @@ -85,7 +85,7 @@ _APP_MAINTENANCE_DELAY= _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 -_APP_MAINTENANCE_RETENTION_AUDIT=1209600 +_APP_MAINTENANCE_RETENTION_AUDIT=project=1209600,console=15778800 _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=3600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 diff --git a/app/worker.php b/app/worker.php index eeefe80000..5d7bd4de41 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,7 +219,7 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); + return System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800'); // project = 14 days, console = 6 months }); Server::setResource('executionRetention', function () { diff --git a/docker-compose.yml b/docker-compose.yml index b6dc80df6a..af6a323ec8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -389,6 +389,7 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES + - _APP_MAINTENANCE_RETENTION_AUDIT appwrite-worker-databases: entrypoint: worker-databases diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 9b0590181a..75380ed3fe 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -751,9 +751,16 @@ class Deletes extends Action $projectId = $project->getId(); $dbForProject = $getProjectDB($project); + [$projectAuditRetention, $consoleAuditRetention] = array_map( + function ($part) { + return DateTime::addSeconds(new \DateTime(), -1 * (int)explode('=', $part)[1]); + }, + explode(',', $auditRetention) + ); + try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::lessThan('time', $auditRetention), + Query::lessThan('time', ($projectId === 'console' ? $consoleAuditRetention : $projectAuditRetention)), Query::orderDesc('time'), Query::orderDesc('$internalId'), ], $dbForProject); From 9d8289221925ddf7eeebbd42f56ba9be311dd2a0 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 11:32:06 +0000 Subject: [PATCH 30/54] chore: update logic --- app/worker.php | 10 +++++++++- src/Appwrite/Platform/Workers/Deletes.php | 15 ++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/app/worker.php b/app/worker.php index 5d7bd4de41..18b51eda25 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,7 +219,15 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800'); // project = 14 days, console = 6 months + return array_map( + function ($part) { + [$key, $value] = explode('=', $part); + return [ + $key => DateTime::addSeconds(new \DateTime(), -1 * (int)$value) + ]; + }, + explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')) + ); }); Server::setResource('executionRetention', function () { diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 75380ed3fe..93f8748e49 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -57,7 +57,7 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log) => $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -746,21 +746,14 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void + private function deleteAuditLogs(Document $project, callable $getProjectDB, array $auditRetention): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); - [$projectAuditRetention, $consoleAuditRetention] = array_map( - function ($part) { - return DateTime::addSeconds(new \DateTime(), -1 * (int)explode('=', $part)[1]); - }, - explode(',', $auditRetention) - ); - try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::lessThan('time', ($projectId === 'console' ? $consoleAuditRetention : $projectAuditRetention)), + Query::lessThan('time', ($projectId === 'console' ? $auditRetention['console'] : $auditRetention['project'])), Query::orderDesc('time'), Query::orderDesc('$internalId'), ], $dbForProject); From 7c3f8504bb65e598f50d07912b5d753c86f5e1f6 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 13:15:14 +0000 Subject: [PATCH 31/54] chore: fix initialization --- app/worker.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/worker.php b/app/worker.php index 18b51eda25..e305ec42d6 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,14 +219,14 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return array_map( - function ($part) { + return array_reduce( + explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), + function ($carry, $part) { [$key, $value] = explode('=', $part); - return [ - $key => DateTime::addSeconds(new \DateTime(), -1 * (int)$value) - ]; + $carry[$key] = DateTime::addSeconds(new \DateTime(), -1 * (int)$value); + return $carry; }, - explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')) + [] ); }); From b0b57935da0e54074766a3f9a144f41fe2081a55 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 20 Mar 2025 13:42:19 +0000 Subject: [PATCH 32/54] chore: add comments --- app/worker.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/worker.php b/app/worker.php index e305ec42d6..d0fa5cf4f8 100644 --- a/app/worker.php +++ b/app/worker.php @@ -215,12 +215,12 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache) { }, ['pools', 'cache']); Server::setResource('abuseRetention', function () { - return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); + return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); // 1 day }); Server::setResource('auditRetention', function () { return array_reduce( - explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), + explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), // project = 14 days, console = 6 months function ($carry, $part) { [$key, $value] = explode('=', $part); $carry[$key] = DateTime::addSeconds(new \DateTime(), -1 * (int)$value); @@ -231,7 +231,7 @@ Server::setResource('auditRetention', function () { }); Server::setResource('executionRetention', function () { - return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); // 14 days }); Server::setResource('cache', function (Registry $register) { From 13750f7dbd5b5850075eefab4bacf3e782752004 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 07:42:10 +0000 Subject: [PATCH 33/54] feat: introduce new env --- .env | 3 ++- app/config/variables.php | 11 ++++++++++- app/views/install/compose.phtml | 3 +++ app/worker.php | 13 ++++--------- docker-compose.yml | 3 +++ 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.env b/.env index c68f0a0d52..c10c12613b 100644 --- a/.env +++ b/.env @@ -85,7 +85,8 @@ _APP_MAINTENANCE_DELAY= _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 -_APP_MAINTENANCE_RETENTION_AUDIT=project=1209600,console=15778800 +_APP_MAINTENANCE_RETENTION_AUDIT=1209600 +_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE=15778800 _APP_USAGE_AGGREGATION_INTERVAL=30 _APP_STATS_RESOURCES_INTERVAL=3600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 diff --git a/app/config/variables.php b/app/config/variables.php index 98dd9ffec1..27463d2fee 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -1048,13 +1048,22 @@ return [ ], [ 'name' => '_APP_MAINTENANCE_RETENTION_AUDIT', - 'description' => 'IThe maximum duration (in seconds) upto which to retain audit logs. The default value is 1209600 seconds (14 days).', + 'description' => 'The maximum duration (in seconds) upto which to retain audit logs. The default value is 1209600 seconds (14 days).', 'introduction' => '0.7.0', 'default' => '1209600', 'required' => false, 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', + 'description' => 'The maximum duration (in seconds) upto which to retain console audit logs. The default value is 15778800 seconds (6 months).', + 'introduction' => '1.6.2', + 'default' => '15778800', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_MAINTENANCE_RETENTION_ABUSE', 'description' => 'The maximum duration (in seconds) upto which to retain abuse logs. The default value is 86400 seconds (1 day).', diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index fd05d2a0b6..7dfe14fcef 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -148,6 +148,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_SMS_PROVIDER @@ -340,6 +341,7 @@ $image = $this->getParam('image', ''); - _APP_EXECUTOR_HOST - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_EMAIL_CERTIFICATES @@ -651,6 +653,7 @@ $image = $this->getParam('image', ''); - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES diff --git a/app/worker.php b/app/worker.php index d0fa5cf4f8..3f99089eef 100644 --- a/app/worker.php +++ b/app/worker.php @@ -219,15 +219,10 @@ Server::setResource('abuseRetention', function () { }); Server::setResource('auditRetention', function () { - return array_reduce( - explode(',', System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 'project=1209600,console=15778800')), // project = 14 days, console = 6 months - function ($carry, $part) { - [$key, $value] = explode('=', $part); - $carry[$key] = DateTime::addSeconds(new \DateTime(), -1 * (int)$value); - return $carry; - }, - [] - ); + return [ + 'project' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', '1209600')), + 'console' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', '15778800')) + ]; }); Server::setResource('executionRetention', function () { diff --git a/docker-compose.yml b/docker-compose.yml index af6a323ec8..05ddba967a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -171,6 +171,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_SMS_PROVIDER @@ -390,6 +391,7 @@ services: - _APP_DATABASE_SHARED_TABLES_V1 - _APP_EMAIL_CERTIFICATES - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE appwrite-worker-databases: entrypoint: worker-databases @@ -722,6 +724,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_MAINTENANCE_DELAY From 7c795da72fd7aa05b280f27012f87357a08d2814 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 07:48:50 +0000 Subject: [PATCH 34/54] chore: fix type --- app/worker.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/worker.php b/app/worker.php index 3f99089eef..744c758c69 100644 --- a/app/worker.php +++ b/app/worker.php @@ -220,8 +220,8 @@ Server::setResource('abuseRetention', function () { Server::setResource('auditRetention', function () { return [ - 'project' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', '1209600')), - 'console' => DateTime::addSeconds(new \DateTime(), -1 * (int) System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', '15778800')) + 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), + 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) ]; }); From c0c0c921dc249f2f235a9bd7e8efdce66ae3c137 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 08:08:30 +0000 Subject: [PATCH 35/54] chore: fix naming --- app/worker.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/worker.php b/app/worker.php index 744c758c69..491f266a18 100644 --- a/app/worker.php +++ b/app/worker.php @@ -220,8 +220,8 @@ Server::setResource('abuseRetention', function () { Server::setResource('auditRetention', function () { return [ - 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), - 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) + 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), // 14 days + 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) // 6 months ]; }); From d661195a5f4bf375f7b72e5629e1a1dfd3fe8d3d Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Fri, 21 Mar 2025 15:59:30 +0000 Subject: [PATCH 36/54] chore: only update total count for prviledged users --- app/controllers/api/teams.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index b45c9fd3b9..dfdccf5e99 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -615,7 +615,10 @@ App::post('/v1/teams/:teamId/memberships') $membership = ($isPrivilegedUser || $isAppUser) ? Authorization::skip(fn () => $dbForProject->createDocument('memberships', $membership)) : $dbForProject->createDocument('memberships', $membership); - Authorization::skip(fn () => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1)); + + if ($isPrivilegedUser || $isAppUser) { + Authorization::skip(fn () => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1)); + } } elseif ($membership->getAttribute('confirm') === false) { $membership->setAttribute('secret', Auth::hash($secret)); From 37dfe0d719b49f32b2540b3d6b3310fcd110e976 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 22 Mar 2025 14:05:39 +0000 Subject: [PATCH 37/54] refactor: initialization of audit retention --- app/worker.php | 14 ++++++-------- src/Appwrite/Platform/Workers/Deletes.php | 10 +++++----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/app/worker.php b/app/worker.php index 491f266a18..90496c0430 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,8 +16,6 @@ use Appwrite\Event\Migration; use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; use Appwrite\Event\StatsUsageDump; -/** remove */ -/** /remove */ use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; use Swoole\Runtime; @@ -218,12 +216,12 @@ Server::setResource('abuseRetention', function () { return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); // 1 day }); -Server::setResource('auditRetention', function () { - return [ - 'project' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)), // 14 days - 'console' => DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)) // 6 months - ]; -}); +Server::setResource('auditRetention', function (Document $project) { + if ($project->getId() === 'console') { + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)); // 6 months + } + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); // 14 days +}, ['project']); Server::setResource('executionRetention', function () { return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); // 14 days diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 93f8748e49..bb5cc81fcc 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -57,7 +57,7 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log) => + fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, array $auditRetention, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -742,18 +742,18 @@ class Deletes extends Action /** * @param Database $dbForPlatform * @param callable $getProjectDB - * @param string $datetime + * @param string $auditRetention * @return void * @throws Exception */ - private function deleteAuditLogs(Document $project, callable $getProjectDB, array $auditRetention): void + private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::lessThan('time', ($projectId === 'console' ? $auditRetention['console'] : $auditRetention['project'])), + Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderDesc('$internalId'), ], $dbForProject); From 26821c83726db7506fbf2d0828f34a908cb5d1d3 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 11:08:39 +0200 Subject: [PATCH 38/54] Bump database 0.62 --- composer.json | 2 +- composer.lock | 311 ++++---------------------------------------------- 2 files changed, 22 insertions(+), 291 deletions(-) diff --git a/composer.json b/composer.json index d7b8505b5c..b1b6aed539 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.12.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.61.*", + "utopia-php/database": "0.62.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index cdba756c72..1ea887fdbc 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e0d7f21b681e4591144fec16c4f0d6aa", + "content-hash": "d2bec8137dcd84994121f89a29932d31", "packages": [ { "name": "adhocore/jwt", @@ -751,65 +751,6 @@ }, "time": "2025-03-13T21:08:17+00:00" }, - { - "name": "jean85/pretty-package-versions", - "version": "2.1.0", - "source": { - "type": "git", - "url": "https://github.com/Jean85/pretty-package-versions.git", - "reference": "3c4e5f62ba8d7de1734312e4fff32f67a8daaf10" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/Jean85/pretty-package-versions/zipball/3c4e5f62ba8d7de1734312e4fff32f67a8daaf10", - "reference": "3c4e5f62ba8d7de1734312e4fff32f67a8daaf10", - "shasum": "" - }, - "require": { - "composer-runtime-api": "^2.1.0", - "php": "^7.4|^8.0" - }, - "require-dev": { - "friendsofphp/php-cs-fixer": "^3.2", - "jean85/composer-provided-replaced-stub-package": "^1.0", - "phpstan/phpstan": "^1.4", - "phpunit/phpunit": "^7.5|^8.5|^9.6", - "vimeo/psalm": "^4.3 || ^5.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.x-dev" - } - }, - "autoload": { - "psr-4": { - "Jean85\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Alessandro Lai", - "email": "alessandro.lai85@gmail.com" - } - ], - "description": "A library to get pretty versions strings of installed dependencies", - "keywords": [ - "composer", - "package", - "release", - "versions" - ], - "support": { - "issues": "https://github.com/Jean85/pretty-package-versions/issues", - "source": "https://github.com/Jean85/pretty-package-versions/tree/2.1.0" - }, - "time": "2024-11-18T16:19:46+00:00" - }, { "name": "league/csv", "version": "9.14.0", @@ -968,75 +909,6 @@ }, "time": "2023-10-02T10:01:54+00:00" }, - { - "name": "mongodb/mongodb", - "version": "1.10.0", - "source": { - "type": "git", - "url": "https://github.com/mongodb/mongo-php-library.git", - "reference": "b0bbd657f84219212487d01a8ffe93a789e1e488" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/b0bbd657f84219212487d01a8ffe93a789e1e488", - "reference": "b0bbd657f84219212487d01a8ffe93a789e1e488", - "shasum": "" - }, - "require": { - "ext-hash": "*", - "ext-json": "*", - "ext-mongodb": "^1.11.0", - "jean85/pretty-package-versions": "^1.2 || ^2.0.1", - "php": "^7.1 || ^8.0", - "symfony/polyfill-php80": "^1.19" - }, - "require-dev": { - "doctrine/coding-standard": "^9.0", - "squizlabs/php_codesniffer": "^3.6", - "symfony/phpunit-bridge": "^5.2" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.10.x-dev" - } - }, - "autoload": { - "files": [ - "src/functions.php" - ], - "psr-4": { - "MongoDB\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "Apache-2.0" - ], - "authors": [ - { - "name": "Andreas Braun", - "email": "andreas.braun@mongodb.com" - }, - { - "name": "Jeremy Mikola", - "email": "jmikola@gmail.com" - } - ], - "description": "MongoDB driver library", - "homepage": "https://jira.mongodb.org/browse/PHPLIB", - "keywords": [ - "database", - "driver", - "mongodb", - "persistence" - ], - "support": { - "issues": "https://github.com/mongodb/mongo-php-library/issues", - "source": "https://github.com/mongodb/mongo-php-library/tree/1.10.0" - }, - "time": "2021-10-20T22:22:37+00:00" - }, { "name": "mustangostang/spyc", "version": "0.6.3", @@ -2371,16 +2243,16 @@ }, { "name": "ramsey/collection", - "version": "2.1.0", + "version": "2.1.1", "source": { "type": "git", "url": "https://github.com/ramsey/collection.git", - "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109" + "reference": "344572933ad0181accbf4ba763e85a0306a8c5e2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ramsey/collection/zipball/3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", - "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", + "url": "https://api.github.com/repos/ramsey/collection/zipball/344572933ad0181accbf4ba763e85a0306a8c5e2", + "reference": "344572933ad0181accbf4ba763e85a0306a8c5e2", "shasum": "" }, "require": { @@ -2441,9 +2313,9 @@ ], "support": { "issues": "https://github.com/ramsey/collection/issues", - "source": "https://github.com/ramsey/collection/tree/2.1.0" + "source": "https://github.com/ramsey/collection/tree/2.1.1" }, - "time": "2025-03-02T04:48:29+00:00" + "time": "2025-03-22T05:38:12+00:00" }, { "name": "ramsey/uuid", @@ -2932,86 +2804,6 @@ ], "time": "2024-09-09T11:45:10+00:00" }, - { - "name": "symfony/polyfill-php80", - "version": "v1.31.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "60328e362d4c2c802a54fcbf04f9d3fb892b4cf8" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/60328e362d4c2c802a54fcbf04f9d3fb892b4cf8", - "reference": "60328e362d4c2c802a54fcbf04f9d3fb892b4cf8", - "shasum": "" - }, - "require": { - "php": ">=7.2" - }, - "type": "library", - "extra": { - "thanks": { - "url": "https://github.com/symfony/polyfill", - "name": "symfony/polyfill" - } - }, - "autoload": { - "files": [ - "bootstrap.php" - ], - "psr-4": { - "Symfony\\Polyfill\\Php80\\": "" - }, - "classmap": [ - "Resources/stubs" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Ion Bazan", - "email": "ion.bazan@gmail.com" - }, - { - "name": "Nicolas Grekas", - "email": "p@tchwork.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill backporting some PHP 8.0+ features to lower PHP versions", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "polyfill", - "portable", - "shim" - ], - "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.31.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2024-09-09T11:45:10+00:00" - }, { "name": "symfony/polyfill-php82", "version": "v1.31.0", @@ -3705,16 +3497,16 @@ }, { "name": "utopia-php/database", - "version": "0.61.2", + "version": "0.62.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436" + "reference": "65dc51466c12552add10395900cdbb4728da4068" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/349fbdf4bc088f7775c7dfb8b80239a617a88436", - "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436", + "url": "https://api.github.com/repos/utopia-php/database/zipball/65dc51466c12552add10395900cdbb4728da4068", + "reference": "65dc51466c12552add10395900cdbb4728da4068", "shasum": "" }, "require": { @@ -3722,8 +3514,7 @@ "ext-pdo": "*", "php": ">=8.1", "utopia-php/cache": "0.12.*", - "utopia-php/framework": "0.33.*", - "utopia-php/mongo": "0.3.*" + "utopia-php/framework": "0.33.*" }, "require-dev": { "fakerphp/faker": "1.23.*", @@ -3755,9 +3546,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.61.2" + "source": "https://github.com/utopia-php/database/tree/0.62.1" }, - "time": "2025-03-15T11:47:42+00:00" + "time": "2025-03-24T08:27:18+00:00" }, { "name": "utopia-php/domains", @@ -4159,16 +3950,16 @@ }, { "name": "utopia-php/migration", - "version": "0.8.1", + "version": "0.8.2", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab" + "reference": "aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab", - "reference": "36ec7af2e6bf78de5d86e1b0a953fd7dcdf69dab", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1", + "reference": "aa3b7a508feb7090f487e7bf9cd71f5c92fbc7c1", "shasum": "" }, "require": { @@ -4176,7 +3967,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.61.*", + "utopia-php/database": "0.62.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4209,69 +4000,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.8.1" + "source": "https://github.com/utopia-php/migration/tree/0.8.2" }, - "time": "2025-03-18T07:48:08+00:00" - }, - { - "name": "utopia-php/mongo", - "version": "0.3.1", - "source": { - "type": "git", - "url": "https://github.com/utopia-php/mongo.git", - "reference": "52326a9a43e2d27ff0c15c48ba746dacbe9a7aee" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/52326a9a43e2d27ff0c15c48ba746dacbe9a7aee", - "reference": "52326a9a43e2d27ff0c15c48ba746dacbe9a7aee", - "shasum": "" - }, - "require": { - "ext-mongodb": "*", - "mongodb/mongodb": "1.10.0", - "php": ">=8.0" - }, - "require-dev": { - "fakerphp/faker": "^1.14", - "laravel/pint": "1.2.*", - "phpstan/phpstan": "1.8.*", - "phpunit/phpunit": "^9.4", - "swoole/ide-helper": "4.8.0" - }, - "type": "library", - "autoload": { - "psr-4": { - "Utopia\\Mongo\\": "src" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - }, - { - "name": "Wess", - "email": "wess@appwrite.io" - } - ], - "description": "A simple library to manage Mongo database", - "keywords": [ - "database", - "mongo", - "php", - "upf", - "utopia" - ], - "support": { - "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/0.3.1" - }, - "time": "2023-09-01T17:25:28+00:00" + "time": "2025-03-24T09:05:31+00:00" }, { "name": "utopia-php/orchestration", From 5479e6f8f1f51691e168564b2592d15df917a421 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Tue, 18 Mar 2025 12:29:32 +0100 Subject: [PATCH 39/54] chore(audits): return queue pre-fetch results --- src/Appwrite/Platform/Workers/Audits.php | 49 +++++++++++++----------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index ed5ff8010a..962bc622d1 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -12,12 +12,12 @@ use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Structure; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Queue\Result\Commit; +use Utopia\Queue\Result\NoCommit; use Utopia\System\System; class Audits extends Action { - protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development - protected const BATCH_SIZE_PRODUCTION = 5_000; protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds private int $lastTriggeredTime = 0; @@ -27,9 +27,7 @@ class Audits extends Action protected function getBatchSize(): int { - return System::getEnv('_APP_ENV', 'development') === 'development' - ? self::BATCH_SIZE_DEVELOPMENT - : self::BATCH_SIZE_PRODUCTION; + return intval(System::getEnv('_APP_QUEUE_PREFETCH_COUNT', 1)); } public static function getName(): string @@ -57,13 +55,13 @@ class Audits extends Action * @param Message $message * @param callable $getProjectDB * @param Document $project - * @return void + * @return Commit|NoCommit * @throws Throwable * @throws \Utopia\Database\Exception * @throws Authorization * @throws Structure */ - public function action(Message $message, callable $getProjectDB, Document $project): void + public function action(Message $message, callable $getProjectDB, Document $project): Commit|NoCommit { $payload = $message->getPayload() ?? []; @@ -119,29 +117,34 @@ class Audits extends Action // Check if we should process the batch by checking both for the batch size and the elapsed time $batchSize = $this->getBatchSize(); - $shouldProcessBatch = \count($this->logs) >= $batchSize; - if (!$shouldProcessBatch && \count($this->logs) > 0) { + $logCount = array_reduce($this->logs, fn (int $current, $logs) => $current + count($logs), 0); + $shouldProcessBatch = $logCount >= $batchSize; + if (!$shouldProcessBatch && $logCount > 0) { $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL; } - if ($shouldProcessBatch) { - try { - foreach ($this->logs as $internalId => $projectLogs) { - $dbForProject = $getProjectDB($projectLogs['project']); + if (!$shouldProcessBatch) { + return new NoCommit(); + } - Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); - $audit = new Audit($dbForProject); + try { + foreach ($this->logs as $internalId => $projectLogs) { + $dbForProject = $getProjectDB($projectLogs['project']); - $audit->logBatch($projectLogs['logs']); - Console::success('Audit logs processed successfully'); + Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events'); + $audit = new Audit($dbForProject); - unset($this->logs[$internalId]); - } - } catch (Throwable $e) { - Console::error('Error processing audit logs: ' . $e->getMessage()); - } finally { - $this->lastTriggeredTime = time(); + $audit->logBatch($projectLogs['logs']); + Console::success('Audit logs processed successfully'); + + unset($this->logs[$internalId]); } + return new Commit(); + } catch (Throwable $e) { + Console::error('Error processing audit logs: ' . $e->getMessage()); + return new NoCommit(); + } finally { + $this->lastTriggeredTime = time(); } } } From 25b6d59e3071b6d223cb53c8e9453631d5f3ec43 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 12:54:19 +0200 Subject: [PATCH 40/54] Add internalId --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index b8cff64214..d1b143ec9b 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -60,11 +60,19 @@ class Base extends Queries 'type' => Database::VAR_STRING, 'array' => false, ]); + + $attributes[] = new Document([ + 'key' => '$internalId', + 'type' => Database::VAR_STRING, + 'array' => false, + ]); + $attributes[] = new Document([ 'key' => '$createdAt', 'type' => Database::VAR_DATETIME, 'array' => false, ]); + $attributes[] = new Document([ 'key' => '$updatedAt', 'type' => Database::VAR_DATETIME, From 42f72b4477ef951aa131429f4f513579b2a4cf46 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 13:33:36 +0200 Subject: [PATCH 41/54] Add internalId to orders --- .../Utopia/Database/Validator/Queries/Base.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index d1b143ec9b..85fbf43e66 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -61,12 +61,6 @@ class Base extends Queries 'array' => false, ]); - $attributes[] = new Document([ - 'key' => '$internalId', - 'type' => Database::VAR_STRING, - 'array' => false, - ]); - $attributes[] = new Document([ 'key' => '$createdAt', 'type' => Database::VAR_DATETIME, @@ -84,8 +78,15 @@ class Base extends Queries new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order($attributes), - ]; + new Order( + array_merge($attributes, [ + new Document([ + 'key' => '$internalId', + 'type' => Database::VAR_STRING, + 'array' => false, + ]) + ]) + )]; parent::__construct($validators); } From 0648de2046c3b7561d37877b3b06c8c811ddc06d Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 14:32:57 +0200 Subject: [PATCH 42/54] Revert Base --- .../Utopia/Database/Validator/Queries/Base.php | 13 ++----------- .../Databases/DatabasesCustomServerTest.php | 4 ++-- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index 85fbf43e66..b8cff64214 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -60,13 +60,11 @@ class Base extends Queries 'type' => Database::VAR_STRING, 'array' => false, ]); - $attributes[] = new Document([ 'key' => '$createdAt', 'type' => Database::VAR_DATETIME, 'array' => false, ]); - $attributes[] = new Document([ 'key' => '$updatedAt', 'type' => Database::VAR_DATETIME, @@ -78,15 +76,8 @@ class Base extends Queries new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order( - array_merge($attributes, [ - new Document([ - 'key' => '$internalId', - 'type' => Database::VAR_STRING, - 'array' => false, - ]) - ]) - )]; + new Order($attributes), + ]; parent::__construct($validators); } diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 57e0b93634..70f8bea4f2 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -120,7 +120,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'queries' => [ - Query::orderDesc()->toString(), + Query::orderDesc()->setAttribute('')->toString(), ], ]); @@ -453,7 +453,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'queries' => [ - Query::orderDesc()->toString(), + Query::orderDesc()->setAttribute('')->toString(), ], ]); From e11e7e6780cbf8a3dc2ac10c4302360b7d4963c5 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 17:33:02 +0200 Subject: [PATCH 43/54] Add internalId --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 8 +++++++- .../e2e/Services/Databases/DatabasesCustomServerTest.php | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index b8cff64214..a13314ab2f 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -71,12 +71,18 @@ class Base extends Queries 'array' => false, ]); + $internalId = new Document([ + 'key' => '$internalId', + 'type' => Database::VAR_STRING, + 'array' => false, + ]); + $validators = [ new Limit(), new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order($attributes), + new Order(array_merge($attributes, [$internalId])), ]; parent::__construct($validators); diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 70f8bea4f2..55cb17b9f8 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -123,7 +123,7 @@ class DatabasesCustomServerTest extends Scope Query::orderDesc()->setAttribute('')->toString(), ], ]); - + var_dump($databases); $this->assertEquals(2, $databases['body']['total']); $this->assertEquals($base[0]['$id'], $databases['body']['databases'][0]['$id']); $this->assertEquals($base[1]['$id'], $databases['body']['databases'][1]['$id']); From 5799ca1041b8f945c45285f557a35ef6b8633381 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 17:37:34 +0200 Subject: [PATCH 44/54] remove var_dump --- tests/e2e/Services/Databases/DatabasesCustomServerTest.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 55cb17b9f8..206567e789 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -120,10 +120,9 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'queries' => [ - Query::orderDesc()->setAttribute('')->toString(), + Query::orderDesc()->toString(), ], ]); - var_dump($databases); $this->assertEquals(2, $databases['body']['total']); $this->assertEquals($base[0]['$id'], $databases['body']['databases'][0]['$id']); $this->assertEquals($base[1]['$id'], $databases['body']['databases'][1]['$id']); @@ -453,7 +452,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'queries' => [ - Query::orderDesc()->setAttribute('')->toString(), + Query::orderDesc()->toString(), ], ]); From 2795505e8288074f8981edb90e5f3279bfaf9938 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 24 Mar 2025 17:38:01 +0200 Subject: [PATCH 45/54] line --- tests/e2e/Services/Databases/DatabasesCustomServerTest.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 206567e789..57e0b93634 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -123,6 +123,7 @@ class DatabasesCustomServerTest extends Scope Query::orderDesc()->toString(), ], ]); + $this->assertEquals(2, $databases['body']['total']); $this->assertEquals($base[0]['$id'], $databases['body']['databases'][0]['$id']); $this->assertEquals($base[1]['$id'], $databases['body']['databases'][1]['$id']); From c0cb4b26a3ebf73822f0abea8c6aeae9dd4d8875 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 15:51:47 +1300 Subject: [PATCH 46/54] Apply suggestions from code review --- src/Appwrite/Platform/Workers/Deletes.php | 35 ++++------------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 8deb227c78..0de79cd67e 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -305,9 +305,6 @@ class Deletes extends Action */ private function deleteExpiredTargets(Document $project, callable $getProjectDB): void { - /** - * todo: No index found for `expired` attribute - */ $this->deleteByGroup( 'targets', [ @@ -323,9 +320,6 @@ class Deletes extends Action private function deleteSessionTargets(Document $project, callable $getProjectDB, Document $session): void { - /** - * todo: No index found for `sessionInternalId` attribute - */ $this->deleteByGroup( 'targets', [ @@ -368,10 +362,6 @@ class Deletes extends Action $queries[] = Query::select(['$internalId', '$id', '$updatedAt']); $queries[] = Query::orderAsc(); - /** - * todo: No index on `resource`, `resourceType`, But it is fine since resource index is a strong index! - */ - $this->deleteByGroup( 'cache', $queries, @@ -446,7 +436,7 @@ class Deletes extends Action Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC + Query::orderDesc(), ], $dbForProject); if ($project->getId() !== 'console') { @@ -459,7 +449,7 @@ class Deletes extends Action Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), - Query::orderDesc(), // _key_period_time `period` ASC, `time `ASC + Query::orderDesc(), ], $dbForLogs); } } @@ -639,11 +629,7 @@ class Deletes extends Action Query::orderAsc() ], $dbForPlatform); - /** - * No projectInternalId in this collection - * todo: No index for `projectId` attribute - */ - // Delete Schedules () + // Delete Schedules $this->deleteByGroup('schedules', [ Query::equal('projectId', [$projectId]), Query::orderAsc() @@ -724,9 +710,6 @@ class Deletes extends Action ], $dbForProject); // Delete identities - /** - * todo: Remove Duplication index `_key_userInternalId` , But lets leave because of index length issue in unique index - */ $this->deleteByGroup('identities', [ Query::equal('userInternalId', [$userInternalId]), Query::orderAsc() @@ -799,7 +782,7 @@ class Deletes extends Action $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime), Query::orderDesc('timestamp'), - Query::orderAsc(), // KEY "_key_timestamp" ("timestamp" DESC), + Query::orderAsc(), ], $dbForPlatform); } @@ -820,7 +803,7 @@ class Deletes extends Action Query::select(['$internalId', '$id', '$updatedAt']), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), - Query::orderAsc(), // KEY "index-time" ("time" DESC) + Query::orderAsc(), ], $dbForProject); } catch (DatabaseException $e) { Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); @@ -845,7 +828,6 @@ class Deletes extends Action /** * Delete rules - * todo: No index for this query, drop _key_projectInternalId and create _key_projectInternalId, resourceInternalId, resourceType */ Console::info("Deleting rules for function " . $functionId); $this->deleteByGroup('rules', [ @@ -859,7 +841,6 @@ class Deletes extends Action /** * Delete Variables - * todo: No index for this query , drop _key_resourceInternalId and create new one with {resourceInternalId, resourceType} */ Console::info("Deleting variables for function " . $functionId); $this->deleteByGroup('variables', [ @@ -885,7 +866,6 @@ class Deletes extends Action /** * Delete builds - * todo: No index for `deploymentInternalId`, perhaps use deploymentId until fixed */ Console::info("Deleting builds for function " . $functionId); @@ -900,7 +880,6 @@ class Deletes extends Action /** * Delete Executions - * todo: No index for `functionInternalId` , perhaps use functionId until fixed */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ @@ -923,9 +902,6 @@ class Deletes extends Action $providerRepositoryId = $document->getAttribute('providerRepositoryId', ''); $projectInternalId = $document->getAttribute('projectInternalId', ''); - /** - * todo: add index to this query - */ $this->deleteByGroup('vcsComments', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::equal('projectInternalId', [$projectInternalId]), @@ -1025,7 +1001,6 @@ class Deletes extends Action /** * Delete builds - * todo: no index for `deploymentInternalId` Same as above index , no need to handle again... */ Console::info("Deleting builds for deployment " . $deploymentId); From a20140eb36dfc7238324cf40989bab9842e89abf Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 15:52:54 +1300 Subject: [PATCH 47/54] Update src/Appwrite/Platform/Workers/Deletes.php --- src/Appwrite/Platform/Workers/Deletes.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 0de79cd67e..93dbaf501c 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -851,7 +851,6 @@ class Deletes extends Action /** * Delete Deployments - * todo: No index for `resourceInternalId`, perhaps use resourceId until fixed */ Console::info("Deleting deployments for function " . $functionId); From c9c9db564053523be084bb5a97832221eb6ee28c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 15:52:59 +1300 Subject: [PATCH 48/54] Update src/Appwrite/Platform/Workers/Deletes.php --- src/Appwrite/Platform/Workers/Deletes.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 93dbaf501c..6dff127784 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -889,7 +889,6 @@ class Deletes extends Action /** * Delete VCS Repositories and VCS Comments - * todo: no index for this query */ Console::info("Deleting VCS repositories and comments linked to function " . $functionId); $this->deleteByGroup('repositories', [ From 6bcdc8c769185b77dc9689ed691e53a4071b1453 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 16:01:27 +1300 Subject: [PATCH 49/54] Update src/Appwrite/Utopia/Database/Validator/Queries/Base.php --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index a13314ab2f..664476af36 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -82,7 +82,7 @@ class Base extends Queries new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order(array_merge($attributes, [$internalId])), + new Order([...$attributes, $internalId])), ]; parent::__construct($validators); From 74d9a3c181ebf09b9a7b6da710b6431dcaac4097 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 25 Mar 2025 16:19:18 +1300 Subject: [PATCH 50/54] Fix syntax --- src/Appwrite/Utopia/Database/Validator/Queries/Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index 664476af36..e8eafba5a0 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -82,7 +82,7 @@ class Base extends Queries new Offset(), new Cursor(), new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES), - new Order([...$attributes, $internalId])), + new Order([...$attributes, $internalId]), ]; parent::__construct($validators); From d89cdbcf5a52717b7dc7e48b88020c2604502f0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 25 Mar 2025 21:34:09 +0100 Subject: [PATCH 51/54] Fix schedules region filtering --- src/Appwrite/Migration/Version/V19.php | 3 +-- src/Appwrite/Platform/Tasks/ScheduleBase.php | 9 ++++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 18234ebdc4..7adf5b85be 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -10,7 +10,6 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception; use Utopia\Database\Query; -use Utopia\System\System; class V19 extends Migration { @@ -731,7 +730,7 @@ class V19 extends Migration if (empty($document->getAttribute('scheduleId', null))) { $schedule = $this->consoleDB->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), // Todo replace with projects region + 'region' => $project->getAttribute('region'), // Todo replace with projects region 'resourceType' => 'function', 'resourceId' => $document->getId(), 'resourceInternalId' => $document->getInternalId(), diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index dad2db0d9a..324b88d07b 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -103,8 +103,15 @@ abstract class ScheduleBase extends Action $paginationQueries[] = Query::cursorAfter($latestDocument); } + // Temporarly accepting both 'fra' and 'default' + // When all migrated, only use _APP_REGION with 'default' as default value + $regions = [System::getEnv('_APP_REGION', 'default')]; + if (!in_array('default', $regions)) { + $regions[] = 'default'; + } + $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', $regions), Query::equal('resourceType', [static::getSupportedResource()]), Query::equal('active', [true]), ])); From f25143b240177b141c487048d5e9de03a85e7029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 25 Mar 2025 21:52:02 +0100 Subject: [PATCH 52/54] Support fra-region on cloud; temporary fix --- src/Appwrite/Migration/Version/V19.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 9 ++++++++- src/Appwrite/Platform/Workers/Deletes.php | 9 ++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 7adf5b85be..4415003bfd 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -730,7 +730,7 @@ class V19 extends Migration if (empty($document->getAttribute('scheduleId', null))) { $schedule = $this->consoleDB->createDocument('schedules', new Document([ - 'region' => $project->getAttribute('region'), // Todo replace with projects region + 'region' => $project->getAttribute('region'), 'resourceType' => 'function', 'resourceId' => $document->getId(), 'resourceInternalId' => $document->getInternalId(), diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index 324b88d07b..a3c36cb96e 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -160,8 +160,15 @@ abstract class ScheduleBase extends Action $paginationQueries[] = Query::cursorAfter($latestDocument); } + // Temporarly accepting both 'fra' and 'default' + // When all migrated, only use _APP_REGION with 'default' as default value + $regions = [System::getEnv('_APP_REGION', 'default')]; + if (!in_array('default', $regions)) { + $regions[] = 'default'; + } + $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', $regions), Query::equal('resourceType', [static::getSupportedResource()]), Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate), ])); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 6dff127784..8eaa08ce65 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -180,10 +180,17 @@ class Deletes extends Action */ private function deleteSchedules(Database $dbForPlatform, callable $getProjectDB, string $datetime): void { + // Temporarly accepting both 'fra' and 'default' + // When all migrated, only use _APP_REGION with 'default' as default value + $regions = [System::getEnv('_APP_REGION', 'default')]; + if (!in_array('default', $regions)) { + $regions[] = 'default'; + } + $this->listByGroup( 'schedules', [ - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::equal('region', $regions), Query::lessThanEqual('resourceUpdatedAt', $datetime), Query::equal('active', [false]), ], From a461d5bf7281841ccbbe1c4a27d31e850b8dc2ad Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 26 Mar 2025 09:20:44 +0200 Subject: [PATCH 53/54] Deletes fix selects --- src/Appwrite/Platform/Workers/Deletes.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 8eaa08ce65..2cc1eaf9af 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -31,6 +31,8 @@ use Utopia\System\System; class Deletes extends Action { + protected array $selects = ['$internalId', '$id', '$collection', '$updatedAt']; + public static function getName(): string { return 'deletes'; @@ -366,7 +368,7 @@ class Deletes extends Action $queries[] = Query::equal('resourceType', [$resourceType]); } - $queries[] = Query::select(['$internalId', '$id', '$updatedAt']); + $queries[] = Query::select($this->selects); $queries[] = Query::orderAsc(); $this->deleteByGroup( @@ -403,7 +405,7 @@ class Deletes extends Action ); $queries = [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::lessThan('accessedAt', $datetime), Query::orderDesc('accessedAt'), Query::orderDesc(), @@ -439,7 +441,7 @@ class Deletes extends Action // Delete Usage stats from projectDB $this->deleteByGroup('stats', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -452,7 +454,7 @@ class Deletes extends Action // Delete Usage stats from logsDB $this->deleteByGroup('stats', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::equal('period', ['1h']), Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::orderDesc('time'), @@ -749,7 +751,7 @@ class Deletes extends Action // Delete Executions $this->deleteByGroup('executions', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::lessThan('$createdAt', $datetime), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -770,7 +772,7 @@ class Deletes extends Action // Delete Sessions $this->deleteByGroup('sessions', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::lessThan('$createdAt', $expired), Query::orderDesc('$createdAt'), Query::orderDesc(), @@ -807,7 +809,7 @@ class Deletes extends Action try { $this->deleteByGroup(Audit::COLLECTION, [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::lessThan('time', $auditRetention), Query::orderDesc('time'), Query::orderAsc(), @@ -889,7 +891,7 @@ class Deletes extends Action */ Console::info("Deleting executions for function " . $functionId); $this->deleteByGroup('executions', [ - Query::select(['$internalId', '$id', '$updatedAt']), + Query::select($this->selects), Query::equal('functionInternalId', [$functionInternalId]), Query::orderAsc() ], $dbForProject); From 424a3575034c31b34b6c2c80a24fb953338642bd Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 26 Mar 2025 10:02:30 +0200 Subject: [PATCH 54/54] Add $permissions --- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 2cc1eaf9af..5033822241 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -31,7 +31,7 @@ use Utopia\System\System; class Deletes extends Action { - protected array $selects = ['$internalId', '$id', '$collection', '$updatedAt']; + protected array $selects = ['$internalId', '$id', '$collection', '$permissions', '$updatedAt']; public static function getName(): string {