diff --git a/app/config/regions.php b/app/config/regions.php index 61029a9996..05e04930fe 100644 --- a/app/config/regions.php +++ b/app/config/regions.php @@ -3,72 +3,8 @@ return [ 'default' => [ '$id' => 'default', - 'name' => 'Frankfurt', + 'name' => 'default', 'disabled' => false, - 'flag' => 'de', - 'default' => true, - ], - 'fra' => [ - '$id' => 'fra', - 'name' => 'Frankfurt', - 'disabled' => false, - 'flag' => 'de', - 'default' => true, - ], - 'nyc' => [ - '$id' => 'nyc', - 'name' => 'New York', - 'disabled' => true, - 'flag' => 'us', - 'default' => true, - ], - 'sfo' => [ - '$id' => 'sfo', - 'name' => 'San Francisco', - 'disabled' => true, - 'flag' => 'us', - 'default' => true, - ], - 'blr' => [ - '$id' => 'blr', - 'name' => 'Bengaluru', - 'disabled' => true, - 'flag' => 'in', - 'default' => true, - ], - 'lon' => [ - '$id' => 'lon', - 'name' => 'London', - 'disabled' => true, - 'flag' => 'gb', - 'default' => true, - ], - 'ams' => [ - '$id' => 'ams', - 'name' => 'Amsterdam', - 'disabled' => true, - 'flag' => 'nl', - 'default' => true, - ], - 'sgp' => [ - '$id' => 'sgp', - 'name' => 'Singapore', - 'disabled' => true, - 'flag' => 'sg', - 'default' => true, - ], - 'tor' => [ - '$id' => 'tor', - 'name' => 'Toronto', - 'disabled' => true, - 'flag' => 'ca', - 'default' => true, - ], - 'syd' => [ - '$id' => 'syd', - 'name' => 'Sydney', - 'disabled' => true, - 'flag' => 'au', 'default' => true, ], ]; diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index d8d496089c..0c37e1a765 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3355,7 +3355,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection @@ -3520,7 +3520,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -3661,7 +3661,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); $response->addHeader('X-Debug-Operations', $operations); @@ -3959,7 +3959,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); $response->addHeader('X-Debug-Operations', $operations); diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 9393f1fbfe..178266db60 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -2936,7 +2936,7 @@ App::post('/v1/messaging/messages/email') break; case MessageStatus::SCHEDULED: $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -3058,7 +3058,7 @@ App::post('/v1/messaging/messages/sms') break; case MessageStatus::SCHEDULED: $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -3275,7 +3275,7 @@ App::post('/v1/messaging/messages/push') break; case MessageStatus::SCHEDULED: $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -3661,7 +3661,7 @@ App::patch('/v1/messaging/messages/email/:messageId') if (\is_null($currentScheduledAt) && !\is_null($scheduledAt)) { $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -3862,7 +3862,7 @@ App::patch('/v1/messaging/messages/sms/:messageId') if (\is_null($currentScheduledAt) && !\is_null($scheduledAt)) { $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), @@ -4035,7 +4035,7 @@ App::patch('/v1/messaging/messages/push/:messageId') if (\is_null($currentScheduledAt) && !\is_null($scheduledAt)) { $schedule = $dbForPlatform->createDocument('schedules', new Document([ - 'region' => System::getEnv('_APP_REGION', 'default'), + 'region' => $project->getAttribute('region'), 'resourceType' => 'message', 'resourceId' => $message->getId(), 'resourceInternalId' => $message->getInternalId(), diff --git a/app/init/constants.php b/app/init/constants.php index d46d3ed79c..e5e7b5900d 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -238,6 +238,7 @@ const METRIC_TOPICS = 'topics'; const METRIC_TARGETS = 'targets'; const METRIC_PROVIDER_TYPE_TARGETS = '{providerType}.targets'; const METRIC_KEYS = 'keys'; +const METRIC_DOMAINS = 'domains'; const METRIC_RESOURCE_TYPE_ID_BUILDS = '{resourceType}.{resourceInternalId}.builds'; const METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE = '{resourceType}.{resourceInternalId}.builds.storage'; const METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments'; diff --git a/app/init/registers.php b/app/init/registers.php index 9c90eb54d2..1ebcbc1691 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -217,7 +217,7 @@ $register->set('pools', function () { return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, + PDO::ATTR_PERSISTENT => false, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_EMULATE_PREPARES => true, PDO::ATTR_STRINGIFY_FETCHES => true diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 12c450bf04..523fee911a 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -165,7 +165,6 @@ $image = $this->getParam('image', ''); - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_ASSISTANT_OPENAI_API_KEY - appwrite-console: <<: *x-logging container_name: appwrite-console diff --git a/composer.json b/composer.json index c25efaca10..66b18c7a7b 100644 --- a/composer.json +++ b/composer.json @@ -62,10 +62,10 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.16.*", - "utopia-php/migration": "0.6.*", + "utopia-php/migration": "0.8.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "0.5.*", + "utopia-php/pools": "0.7.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.9.*", "utopia-php/registry": "0.5.*", diff --git a/docker-compose.yml b/docker-compose.yml index 4d6dfaf682..64aee1c2b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -764,7 +764,7 @@ services: - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES - _APP_STATS_RESOURCES_INTERVAL - + appwrite-worker-stats-resources: entrypoint: worker-stats-resources <<: *x-logging @@ -795,7 +795,7 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - + appwrite-worker-stats-usage: entrypoint: worker-stats-usage <<: *x-logging @@ -858,7 +858,7 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL - _APP_DATABASE_SHARED_TABLES - _APP_STATS_USAGE_DUAL_WRITING_DBS - + appwrite-task-scheduler-functions: entrypoint: schedule-functions <<: *x-logging diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index 1534dde397..198b701044 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -47,13 +47,15 @@ class Maintenance extends Action Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds"); - $this->foreachProject($dbForPlatform, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) { + $dbForPlatform->foreach('projects', function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) { $queueForDeletes ->setType(DELETE_TYPE_MAINTENANCE) ->setProject($project) ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) ->trigger(); - }); + }, [ + Query::limit(100), + ]); $queueForDeletes ->setType(DELETE_TYPE_MAINTENANCE) @@ -68,33 +70,6 @@ class Maintenance extends Action }, $interval, $delay); } - protected function foreachProject(Database $dbForPlatform, callable $callback): void - { - // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document - $count = 0; - $chunk = 0; - $limit = 50; - $sum = $limit; - $executionStart = \microtime(true); - - while ($sum === $limit) { - $projects = $dbForPlatform->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]); - - $chunk++; - - /** @var string[] $projectIds */ - $sum = count($projects); - - foreach ($projects as $project) { - $callback($project); - $count++; - } - } - - $executionEnd = \microtime(true); - Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds"); - } - private function notifyDeleteConnections(Delete $queueForDeletes): void { $queueForDeletes diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index f683c268a0..6b8543f299 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -34,6 +34,9 @@ class Migrations extends Action protected Document $project; + /** + * @var callable + */ protected $logError; public static function getName(): string diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index ef0b24b5c3..dae2967264 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -114,6 +114,13 @@ class StatsResources extends Action $keys = $dbForPlatform->count('keys', [ Query::equal('projectInternalId', [$project->getInternalId()]) ]); + + $domains = $dbForPlatform->count('rules', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::equal('owner', ['']), + ]); + + $databases = $dbForProject->count('databases'); $buckets = $dbForProject->count('buckets'); $users = $dbForProject->count('users'); @@ -162,6 +169,7 @@ class StatsResources extends Action METRIC_PROVIDERS => $providers, METRIC_TOPICS => $topics, METRIC_KEYS => $keys, + METRIC_DOMAINS => $domains, METRIC_TARGETS => $targets, str_replace('{providerType}', MESSAGE_TYPE_EMAIL, METRIC_PROVIDER_TYPE_TARGETS) => $emailTargets, str_replace('{providerType}', MESSAGE_TYPE_PUSH, METRIC_PROVIDER_TYPE_TARGETS) => $pushTargets, diff --git a/tests/e2e/Scopes/ProjectCustom.php b/tests/e2e/Scopes/ProjectCustom.php index f8f779027b..a1616f2f74 100644 --- a/tests/e2e/Scopes/ProjectCustom.php +++ b/tests/e2e/Scopes/ProjectCustom.php @@ -4,6 +4,7 @@ namespace Tests\E2E\Scopes; use Tests\E2E\Client; use Utopia\Database\Helpers\ID; +use Utopia\System\System; trait ProjectCustom { @@ -42,7 +43,7 @@ trait ProjectCustom 'x-appwrite-project' => 'console', ], [ 'projectId' => ID::unique(), - 'region' => 'default', + 'region' => System::getEnv('_APP_REGION', 'default'), 'name' => 'Demo Project', 'teamId' => $team['body']['$id'], 'description' => 'Demo Project Description', diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 86381fa221..79b20571a5 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -8,6 +8,7 @@ use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideClient; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Role; +use Utopia\System\System; class FunctionsCustomClientTest extends Scope { @@ -109,7 +110,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('default', $output['APPWRITE_REGION']); + $this->assertEquals(System::getEnv('_APP_REGION', 'default'), $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); @@ -219,7 +220,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('PHP', $output['APPWRITE_FUNCTION_RUNTIME_NAME']); $this->assertEquals('8.0', $output['APPWRITE_FUNCTION_RUNTIME_VERSION']); $this->assertEquals(APP_VERSION_STABLE, $output['APPWRITE_VERSION']); - $this->assertEquals('default', $output['APPWRITE_REGION']); + $this->assertEquals(System::getEnv('_APP_REGION', 'default'), $output['APPWRITE_REGION']); $this->assertEquals('', $output['APPWRITE_FUNCTION_EVENT']); $this->assertEquals('foobar', $output['APPWRITE_FUNCTION_DATA']); $this->assertEquals($this->getUser()['$id'], $output['APPWRITE_FUNCTION_USER_ID']); diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index ed9171e46a..afd1189496 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -14,6 +14,7 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; +use Utopia\System\System; class ProjectsConsoleClientTest extends Scope { @@ -49,7 +50,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default', + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -89,7 +90,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => '', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(400, $response['headers']['status-code']); @@ -100,7 +101,7 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders()), [ 'projectId' => ID::unique(), 'name' => 'Project Test', - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(401, $response['headers']['status-code']); @@ -129,7 +130,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => $projectId, 'name' => 'Project Duplicate', 'teamId' => $teamId, - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(409, $response['headers']['status-code']); @@ -178,7 +179,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Team 1 Project', 'teamId' => $team1, - 'region' => 'default', + 'region' => System::getEnv('_APP_REGION', 'default'), ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -277,7 +278,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test 2', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $response['headers']['status-code']); @@ -2042,7 +2043,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -2135,7 +2136,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Project Test', 'teamId' => $team['body']['$id'], - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3749,7 +3750,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project', 'teamId' => $teamId, - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $this->assertEquals(201, $project['headers']['status-code']); @@ -3820,7 +3821,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 1', 'teamId' => $teamId, - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $project2 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([ @@ -3830,7 +3831,7 @@ class ProjectsConsoleClientTest extends Scope 'projectId' => ID::unique(), 'name' => 'Amazing Project 2', 'teamId' => $teamId, - 'region' => 'default' + 'region' => System::getEnv('_APP_REGION', 'default') ]); $project1Id = $project1['body']['$id'];