From 91c818c6386a82c7403be54b09fd657de9bbabcd Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 5 Jan 2023 19:35:24 +0200 Subject: [PATCH] deletes-worker --- app/init.php | 4 +- app/workers/deletes.php | 1122 +++++++++---------- docker-compose.yml | 5 +- src/Appwrite/Platform/Tasks/EdgeSync.php | 2 +- src/Appwrite/Platform/Tasks/Maintenance.php | 4 +- 5 files changed, 570 insertions(+), 567 deletions(-) diff --git a/app/init.php b/app/init.php index acfc16ffeb..090f51d39f 100644 --- a/app/init.php +++ b/app/init.php @@ -41,7 +41,6 @@ use Appwrite\URL\URL as AppwriteURL; use Appwrite\Usage\Stats; use Utopia\App; use Utopia\Queue\Client; -use Utopia\Queue\Server; use Utopia\Validator\Range; use Utopia\Validator\WhiteList; use Utopia\Database\ID; @@ -860,6 +859,9 @@ App::setResource('messaging', fn() => new Phone()); App::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); +App::setResource('queueForDeletes', function (Connection $queue) { + return new Delete($queue); +}, ['queue']); App::setResource('queueForFunctions', function (Connection $queue) { return new Func($queue); }, ['queue']); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 6bf29cc3ff..a17c7f4229 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -1,5 +1,7 @@ job() + ->inject('message') + ->action(function (Message $message) { + $payload = $message->getPayload() ?? []; -class DeletesV1 extends Worker -{ - public function getName(): string - { - return "deletes"; - } + if (empty($payload)) { + throw new Exception('Missing payload'); + } - public function init(): void - { - } - - public function run(): void - { - $project = new Document($this->args['project'] ?? []); - $type = $this->args['type'] ?? ''; + $project = new Document($payload['project'] ?? []); + $type = $payload['type'] ?? ''; switch (strval($type)) { case DELETE_TYPE_DOCUMENT: - $document = new Document($this->args['document'] ?? []); + $document = new Document($payload['document'] ?? []); switch ($document->getCollection()) { case DELETE_TYPE_DATABASES: - $this->deleteDatabase($document, $project); + deleteDatabase($document, $project); break; case DELETE_TYPE_COLLECTIONS: - $this->deleteCollection($document, $project); + deleteCollection($document, $project); break; case DELETE_TYPE_PROJECTS: - $this->deleteProject($document); + deleteProject($document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($document, $project); + deleteFunction($document, $project); break; case DELETE_TYPE_DEPLOYMENTS: - $this->deleteDeployment($document, $project); + deleteDeployment($document, $project); break; case DELETE_TYPE_USERS: - $this->deleteUser($document, $project); + deleteUser($document, $project); break; case DELETE_TYPE_TEAMS: - $this->deleteMemberships($document, $project); + deleteMemberships($document, $project); break; case DELETE_TYPE_BUCKETS: - $this->deleteBucket($document, $project); + deleteBucket($document, $project); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -72,664 +69,663 @@ class DeletesV1 extends Worker break; case DELETE_TYPE_EXECUTIONS: - $this->deleteExecutionLogs($this->args['datetime']); + deleteExecutionLogs($payload['datetime']); break; case DELETE_TYPE_AUDIT: - $datetime = $this->args['datetime'] ?? null; + $datetime = $payload['datetime'] ?? null; if (!empty($datetime)) { - $this->deleteAuditLogs($datetime); + deleteAuditLogs($datetime); } - $document = new Document($this->args['document'] ?? []); + $document = new Document($payload['document'] ?? []); if (!$document->isEmpty()) { - $this->deleteAuditLogsByResource('document/' . $document->getId(), $project); + deleteAuditLogsByResource('document/' . $document->getId(), $project); } break; case DELETE_TYPE_ABUSE: - $this->deleteAbuseLogs($this->args['datetime']); + deleteAbuseLogs($payload['datetime']); break; case DELETE_TYPE_REALTIME: - $this->deleteRealtimeUsage($this->args['datetime']); + deleteRealtimeUsage($payload['datetime']); break; case DELETE_TYPE_SESSIONS: - $this->deleteExpiredSessions(); + deleteExpiredSessions(); break; case DELETE_TYPE_CERTIFICATES: - $document = new Document($this->args['document']); - $this->deleteCertificates($document); + $document = new Document($payload['document']); + deleteCertificates($document); break; case DELETE_TYPE_USAGE: - $this->deleteUsageStats($this->args['hourlyUsageRetentionDatetime']); + deleteUsageStats($payload['hourlyUsageRetentionDatetime']); break; case DELETE_TYPE_CACHE_BY_RESOURCE: - $this->deleteCacheByResource($this->args['resource']); + deleteCacheByResource($payload['resource']); break; case DELETE_TYPE_CACHE_BY_TIMESTAMP: - $this->deleteCacheByDate(); + deleteCacheByDate($payload); break; case DELETE_TYPE_SCHEDULES: - $this->deleteSchedules($this->args['datetime']); + deleteSchedules($payload['datetime']); break; default: Console::error('No delete operation for type: ' . $type); break; } - } + }); - public function shutdown(): void - { - } - /** - * @throws Exception - */ - protected function deleteSchedules(string $datetime): void - { - $this->listByGroup( - 'schedules', - [ - Query::equal('region', [App::getEnv('_APP_REGION', 'default')]), - Query::equal('resourceType', ['function']), - Query::lessThanEqual('resourceUpdatedAt', $datetime), - Query::equal('active', [false]), - ], - $this->getConsoleDB(), - function (Document $document) { - $project = $this->getConsoleDB()->getDocument('projects', $document->getAttribute('projectId')); +/** + * @throws Exception + */ +function deleteSchedules(string $datetime): void +{ + listByGroup( + 'schedules', + [ + Query::equal('region', [App::getEnv('_APP_REGION', 'default')]), + Query::equal('resourceType', ['function']), + Query::lessThanEqual('resourceUpdatedAt', $datetime), + Query::equal('active', [false]), + ], + getConsoleDB(), + function (Document $document) { + $project = getConsoleDB()->getDocument('projects', $document->getAttribute('projectId')); - if ($project->isEmpty()) { - Console::warning('Unable to delete schedule for function ' . $document->getAttribute('resourceId')); - return; - } + if ($project->isEmpty()) { + Console::warning('Unable to delete schedule for function ' . $document->getAttribute('resourceId')); + return; + } - $function = $this->getProjectDB($project)->getDocument('functions', $document->getAttribute('resourceId')); + $function = getProjectDB($project)->getDocument('functions', $document->getAttribute('resourceId')); - if ($function->isEmpty()) { - $this->getConsoleDB()->deleteDocument('schedules', $document->getId()); - Console::success('Deleting schedule for function ' . $document->getAttribute('resourceId')); + if ($function->isEmpty()) { + getConsoleDB()->deleteDocument('schedules', $document->getId()); + Console::success('Deleting schedule for function ' . $document->getAttribute('resourceId')); + } + } + ); +} + +/** + * @param string $resource + */ +function deleteCacheByResource(string $resource): void +{ + deleteCacheFiles([ + Query::equal('resource', [$resource]), + ]); +} + +function deleteCacheByDate($payload): void +{ + deleteCacheFiles([ + Query::lessThan('accessedAt', $payload['datetime']), + ]); +} + +function deleteCacheFiles($query): void +{ + deleteForProjectIds(function (Document $project) use ($query) { + + $projectId = $project->getId(); + $dbForProject = getProjectDB($project); + $cache = new Cache( + new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) + ); + + deleteByGroup( + 'cache', + $query, + $dbForProject, + function (Document $document) use ($cache, $projectId) { + $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); + + if ($cache->purge($document->getId())) { + Console::success('Deleting cache file: ' . $path); + } else { + Console::error('Failed to delete cache file: ' . $path); } } ); - } + }); +} - /** - * @param string $resource - */ - protected function deleteCacheByResource(string $resource): void - { - $this->deleteCacheFiles([ - Query::equal('resource', [$resource]), - ]); - } +/** + * @param Document $document database document + * @param Document $projectId + */ +function deleteDatabase(Document $document, Document $project): void +{ + $databaseId = $document->getId(); + $projectId = $project->getId(); - protected function deleteCacheByDate(): void - { - $this->deleteCacheFiles([ - Query::lessThan('accessedAt', $this->args['datetime']), - ]); - } + $dbForProject = getProjectDB($project); - protected function deleteCacheFiles($query): void - { - $this->deleteForProjectIds(function (Document $project) use ($query) { + deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($project) { + deleteCollection($document, $project); + }); - $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $cache = new Cache( - new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) - ); + $dbForProject->deleteCollection('database_' . $document->getInternalId()); - $this->deleteByGroup( - 'cache', - $query, - $dbForProject, - function (Document $document) use ($cache, $projectId) { - $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); + deleteAuditLogsByResource('database/' . $databaseId, $project); +} - if ($cache->purge($document->getId())) { - Console::success('Deleting cache file: ' . $path); - } else { - Console::error('Failed to delete cache file: ' . $path); - } - } - ); - }); - } +/** + * @param Document $document teams document + * @param Document $project + */ +function deleteCollection(Document $document, Document $project): void +{ + $collectionId = $document->getId(); + $databaseId = $document->getAttribute('databaseId'); + $databaseInternalId = $document->getAttribute('databaseInternalId'); - /** - * @param Document $document database document - * @param Document $projectId - */ - protected function deleteDatabase(Document $document, Document $project): void - { - $databaseId = $document->getId(); - $projectId = $project->getId(); + $dbForProject = getProjectDB($project); - $dbForProject = $this->getProjectDB($project); + $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId()); - $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($project) { - $this->deleteCollection($document, $project); - }); + deleteByGroup('attributes', [ + Query::equal('databaseId', [$databaseId]), + Query::equal('collectionId', [$collectionId]) + ], $dbForProject); - $dbForProject->deleteCollection('database_' . $document->getInternalId()); + deleteByGroup('indexes', [ + Query::equal('databaseId', [$databaseId]), + Query::equal('collectionId', [$collectionId]) + ], $dbForProject); - $this->deleteAuditLogsByResource('database/' . $databaseId, $project); - } + deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project); +} - /** - * @param Document $document teams document - * @param Document $project - */ - protected function deleteCollection(Document $document, Document $project): void - { - $collectionId = $document->getId(); - $databaseId = $document->getAttribute('databaseId'); - $databaseInternalId = $document->getAttribute('databaseInternalId'); - - $dbForProject = $this->getProjectDB($project); - - $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId()); - - $this->deleteByGroup('attributes', [ - Query::equal('databaseId', [$databaseId]), - Query::equal('collectionId', [$collectionId]) +/** + * @param string $hourlyUsageRetentionDatetime + */ +function deleteUsageStats(string $hourlyUsageRetentionDatetime) +{ + deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) { + $dbForProject = getProjectDB($project); + // Delete Usage stats + deleteByGroup('stats', [ + Query::lessThan('time', $hourlyUsageRetentionDatetime), + Query::equal('period', ['1h']), ], $dbForProject); + }); +} - $this->deleteByGroup('indexes', [ - Query::equal('databaseId', [$databaseId]), - Query::equal('collectionId', [$collectionId]) - ], $dbForProject); +/** + * @param Document $document teams document + * @param Document $project + */ +function deleteMemberships(Document $document, Document $project): void +{ + $teamId = $document->getAttribute('teamId', ''); - $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project); - } + // Delete Memberships + deleteByGroup('memberships', [ + Query::equal('teamId', [$teamId]) + ], getProjectDB($project)); +} - /** - * @param string $hourlyUsageRetentionDatetime - */ - protected function deleteUsageStats(string $hourlyUsageRetentionDatetime) - { - $this->deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) { - $dbForProject = $this->getProjectDB($project); - // Delete Usage stats - $this->deleteByGroup('stats', [ - Query::lessThan('time', $hourlyUsageRetentionDatetime), - Query::equal('period', ['1h']), - ], $dbForProject); - }); - } +/** + * @param Document $document project document + */ +function deleteProject(Document $document): void +{ + $projectId = $document->getId(); - /** - * @param Document $document teams document - * @param Document $project - */ - protected function deleteMemberships(Document $document, Document $project): void - { - $teamId = $document->getAttribute('teamId', ''); + // Delete all DBs + getProjectDB($document)->delete($projectId); - // Delete Memberships - $this->deleteByGroup('memberships', [ - Query::equal('teamId', [$teamId]) - ], $this->getProjectDB($project)); - } + // Delete all storage directories + $uploads = getFilesDevice($document->getId()); + $cache = new Local(APP_STORAGE_CACHE . '/app-' . $document->getId()); - /** - * @param Document $document project document - */ - protected function deleteProject(Document $document): void - { - $projectId = $document->getId(); + $uploads->delete($uploads->getRoot(), true); + $cache->delete($cache->getRoot(), true); +} - // Delete all DBs - $this->getProjectDB($document)->delete($projectId); +/** + * @param Document $document user document + * @param Document $project + */ +function deleteUser(Document $document, Document $project): void +{ + $userId = $document->getId(); - // Delete all storage directories - $uploads = $this->getFilesDevice($document->getId()); - $cache = new Local(APP_STORAGE_CACHE . '/app-' . $document->getId()); + $dbForProject = getProjectDB($project); - $uploads->delete($uploads->getRoot(), true); - $cache->delete($cache->getRoot(), true); - } + // Delete all sessions of this user from the sessions table and update the sessions field of the user record + deleteByGroup('sessions', [ + Query::equal('userId', [$userId]) + ], $dbForProject); - /** - * @param Document $document user document - * @param Document $project - */ - protected function deleteUser(Document $document, Document $project): void - { - $userId = $document->getId(); + $dbForProject->deleteCachedDocument('users', $userId); - $dbForProject = $this->getProjectDB($project); - - // Delete all sessions of this user from the sessions table and update the sessions field of the user record - $this->deleteByGroup('sessions', [ - Query::equal('userId', [$userId]) - ], $dbForProject); - - $dbForProject->deleteCachedDocument('users', $userId); - - // Delete Memberships and decrement team membership counts - $this->deleteByGroup('memberships', [ - Query::equal('userId', [$userId]) - ], $dbForProject, function (Document $document) use ($dbForProject) { - if ($document->getAttribute('confirm')) { // Count only confirmed members - $teamId = $document->getAttribute('teamId'); - $team = $dbForProject->getDocument('teams', $teamId); - if (!$team->isEmpty()) { - $team = $dbForProject->updateDocument( - 'teams', - $teamId, - // Ensure that total >= 0 - $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0)) - ); - } + // Delete Memberships and decrement team membership counts + deleteByGroup('memberships', [ + Query::equal('userId', [$userId]) + ], $dbForProject, function (Document $document) use ($dbForProject) { + if ($document->getAttribute('confirm')) { // Count only confirmed members + $teamId = $document->getAttribute('teamId'); + $team = $dbForProject->getDocument('teams', $teamId); + if (!$team->isEmpty()) { + $team = $dbForProject->updateDocument( + 'teams', + $teamId, + // Ensure that total >= 0 + $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0)) + ); } - }); - - // Delete tokens - $this->deleteByGroup('tokens', [ - Query::equal('userId', [$userId]) - ], $dbForProject); - } - - /** - * @param string $datetime - */ - protected function deleteExecutionLogs(string $datetime): void - { - $this->deleteForProjectIds(function (Document $project) use ($datetime) { - $dbForProject = $this->getProjectDB($project); - // Delete Executions - $this->deleteByGroup('executions', [ - Query::lessThan('$createdAt', $datetime) - ], $dbForProject); - }); - } - - protected function deleteExpiredSessions(): void - { - $consoleDB = $this->getConsoleDB(); - - $this->deleteForProjectIds(function (Document $project) use ($consoleDB) { - $dbForProject = $this->getProjectDB($project); - - $project = $consoleDB->getDocument('projects', $project->getId()); - $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; - $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration); - - // Delete Sessions - $this->deleteByGroup('sessions', [ - Query::lessThan('$createdAt', $expired) - ], $dbForProject); - }); - } - - /** - * @param string $datetime - */ - protected function deleteRealtimeUsage(string $datetime): void - { - $this->deleteForProjectIds(function (Document $project) use ($datetime) { - $dbForProject = $this->getProjectDB($project); - // Delete Dead Realtime Logs - $this->deleteByGroup('realtime', [ - Query::lessThan('timestamp', $datetime) - ], $dbForProject); - }); - } - - /** - * @param string $datetime - * @throws Exception - */ - protected function deleteAbuseLogs(string $datetime): void - { - if (empty($datetime)) { - throw new Exception('Failed to delete audit logs. No datetime provided'); } + }); - $this->deleteForProjectIds(function (Document $project) use ($datetime) { - $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $timeLimit = new TimeLimit("", 0, 1, $dbForProject); - $abuse = new Abuse($timeLimit); - $status = $abuse->cleanup($datetime); - if (!$status) { - throw new Exception('Failed to delete Abuse logs for project ' . $projectId); - } - }); - } + // Delete tokens + deleteByGroup('tokens', [ + Query::equal('userId', [$userId]) + ], $dbForProject); +} - /** - * @param string $datetime - * @throws Exception - */ - protected function deleteAuditLogs(string $datetime): void - { - if (empty($datetime)) { - throw new Exception('Failed to delete audit logs. No datetime provided'); - } - - $this->deleteForProjectIds(function (Document $project) use ($datetime) { - $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $audit = new Audit($dbForProject); - $status = $audit->cleanup($datetime); - if (!$status) { - throw new Exception('Failed to delete Audit logs for project' . $projectId); - } - }); - } - - /** - * @param string $resource - * @param Document $project - */ - protected function deleteAuditLogsByResource(string $resource, Document $project): void - { - $dbForProject = $this->getProjectDB($project); - - $this->deleteByGroup(Audit::COLLECTION, [ - Query::equal('resource', [$resource]) +/** + * @param string $datetime + */ +function deleteExecutionLogs(string $datetime): void +{ + deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = getProjectDB($project); + // Delete Executions + deleteByGroup('executions', [ + Query::lessThan('$createdAt', $datetime) ], $dbForProject); + }); +} + +function deleteExpiredSessions(): void +{ + $consoleDB = getConsoleDB(); + + deleteForProjectIds(function (Document $project) use ($consoleDB) { + $dbForProject = getProjectDB($project); + + $project = $consoleDB->getDocument('projects', $project->getId()); + $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration); + + // Delete Sessions + deleteByGroup('sessions', [ + Query::lessThan('$createdAt', $expired) + ], $dbForProject); + }); +} + +/** + * @param string $datetime + */ +function deleteRealtimeUsage(string $datetime): void +{ + deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = getProjectDB($project); + // Delete Dead Realtime Logs + deleteByGroup('realtime', [ + Query::lessThan('timestamp', $datetime) + ], $dbForProject); + }); +} + +/** + * @param string $datetime + * @throws Exception + */ +function deleteAbuseLogs(string $datetime): void +{ + if (empty($datetime)) { + throw new Exception('Failed to delete audit logs. No datetime provided'); } - /** - * @param Document $document function document - * @param Document $project - */ - protected function deleteFunction(Document $document, Document $project): void - { + deleteForProjectIds(function (Document $project) use ($datetime) { $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $functionId = $document->getId(); - - /** - * Delete Variables - */ - Console::info("Deleting variables for function " . $functionId); - $this->deleteByGroup('variables', [ - Query::equal('functionId', [$functionId]) - ], $dbForProject); - - /** - * Delete Deployments - */ - Console::info("Deleting deployments for function " . $functionId); - $storageFunctions = $this->getFunctionsDevice($projectId); - $deploymentIds = []; - $this->deleteByGroup('deployments', [ - Query::equal('resourceId', [$functionId]) - ], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) { - $deploymentIds[] = $document->getId(); - if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { - Console::success('Deleted deployment files: ' . $document->getAttribute('path', '')); - } else { - Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); - } - }); - - /** - * Delete builds - */ - Console::info("Deleting builds for function " . $functionId); - $storageBuilds = $this->getBuildsDevice($projectId); - foreach ($deploymentIds as $deploymentId) { - $this->deleteByGroup('builds', [ - Query::equal('deploymentId', [$deploymentId]) - ], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) { - if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('path', '')); - } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); - } - }); + $dbForProject = getProjectDB($project); + $timeLimit = new TimeLimit("", 0, 1, $dbForProject); + $abuse = new Abuse($timeLimit); + $status = $abuse->cleanup($datetime); + if (!$status) { + throw new Exception('Failed to delete Abuse logs for project ' . $projectId); } + }); +} - /** - * Delete Executions - */ - Console::info("Deleting executions for function " . $functionId); - $this->deleteByGroup('executions', [ - Query::equal('functionId', [$functionId]) - ], $dbForProject); - - // TODO: Request executor to delete runtime +/** + * @param string $datetime + * @throws Exception + */ +function deleteAuditLogs(string $datetime): void +{ + if (empty($datetime)) { + throw new Exception('Failed to delete audit logs. No datetime provided'); } - /** - * @param Document $document deployment document - * @param Document $project - */ - protected function deleteDeployment(Document $document, Document $project): void - { + deleteForProjectIds(function (Document $project) use ($datetime) { $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $deploymentId = $document->getId(); - $functionId = $document->getAttribute('resourceId'); + $dbForProject = getProjectDB($project); + $audit = new Audit($dbForProject); + $status = $audit->cleanup($datetime); + if (!$status) { + throw new Exception('Failed to delete Audit logs for project' . $projectId); + } + }); +} - /** - * Delete deployment files - */ - Console::info("Deleting deployment files for deployment " . $deploymentId); - $storageFunctions = $this->getFunctionsDevice($projectId); +/** + * @param string $resource + * @param Document $project + */ +function deleteAuditLogsByResource(string $resource, Document $project): void +{ + $dbForProject = getProjectDB($project); + + deleteByGroup(Audit::COLLECTION, [ + Query::equal('resource', [$resource]) + ], $dbForProject); +} + +/** + * @param Document $document function document + * @param Document $project + */ +function deleteFunction(Document $document, Document $project): void +{ + $projectId = $project->getId(); + $dbForProject = getProjectDB($project); + $functionId = $document->getId(); + + /** + * Delete Variables + */ + Console::info("Deleting variables for function " . $functionId); + deleteByGroup('variables', [ + Query::equal('functionId', [$functionId]) + ], $dbForProject); + + /** + * Delete Deployments + */ + Console::info("Deleting deployments for function " . $functionId); + $storageFunctions = getFunctionsDevice($projectId); + $deploymentIds = []; + deleteByGroup('deployments', [ + Query::equal('resourceId', [$functionId]) + ], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) { + $deploymentIds[] = $document->getId(); if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { Console::success('Deleted deployment files: ' . $document->getAttribute('path', '')); } else { Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); } + }); - /** - * Delete builds - */ - Console::info("Deleting builds for deployment " . $deploymentId); - $storageBuilds = $this->getBuildsDevice($projectId); - $this->deleteByGroup('builds', [ + /** + * Delete builds + */ + Console::info("Deleting builds for function " . $functionId); + $storageBuilds = getBuildsDevice($projectId); + foreach ($deploymentIds as $deploymentId) { + deleteByGroup('builds', [ Query::equal('deploymentId', [$deploymentId]) - ], $dbForProject, function (Document $document) use ($storageBuilds) { + ], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) { if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { Console::success('Deleted build files: ' . $document->getAttribute('path', '')); } else { Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); } }); - - // TODO: Request executor to delete runtime } + /** + * Delete Executions + */ + Console::info("Deleting executions for function " . $functionId); + deleteByGroup('executions', [ + Query::equal('functionId', [$functionId]) + ], $dbForProject); + + // TODO: Request executor to delete runtime +} + +/** + * @param Document $document deployment document + * @param Document $project + */ +function deleteDeployment(Document $document, Document $project): void +{ + $projectId = $project->getId(); + $dbForProject = getProjectDB($project); + $deploymentId = $document->getId(); + $functionId = $document->getAttribute('resourceId'); /** - * @param Document $document to be deleted - * @param Database $database to delete it from - * @param callable $callback to perform after document is deleted - * - * @return bool + * Delete deployment files */ - protected function deleteById(Document $document, Database $database, callable $callback = null): bool - { - if ($database->deleteDocument($document->getCollection(), $document->getId())) { - Console::success('Deleted document "' . $document->getId() . '" successfully'); + Console::info("Deleting deployment files for deployment " . $deploymentId); + $storageFunctions = getFunctionsDevice($projectId); + if ($storageFunctions->delete($document->getAttribute('path', ''), true)) { + Console::success('Deleted deployment files: ' . $document->getAttribute('path', '')); + } else { + Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', '')); + } + /** + * Delete builds + */ + Console::info("Deleting builds for deployment " . $deploymentId); + $storageBuilds = getBuildsDevice($projectId); + deleteByGroup('builds', [ + Query::equal('deploymentId', [$deploymentId]) + ], $dbForProject, function (Document $document) use ($storageBuilds) { + if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('path', '')); + } else { + Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); + } + }); + + // TODO: Request executor to delete runtime +} + + +/** + * @param Document $document to be deleted + * @param Database $database to delete it from + * @param callable $callback to perform after document is deleted + * + * @return bool + */ +function deleteById(Document $document, Database $database, callable $callback = null): bool +{ + if ($database->deleteDocument($document->getCollection(), $document->getId())) { + Console::success('Deleted document "' . $document->getId() . '" successfully'); + + if (is_callable($callback)) { + $callback($document); + } + + return true; + } else { + Console::error('Failed to delete document: ' . $document->getId()); + return false; + } +} + +/** + * @param callable $callback + */ +function deleteForProjectIds(callable $callback): void +{ + // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document + $count = 0; + $chunk = 0; + $limit = 50; + $projects = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $projects = getConsoleDB()->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]); + + $chunk++; + + /** @var string[] $projectIds */ + $sum = count($projects); + + Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects'); + foreach ($projects as $project) { + $callback($project); + $count++; + } + } + + $executionEnd = \microtime(true); + Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds"); +} + +/** + * @param string $collection collectionID + * @param Query[] $queries + * @param Database $database + * @param callable $callback + */ +function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void +{ + $count = 0; + $chunk = 0; + $limit = 50; + $results = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + + $sum = count($results); + + Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents'); + + foreach ($results as $document) { + deleteById($document, $database, $callback); + $count++; + } + } + + $executionEnd = \microtime(true); + + Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); +} + +/** + * @param string $collection collectionID + * @param Query[] $queries + * @param Database $database + * @param callable $callback + */ +function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void +{ + $count = 0; + $chunk = 0; + $limit = 50; + $results = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + + $sum = count($results); + + foreach ($results as $document) { if (is_callable($callback)) { $callback($document); } - return true; - } else { - Console::error('Failed to delete document: ' . $document->getId()); - return false; + $count++; } } - /** - * @param callable $callback - */ - protected function deleteForProjectIds(callable $callback): void - { - // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document - $count = 0; - $chunk = 0; - $limit = 50; - $projects = []; - $sum = $limit; + $executionEnd = \microtime(true); - $executionStart = \microtime(true); + Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); +} - while ($sum === $limit) { - $projects = $this->getConsoleDB()->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]); +/** + * @param Document $document certificates document + */ +function deleteCertificates(Document $document): void +{ + $consoleDB = getConsoleDB(); - $chunk++; + // If domain has certificate generated + if (isset($document['certificateId'])) { + $domainUsingCertificate = $consoleDB->findOne('domains', [ + Query::equal('certificateId', [$document['certificateId']]) + ]); - /** @var string[] $projectIds */ - $sum = count($projects); - - Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects'); - foreach ($projects as $project) { - $callback($project); - $count++; + if (!$domainUsingCertificate) { + $mainDomain = App::getEnv('_APP_DOMAIN_TARGET', ''); + if ($mainDomain === $document->getAttribute('domain')) { + $domainUsingCertificate = $mainDomain; } } - $executionEnd = \microtime(true); - Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds"); - } - - /** - * @param string $collection collectionID - * @param Query[] $queries - * @param Database $database - * @param callable $callback - */ - protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void - { - $count = 0; - $chunk = 0; - $limit = 50; - $results = []; - $sum = $limit; - - $executionStart = \microtime(true); - - while ($sum === $limit) { - $chunk++; - - $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); - - $sum = count($results); - - Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents'); - - foreach ($results as $document) { - $this->deleteById($document, $database, $callback); - $count++; - } + // If certificate is still used by some domain, mark we can't delete. + // Current domain should not be found, because we only have copy. Original domain is already deleted from database. + if ($domainUsingCertificate) { + Console::warning("Skipping certificate deletion, because a domain is still using it."); + return; } - - $executionEnd = \microtime(true); - - Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } - /** - * @param string $collection collectionID - * @param Query[] $queries - * @param Database $database - * @param callable $callback - */ - protected function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void - { - $count = 0; - $chunk = 0; - $limit = 50; - $results = []; - $sum = $limit; + $domain = $document->getAttribute('domain'); + $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; + $checkTraversal = realpath($directory) === $directory; - $executionStart = \microtime(true); - - while ($sum === $limit) { - $chunk++; - - $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); - - $sum = count($results); - - foreach ($results as $document) { - if (is_callable($callback)) { - $callback($document); - } - - $count++; - } - } - - $executionEnd = \microtime(true); - - Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); - } - - /** - * @param Document $document certificates document - */ - protected function deleteCertificates(Document $document): void - { - $consoleDB = $this->getConsoleDB(); - - // If domain has certificate generated + if ($domain && $checkTraversal && is_dir($directory)) { + // Delete certificate document, so Appwrite is aware of change if (isset($document['certificateId'])) { - $domainUsingCertificate = $consoleDB->findOne('domains', [ - Query::equal('certificateId', [$document['certificateId']]) - ]); - - if (!$domainUsingCertificate) { - $mainDomain = App::getEnv('_APP_DOMAIN_TARGET', ''); - if ($mainDomain === $document->getAttribute('domain')) { - $domainUsingCertificate = $mainDomain; - } - } - - // If certificate is still used by some domain, mark we can't delete. - // Current domain should not be found, because we only have copy. Original domain is already deleted from database. - if ($domainUsingCertificate) { - Console::warning("Skipping certificate deletion, because a domain is still using it."); - return; - } + $consoleDB->deleteDocument('certificates', $document['certificateId']); } - $domain = $document->getAttribute('domain'); - $directory = APP_STORAGE_CERTIFICATES . '/' . $domain; - $checkTraversal = realpath($directory) === $directory; - - if ($domain && $checkTraversal && is_dir($directory)) { - // Delete certificate document, so Appwrite is aware of change - if (isset($document['certificateId'])) { - $consoleDB->deleteDocument('certificates', $document['certificateId']); - } - - // Delete files, so Traefik is aware of change - array_map('unlink', glob($directory . '/*.*')); - rmdir($directory); - Console::info("Deleted certificate files for {$domain}"); - } else { - Console::info("No certificate files found for {$domain}"); - } - } - - protected function deleteBucket(Document $document, Document $project) - { - $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($project); - $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); - - $device = $this->getFilesDevice($projectId); - - $device->deletePath($document->getId()); + // Delete files, so Traefik is aware of change + array_map('unlink', glob($directory . '/*.*')); + rmdir($directory); + Console::info("Deleted certificate files for {$domain}"); + } else { + Console::info("No certificate files found for {$domain}"); } } + +function deleteBucket(Document $document, Document $project) +{ + $projectId = $project->getId(); + $dbForProject = getProjectDB($project); + $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); + + $device = getFilesDevice($projectId); + + $device->deletePath($document->getId()); +} + +$server->workerStart(); +$server->start(); \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 813a785a7d..fe430fbb07 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -392,6 +392,7 @@ services: - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST + - _APP_REGION appwrite-worker-databases: entrypoint: worker-databases @@ -505,6 +506,7 @@ services: - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + - _APP_REGION appwrite-worker-functions: entrypoint: worker-functions @@ -637,9 +639,10 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE diff --git a/src/Appwrite/Platform/Tasks/EdgeSync.php b/src/Appwrite/Platform/Tasks/EdgeSync.php index ce30729960..f4450785dc 100644 --- a/src/Appwrite/Platform/Tasks/EdgeSync.php +++ b/src/Appwrite/Platform/Tasks/EdgeSync.php @@ -83,7 +83,7 @@ class EdgeSync extends Action ]); } } - if (!$found) { + if (!$found) { Console::info("[{$time}] No keys where found for region {$code}."); } } diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index c25341bce7..2f236d32ca 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -1,4 +1,5 @@