diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 93df84a6f6..d6aaaa24bb 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1110,7 +1110,7 @@ App::put('/v1/account/sessions/magic-url') ->inject('locale') ->inject('geodb') ->inject('queueForEvents') - ->action(function (string $userId, string $secret, Request $request, Response $response,Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents) { + ->action(function (string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents) { /** @var Utopia\Database\Document $user */ diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index b4709e656e..cdfdd8172b 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -674,9 +674,9 @@ App::delete('/v1/databases/:databaseId') ->param('databaseId', '', new UID(), 'Database ID.') ->inject('response') ->inject('dbForProject') + ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForDeletes') - ->action(function (string $databaseId, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) { + ->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { $database = $dbForProject->getDocument('databases', $databaseId); @@ -691,9 +691,9 @@ App::delete('/v1/databases/:databaseId') $dbForProject->deleteCachedDocument('databases', $database->getId()); $dbForProject->deleteCachedCollection('databases_' . $database->getInternalId()); - $queueForDeletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($database); + $queueForDatabase + ->setType(DATABASE_TYPE_DELETE_DATABASE) + ->setDatabase($database); $queueForEvents ->setParam('databaseId', $database->getId()) @@ -1040,10 +1040,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId') ->param('collectionId', '', new UID(), 'Collection ID.') ->inject('response') ->inject('dbForProject') - ->inject('mode') + ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForDeletes') - ->action(function (string $databaseId, string $collectionId, Response $response, Database $dbForProject, string $mode, Event $queueForEvents, Delete $queueForDeletes) { + ->inject('mode') + ->action(function (string $databaseId, string $collectionId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, string $mode) { $database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); @@ -1063,9 +1063,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId') $dbForProject->deleteCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); - $queueForDeletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($collection); + $queueForDatabase + ->setType(DATABASE_TYPE_DELETE_COLLECTION) + ->setDatabase($database) + ->setCollection($collection); $queueForEvents ->setContext('database', $database) @@ -2291,8 +2292,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->setType(DATABASE_TYPE_DELETE_ATTRIBUTE) ->setCollection($collection) ->setDatabase($db) - ->setDocument($attribute) - ; + ->setDocument($attribute); // Select response model based on type and format $type = $attribute->getAttribute('type'); @@ -3458,10 +3458,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('requestTimestamp') ->inject('response') ->inject('dbForProject') + ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForDeletes') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes, string $mode) { + ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, string $mode) { $database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); @@ -3532,8 +3532,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $processDocument($collection, $document); - $queueForDeletes - ->setType(DELETE_TYPE_AUDIT) + $queueForDatabase + ->setType(DATABASE_TYPE_DELETE_DOCUMENT) + ->setDatabase($database) + ->setCollection($collection) ->setDocument($document); $queueForEvents diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 383adebe67..3a1c96690e 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1680,8 +1680,6 @@ App::post('/v1/functions/:functionId/executions') $execution->setAttribute('logs', $executionResponse['logs']); $execution->setAttribute('errors', $executionResponse['errors']); $execution->setAttribute('duration', $executionResponse['duration']); - - } catch (\Throwable $th) { $durationEnd = \microtime(true); diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 99fcfeb38c..4a0378105f 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -257,7 +257,7 @@ App::put('/v1/teams/:teamId') ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->action(function (string $teamId, string $name, ?\DateTime $requestTimestamp , Response $response, Database $dbForProject, Event $queueForEvents) { + ->action(function (string $teamId, string $name, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents) { $team = $dbForProject->getDocument('teams', $teamId); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index bd709ee272..a992d51965 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -130,7 +130,7 @@ App::post('/v1/users') ->inject('project') ->inject('dbForProject') ->inject('queueForEvents') - ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response,Document $project, Database $dbForProject, Event $queueForEvents) { + ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) { $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents); $response diff --git a/app/init.php b/app/init.php index be152f830b..6d0f52e94e 100644 --- a/app/init.php +++ b/app/init.php @@ -147,6 +147,9 @@ const DATABASE_TYPE_CREATE_ATTRIBUTE = 'createAttribute'; const DATABASE_TYPE_CREATE_INDEX = 'createIndex'; const DATABASE_TYPE_DELETE_ATTRIBUTE = 'deleteAttribute'; const DATABASE_TYPE_DELETE_INDEX = 'deleteIndex'; +const DATABASE_TYPE_DELETE_COLLECTION = 'deleteCollection'; +const DATABASE_TYPE_DELETE_DOCUMENT = 'deleteDocument'; +const DATABASE_TYPE_DELETE_DATABASE = 'deleteDatabase'; // Build Worker Types const BUILD_TYPE_DEPLOYMENT = 'deployment'; const BUILD_TYPE_RETRY = 'retry'; diff --git a/src/Appwrite/Event/Audit.php b/src/Appwrite/Event/Audit.php index 2321fe77dd..75b4ef573d 100644 --- a/src/Appwrite/Event/Audit.php +++ b/src/Appwrite/Event/Audit.php @@ -19,7 +19,6 @@ class Audit extends Event $this ->setQueue(Event::AUDITS_QUEUE_NAME) ->setClass(Event::BUILDS_CLASS_NAME); - } /** diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 788924a238..9a2caf9c4e 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -4,14 +4,18 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Event; use Appwrite\Messaging\Adapter\Realtime; +use Appwrite\Utopia\Response\Model\Platform; use Exception; +use Utopia\Audit\Audit; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Conflict; +use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Exception as DatabaseException; +use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\Queue\Message; @@ -56,19 +60,18 @@ class Databases extends Action $document = new Document($payload['document'] ?? []); $database = new Document($payload['database'] ?? []); - if ($collection->isEmpty()) { - throw new Exception('Missing collection'); - } - - if ($document->isEmpty()) { - throw new Exception('Missing document'); + if ($database->isEmpty()) { + throw new Exception('Missing database'); } match (strval($type)) { + DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject), + DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject), DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject), DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject), DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject), DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject), + DATABASE_TYPE_DELETE_DOCUMENT => $this->deleteDocument($database, $collection, $document, $project, $dbForProject), default => Console::error('No database operation for type: ' . $type), }; } @@ -87,6 +90,12 @@ class Databases extends Action */ private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($attribute->isEmpty()) { + throw new Exception('Missing attribute'); + } $projectId = $project->getId(); @@ -173,25 +182,7 @@ class Databases extends Action ); } } finally { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $attribute, - project: $project, - ); - - Realtime::send( - projectId: 'console', - payload: $attribute->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $this->trigger($database, $collection, $attribute, $project, $projectId, $events); } if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) { @@ -215,6 +206,13 @@ class Databases extends Action **/ private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($attribute->isEmpty()) { + throw new Exception('Missing attribute'); + } + $projectId = $project->getId(); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [ @@ -284,25 +282,7 @@ class Databases extends Action ); } } finally { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $attribute, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $attribute->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $this->trigger($database, $collection, $attribute, $project, $projectId, $events); } // The underlying database removes/rebuilds indexes when attribute is removed @@ -380,6 +360,13 @@ class Databases extends Action */ private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($index->isEmpty()) { + throw new Exception('Missing index'); + } + $projectId = $project->getId(); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [ @@ -412,25 +399,7 @@ class Databases extends Action $index->setAttribute('status', 'failed') ); } finally { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $index, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $index->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $this->trigger($database, $collection, $index, $project, $projectId, $events); } $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId); @@ -451,6 +420,13 @@ class Databases extends Action */ private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($index->isEmpty()) { + throw new Exception('Missing index'); + } + $projectId = $project->getId(); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [ @@ -470,7 +446,6 @@ class Databases extends Action } catch (\Exception $e) { Console::error($e->getMessage()); - if ($e instanceof DatabaseException) { $index->setAttribute('error', $e->getMessage()); } @@ -480,27 +455,179 @@ class Databases extends Action $index->setAttribute('status', 'stuck') ); } finally { - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $events[0], - payload: $index, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $index->getArrayCopy(), - events: $events, - channels: $target['channels'], - roles: $target['roles'], - options: [ - 'projectId' => $projectId, - 'databaseId' => $database->getId(), - 'collectionId' => $collection->getId() - ] - ); + $this->trigger($database, $collection, $index, $project, $projectId, $events); } $dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collection->getId()); } + + /** + * @param Document $database + * @param Document $project + * @param $dbForProject + * @return void + * @throws Exception + */ + protected function deleteDatabase(Document $database, Document $project, $dbForProject): void + { + $this->deleteByGroup('database_' . $database->getInternalId(), [], $dbForProject, function ($collection) use ($database, $project, $dbForProject) { + $this->deleteCollection($database, $collection, $project, $dbForProject); + }); + + $dbForProject->deleteCollection('database_' . $database->getInternalId()); + + $this->deleteAuditLogsByResource('database/' . $database->getId(), $project, $dbForProject); + } + + /** + * @param Document $database + * @param Document $collection + * @param Document $project + * @param Database $dbForProject + * @return void + * @throws Authorization + * @throws Conflict + * @throws DatabaseException + * @throws Restricted + * @throws Structure + */ + protected function deleteCollection(Document $database, Document $collection, Document $project, Database $dbForProject): void + { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + + $collectionId = $collection->getId(); + $collectionInternalId = $collection->getInternalId(); + $databaseId = $database->getId(); + $databaseInternalId = $database->getInternalId(); + + $relationships = \array_filter( + $collection->getAttribute('attributes'), + fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP + ); + + foreach ($relationships as $relationship) { + if (!$relationship['twoWay']) { + continue; + } + $relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']); + $dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']); + $dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId()); + $dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId()); + } + + $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId()); + + $this->deleteByGroup('attributes', [ + Query::equal('databaseInternalId', [$databaseInternalId]), + Query::equal('collectionInternalId', [$collectionInternalId]) + ], $dbForProject); + + $this->deleteByGroup('indexes', [ + Query::equal('databaseInternalId', [$databaseInternalId]), + Query::equal('collectionInternalId', [$collectionInternalId]) + ], $dbForProject); + + $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project, $dbForProject); + } + + private function deleteDocument(Document $database, Document $collection, Document $document, Document $project, Database $dbForProject) + { + if ($collection->isEmpty()) { + throw new Exception('Missing collection'); + } + if ($document->isEmpty()) { + throw new Exception('Missing document'); + } + + $this->deleteAuditLogsByResource('document/' . $document->getId(), $project, $dbForProject); + } + + /** + * @param string $resource + * @param Document $project + * @param Database $dbForProject + * @return void + * @throws Exception + */ + protected function deleteAuditLogsByResource(string $resource, Document $project, Database $dbForProject): void + { + $this->deleteByGroup(Audit::COLLECTION, [ + Query::equal('resource', [$resource]) + ], $dbForProject); + } + + /** + * @param string $collection collectionID + * @param array $queries + * @param Database $database + * @param callable|null $callback + * @return void + * @throws Exception + */ + protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void + { + $count = 0; + $chunk = 0; + $limit = 50; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + + $sum = count($results); + + Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents'); + + foreach ($results as $document) { + if ($database->deleteDocument($document->getCollection(), $document->getId())) { + Console::success('Deleted document "' . $document->getId() . '" successfully'); + + if (\is_callable($callback)) { + $callback($document); + } + } else { + Console::error('Failed to delete document: ' . $document->getId()); + } + $count++; + } + } + + $executionEnd = \microtime(true); + + Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); + } + + protected function trigger( + Document $database, + Document $collection, + Document $attribute, + Document $project, + string $projectId, + array $events + ): void { + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $events[0], + payload: $attribute, + project: $project, + ); + Realtime::send( + projectId: 'console', + payload: $attribute->getArrayCopy(), + events: $events, + channels: $target['channels'], + roles: $target['roles'], + options: [ + 'projectId' => $projectId, + 'databaseId' => $database->getId(), + 'collectionId' => $collection->getId() + ] + ); + } } diff --git a/tests/unit/Event/EventTest.php b/tests/unit/Event/EventTest.php index b48fc8f849..a430a7fdc6 100644 --- a/tests/unit/Event/EventTest.php +++ b/tests/unit/Event/EventTest.php @@ -15,7 +15,6 @@ require_once __DIR__ . '/../../../app/init.php'; class EventTest extends TestCase { - protected ?Event $object = null; protected string $queue = '';