diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 7e9450830e..0a8dec4421 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -252,8 +252,8 @@ App::delete('/v1/database/collections/:collectionId') } $deletes - ->setParam('document', $collection) ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $collection) ; $events diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index a664d0ebb7..476db04724 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -365,8 +365,8 @@ App::delete('/v1/functions/:functionId') } $deletes - ->setParam('document', $function->getArrayCopy()) ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $function->getArrayCopy()) ; $response->noContent(); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 3d9b33f012..f8aa16936f 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -430,8 +430,8 @@ App::delete('/v1/projects/:projectId') } $deletes - ->setParam('document', $project->getArrayCopy()) ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $project->getArrayCopy()) ; foreach (['keys', 'webhooks', 'tasks', 'platforms', 'domains'] as $key) { // Delete all children (keys, webhooks, tasks [stop tasks?], platforms) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index cbddcac56c..90ff6cd1b9 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -516,8 +516,8 @@ App::delete('/v1/users/:userId') } $deletes - ->setParam('document', $user) ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $user) ; $events diff --git a/app/init.php b/app/init.php index 1db02ead44..df8e9b0328 100644 --- a/app/init.php +++ b/app/init.php @@ -55,6 +55,7 @@ const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; // Deletion Types const DELETE_TYPE_DOCUMENT = 'document'; +const DELETE_TYPE_EXECUTION_LOGS = 'execution_logs'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 09a62928b6..ced04c47c3 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -26,36 +26,29 @@ function getConsoleDB() { return $consoleDB; } -function notifyDeleteExecutionLogs(array $projectIds) +function notifyDeleteExecutionLogs() { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ - 'type' => DELETE_TYPE_DOCUMENT, + 'type' => DELETE_TYPE_EXECUTION_LOGS, 'document' => new Document([ - '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, - 'projectIds' => $projectIds + '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS ]) ]); } -function notifyDeleteAbuseLogs(array $projectIds, int $interval) +function notifyDeleteAbuseLogs(int $interval) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ 'type' => DELETE_TYPE_ABUSE, - 'document' => new Document([ - 'projectIds' => $projectIds, - 'timestamp' => time() - $interval, - ]) + 'timestamp' => time() - $interval ]); } -function notifyDeleteAuditLogs(array $projectIds, int $interval) +function notifyDeleteAuditLogs(int $interval) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ 'type' => DELETE_TYPE_AUDIT, - 'document' => new Document([ - 'projectIds' => $projectIds, - 'timestamp' => time() - $interval - ]) + 'timestamp' => time() - $interval ]); } @@ -72,23 +65,11 @@ $cli Console::loop(function() use ($consoleDB, $interval){ - Authorization::disable(); - $projects = $consoleDB->getCollection([ - 'filters' => [ - '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, - ], - ]); - Authorization::reset(); - - $projectIds = array_map (function ($project) { - return $project->getId(); - }, $projects); - Console::info("[ MAINTENANCE TASK ] Notifying deletes workers every {$interval} seconds"); - // notifyDeleteExecutionLogs($projectIds); - notifyDeleteAbuseLogs($projectIds, $interval); - // notifyDeleteAuditLogs($projectIds, $interval); + notifyDeleteExecutionLogs(); + notifyDeleteAbuseLogs($interval); + notifyDeleteAuditLogs($interval); }, $intervalMicroseconds); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f7fd9439cb..3052439b4e 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -47,9 +47,6 @@ class DeletesV1 case Database::SYSTEM_COLLECTION_FUNCTIONS: $this->deleteFunction($document, $projectId); break; - case Database::SYSTEM_COLLECTION_EXECUTIONS: - $this->deleteExecutionLogs($document); - break; case Database::SYSTEM_COLLECTION_USERS: $this->deleteUser($document, $projectId); break; @@ -61,6 +58,11 @@ class DeletesV1 break; } break; + + case DELETE_TYPE_EXECUTION_LOGS: + $this->deleteExecutionLogs($document); + break; + case DELETE_TYPE_AUDIT: $this->deleteAuditLogs($document); break; @@ -109,7 +111,7 @@ class DeletesV1 foreach ($tokens as $token) { if (!$this->getProjectDB($projectId)->deleteDocument($token->getId())) { - throw new Exception('Failed to remove token from DB', 500); + throw new Exception('Failed to remove token from DB'); } } @@ -122,10 +124,10 @@ class DeletesV1 protected function deleteExecutionLogs(Document $document) { - $projectIds = $document->getAttribute('projectIds', []); + $projectIds = $this->getProjectIds(); foreach ($projectIds as $projectId) { if (!($projectDB = $this->getProjectDB($projectId))) { - throw new Exception('Failed to get projectDB for project '.$projectId, 500); + throw new Exception('Failed to get projectDB for project '.$projectId); } // Delete Executions @@ -139,7 +141,7 @@ class DeletesV1 protected function deleteAbuseLogs($document) { global $register; - $projectIds = $document->getAttribute('projectIds', []); + $projectIds = $this->getProjectIds(); $timestamp = $document->getAttribute('timestamp', 0); if($timestamp == 0) { @@ -151,13 +153,13 @@ class DeletesV1 }); foreach ($projectIds as $projectId) { - Console::success("Deleting abuse logs for Project: ", $projectId); + Console::success("Deleting abuse logs for Project: ".$projectId); $timeLimit->setNamespace('app_'.$projectId); $abuse = new Abuse($timeLimit); $status = $abuse->cleanup($timestamp); if (!$status) { - throw new Exception('Failed to delete Abuse logs for project '.$projectId, 500); + throw new Exception('Failed to delete Abuse logs for project '.$projectId); } } @@ -166,7 +168,7 @@ class DeletesV1 protected function deleteAuditLogs($document) { global $register; - $projectIds = $document->getAttribute('projectIds', []); + $projectIds = $this->getProjectIds(); $timestamp = $document->getAttribute('timestamp', 0); if($timestamp == 0) { @@ -179,7 +181,7 @@ class DeletesV1 $audit = new Audit($adapter); $status = $audit->cleanup($timestamp); if (!$status) { - throw new Exception('Failed to delete Audit logs for project'.$projectId, 500); + throw new Exception('Failed to delete Audit logs for project'.$projectId); } } } @@ -231,6 +233,23 @@ class DeletesV1 Authorization::reset(); } + protected function getProjectIds(): array + { + Authorization::disable(); + $projects = $this->getConsoleDB()->getCollection([ + 'filters' => [ + '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, + ], + ]); + Authorization::reset(); + + $projectIds = array_map (function ($project) { + return $project->getId(); + }, $projects); + + return $projectIds; + } + protected function deleteByGroup(array $filters, Database $database, callable $callback = null) { $count = 0; diff --git a/composer.json b/composer.json index 8628e48e21..d691919a50 100644 --- a/composer.json +++ b/composer.json @@ -35,8 +35,8 @@ "appwrite/php-clamav": "1.0.*", "utopia-php/framework": "0.9.8", - "utopia-php/abuse": "0.3.*", - "utopia-php/audit": "0.5.*", + "utopia-php/abuse": "0.3.1", + "utopia-php/audit": "0.5.1", "utopia-php/cache": "0.2.*", "utopia-php/cli": "0.8.0", "utopia-php/config": "0.2.*", diff --git a/composer.lock b/composer.lock index e070af3683..d260828c0a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f366e884df2132577f44576e459e3a4b", + "content-hash": "3e96a4fbfbff083f2ca166fcfa6fbbbd", "packages": [ { "name": "appwrite/php-clamav", @@ -1112,16 +1112,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.3.0", + "version": "0.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "5047cb311de8d7609e2678fdc405651f43f7e550" + "reference": "23c2eb533bca8f3ef5548ae265398fa7d4d39a1c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/5047cb311de8d7609e2678fdc405651f43f7e550", - "reference": "5047cb311de8d7609e2678fdc405651f43f7e550", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/23c2eb533bca8f3ef5548ae265398fa7d4d39a1c", + "reference": "23c2eb533bca8f3ef5548ae265398fa7d4d39a1c", "shasum": "" }, "require": { @@ -1158,22 +1158,22 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.3.0" + "source": "https://github.com/utopia-php/abuse/tree/0.3.1" }, - "time": "2020-12-17T18:39:44+00:00" + "time": "2020-12-21T17:28:03+00:00" }, { "name": "utopia-php/audit", - "version": "0.5.0", + "version": "0.5.1", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "5000fdbe755bbfb1e8bc26c61b90e28c40e5744a" + "reference": "154a850170a58667a15e4b65fbabb6cd0b709dd9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/5000fdbe755bbfb1e8bc26c61b90e28c40e5744a", - "reference": "5000fdbe755bbfb1e8bc26c61b90e28c40e5744a", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/154a850170a58667a15e4b65fbabb6cd0b709dd9", + "reference": "154a850170a58667a15e4b65fbabb6cd0b709dd9", "shasum": "" }, "require": { @@ -1210,9 +1210,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.5.0" + "source": "https://github.com/utopia-php/audit/tree/0.5.1" }, - "time": "2020-12-17T18:41:34+00:00" + "time": "2020-12-21T17:28:53+00:00" }, { "name": "utopia-php/cache", @@ -2551,16 +2551,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.10.3", + "version": "v4.10.4", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "dbe56d23de8fcb157bbc0cfb3ad7c7de0cfb0984" + "reference": "c6d052fc58cb876152f89f532b95a8d7907e7f0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/dbe56d23de8fcb157bbc0cfb3ad7c7de0cfb0984", - "reference": "dbe56d23de8fcb157bbc0cfb3ad7c7de0cfb0984", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/c6d052fc58cb876152f89f532b95a8d7907e7f0e", + "reference": "c6d052fc58cb876152f89f532b95a8d7907e7f0e", "shasum": "" }, "require": { @@ -2601,9 +2601,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.10.3" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.10.4" }, - "time": "2020-12-03T17:45:45+00:00" + "time": "2020-12-20T10:01:03+00:00" }, { "name": "openlss/lib-array2xml", @@ -2933,16 +2933,16 @@ }, { "name": "phpspec/prophecy", - "version": "1.12.1", + "version": "1.12.2", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "8ce87516be71aae9b956f81906aaf0338e0d8a2d" + "reference": "245710e971a030f42e08f4912863805570f23d39" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/8ce87516be71aae9b956f81906aaf0338e0d8a2d", - "reference": "8ce87516be71aae9b956f81906aaf0338e0d8a2d", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/245710e971a030f42e08f4912863805570f23d39", + "reference": "245710e971a030f42e08f4912863805570f23d39", "shasum": "" }, "require": { @@ -2954,7 +2954,7 @@ }, "require-dev": { "phpspec/phpspec": "^6.0", - "phpunit/phpunit": "^8.0 || ^9.0 <9.3" + "phpunit/phpunit": "^8.0 || ^9.0" }, "type": "library", "extra": { @@ -2994,9 +2994,9 @@ ], "support": { "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/1.12.1" + "source": "https://github.com/phpspec/prophecy/tree/1.12.2" }, - "time": "2020-09-29T09:10:42+00:00" + "time": "2020-12-19T10:15:11+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/docker-compose.yml b/docker-compose.yml index 83bbd5e76e..b43f1d003c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -191,7 +191,8 @@ services: volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw - - ./vendor:/usr/src/code/vendor + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src environment: - _APP_ENV - _APP_REDIS_HOST