diff --git a/.env b/.env index f881ca35d3..5ce3837137 100644 --- a/.env +++ b/.env @@ -30,3 +30,4 @@ _APP_FUNCTIONS_CONTAINERS=10 _APP_FUNCTIONS_CPUS=1 _APP_FUNCTIONS_MEMORY=128 _APP_FUNCTIONS_MEMORY_SWAP=128 +_APP_MAINTENANCE_INTERVAL=86400 diff --git a/Dockerfile b/Dockerfile index fc7a03f936..fb578a6f1a 100755 --- a/Dockerfile +++ b/Dockerfile @@ -96,7 +96,9 @@ ENV _APP_SERVER=swoole \ _APP_FUNCTIONS_MEMORY=128 \ _APP_FUNCTIONS_MEMORY_SWAP=128 \ _APP_SETUP=self-hosted \ - _APP_VERSION=$VERSION + _APP_VERSION=$VERSION \ + # 1 Day = 86400 s + _APP_MAINTENANCE_INTERVAL=86400 #ENV _APP_SMTP_SECURE '' #ENV _APP_SMTP_USERNAME '' #ENV _APP_SMTP_PASSWORD '' @@ -158,6 +160,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ + chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/schedule && \ diff --git a/app/cli.php b/app/cli.php index 9c99890602..ced2fb8da4 100644 --- a/app/cli.php +++ b/app/cli.php @@ -9,6 +9,7 @@ use Utopia\CLI\Console; $cli = new CLI(); include 'tasks/doctor.php'; +include 'tasks/maintenance.php'; include 'tasks/install.php'; include 'tasks/migrate.php'; include 'tasks/sdks.php'; diff --git a/app/config/variables.php b/app/config/variables.php index f752e90201..0893dfd252 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -169,4 +169,10 @@ return [ 'required' => false, 'question' => '', ], + [ + 'name' => '_APP_MAINTENANCE_INTERVAL', + 'default' => '86400', + 'required' => false, + 'question' => '', + ], ]; \ No newline at end of file diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index a3018ed026..ad32e80116 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -265,6 +265,7 @@ App::delete('/v1/database/collections/:collectionId') } $deletes + ->setParam('type', DELETE_TYPE_DOCUMENT) ->setParam('document', $collection) ; diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index da5572e80d..5fd882fc7c 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -380,6 +380,7 @@ App::delete('/v1/functions/:functionId') } $deletes + ->setParam('type', DELETE_TYPE_DOCUMENT) ->setParam('document', $function->getArrayCopy()) ; diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index c1c9de4755..90efef16c5 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -5,6 +5,7 @@ use Utopia\Exception; use Appwrite\Storage\Device\Local; use Appwrite\Storage\Storage; use Appwrite\ClamAV\Network; +use Appwrite\Event\Event; App::get('/v1/health') ->desc('Get HTTP') @@ -130,8 +131,8 @@ App::get('/v1/health/queue/webhooks') ->action(function ($response) { /** @var Appwrite\Utopia\Response $response */ - $response->json(['size' => Resque::size('v1-webhooks')]); - }); + $response->json(['size' => Resque::size(Event::WEBHOOK_QUEUE_NAME)]); + }, ['response']); App::get('/v1/health/queue/tasks') ->desc('Get Tasks Queue') @@ -145,8 +146,8 @@ App::get('/v1/health/queue/tasks') ->action(function ($response) { /** @var Appwrite\Utopia\Response $response */ - $response->json(['size' => Resque::size('v1-tasks')]); - }); + $response->json(['size' => Resque::size(Event::TASK_QUEUE_NAME)]); + }, ['response']); App::get('/v1/health/queue/logs') ->desc('Get Logs Queue') @@ -160,8 +161,8 @@ App::get('/v1/health/queue/logs') ->action(function ($response) { /** @var Appwrite\Utopia\Response $response */ - $response->json(['size' => Resque::size('v1-audit')]); - }); + $response->json(['size' => Resque::size(Event::AUDITS_QUEUE_NAME)]); + }, ['response']); App::get('/v1/health/queue/usage') ->desc('Get Usage Queue') @@ -175,8 +176,8 @@ App::get('/v1/health/queue/usage') ->action(function ($response) { /** @var Appwrite\Utopia\Response $response */ - $response->json(['size' => Resque::size('v1-usage')]); - }); + $response->json(['size' => Resque::size(Event::USAGE_QUEUE_NAME)]); + }, ['response']); App::get('/v1/health/queue/certificates') ->desc('Get Certificate Queue') @@ -190,8 +191,8 @@ App::get('/v1/health/queue/certificates') ->action(function ($response) { /** @var Appwrite\Utopia\Response $response */ - $response->json(['size' => Resque::size('v1-certificates')]); - }); + $response->json(['size' => Resque::size(Event::CERTIFICATES_QUEUE_NAME)]); + }, ['response']); App::get('/v1/health/queue/functions') ->desc('Get Functions Queue') @@ -205,8 +206,8 @@ App::get('/v1/health/queue/functions') ->action(function ($response) { /** @var Appwrite\Utopia\Response $response */ - $response->json(['size' => Resque::size('v1-functions')]); - }); + $response->json(['size' => Resque::size(Event::FUNCTIONS_QUEUE_NAME)]); + }, ['response']); App::get('/v1/health/storage/local') ->desc('Get Local Storage') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index f2f5f7ad28..377def2960 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -464,7 +464,10 @@ App::delete('/v1/projects/:projectId') throw new Exception('Project not found', 404); } - $deletes->setParam('document', $project->getArrayCopy()); + $deletes + ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $project->getArrayCopy()) + ; foreach (['keys', 'webhooks', 'tasks', 'platforms', 'domains'] as $key) { // Delete all children (keys, webhooks, tasks [stop tasks?], platforms) $list = $project->getAttribute('webhooks', []); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index ba12b4ba5f..06dc87a42b 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -545,6 +545,7 @@ App::delete('/v1/users/:userId') } $deletes + ->setParam('type', DELETE_TYPE_DOCUMENT) ->setParam('document', $user) ; diff --git a/app/controllers/general.php b/app/controllers/general.php index ce1d6cfbe2..d77b3d5607 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -294,7 +294,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits $audits->trigger(); } - if (!empty($deletes->getParam('document'))) { + if (!empty($deletes->getParam('type')) && !empty($deletes->getParam('document'))) { $deletes->trigger(); } diff --git a/app/init.php b/app/init.php index 34ef3a5823..887cac9b23 100644 --- a/app/init.php +++ b/app/init.php @@ -52,7 +52,12 @@ const APP_SOCIAL_INSTAGRAM = 'https://www.instagram.com/appwrite.io'; const APP_SOCIAL_GITHUB = 'https://github.com/appwrite'; const APP_SOCIAL_DISCORD = 'https://appwrite.io/discord'; const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; -const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; +const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; +// Deletion Types +const DELETE_TYPE_DOCUMENT = 'document'; +const DELETE_TYPE_EXECUTIONS = 'executions'; +const DELETE_TYPE_AUDIT = 'audit'; +const DELETE_TYPE_ABUSE = 'abuse'; $register = new Registry(); @@ -298,19 +303,19 @@ App::setResource('events', function($register) { }, ['register']); App::setResource('audits', function($register) { - return new Event('v1-audits', 'AuditsV1'); + return new Event(Event::AUDITS_QUEUE_NAME, Event::AUDITS_CLASS_NAME); }, ['register']); App::setResource('usage', function($register) { - return new Event('v1-usage', 'UsageV1'); + return new Event(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME); }, ['register']); App::setResource('mails', function($register) { - return new Event('v1-mails', 'MailsV1'); + return new Event(Event::MAILS_QUEUE_NAME, Event::MAILS_CLASS_NAME); }, ['register']); App::setResource('deletes', function($register) { - return new Event('v1-deletes', 'DeletesV1'); + return new Event(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME); }, ['register']); // Test Mock diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php new file mode 100644 index 0000000000..f8786bcac8 --- /dev/null +++ b/app/tasks/maintenance.php @@ -0,0 +1,67 @@ +setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)); + $consoleDB->setNamespace('app_console'); // Main DB + $consoleDB->setMocks(Config::getParam('collections', [])); + return $consoleDB; +} + +function notifyDeleteExecutionLogs() +{ + Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ + 'type' => DELETE_TYPE_EXECUTIONS + ]); +} + +function notifyDeleteAbuseLogs(int $interval) +{ + Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ + 'type' => DELETE_TYPE_ABUSE, + 'timestamp' => time() - $interval + ]); +} + +function notifyDeleteAuditLogs(int $interval) +{ + Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ + 'type' => DELETE_TYPE_AUDIT, + 'timestamp' => time() - $interval + ]); +} + +$cli + ->task('maintenance') + ->desc('Schedules maintenance tasks and publishes them to resque') + ->action(function () { + // # of days in seconds (1 day = 86400s) + $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); + //Convert Seconds to microseconds + $intervalMicroseconds = $interval * 1000000; + + $consoleDB = getConsoleDB(); + + Console::loop(function() use ($consoleDB, $interval){ + Console::info("[ MAINTENANCE TASK ] Notifying deletes workers every {$interval} seconds"); + notifyDeleteExecutionLogs(); + notifyDeleteAbuseLogs($interval); + notifyDeleteAuditLogs($interval); + + }, $intervalMicroseconds); + + }); \ No newline at end of file diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 00f0fbb9db..5d74fd8ef9 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -261,6 +261,27 @@ services: - _APP_SMTP_SECURE - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD + + appwrite-maintenance: + entrypoint: maintenance + container_name: appwrite-maintenance + build: + context: . + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + environment: + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_MAINTENANCE_INTERVAL + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS appwrite-schedule: image: appwrite/appwrite: diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 3f8acfe777..36d68f0f9f 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -1,22 +1,27 @@ args['projectId']; - $document = $this->args['document']; - - $document = new Document($document); + $projectId = $this->args['projectId']; + $type = $this->args['type']; - switch (strval($document->getCollection())) { - case Database::SYSTEM_COLLECTION_PROJECTS: - $this->deleteProject($document); + switch (strval($type)) { + case DELETE_TYPE_DOCUMENT: + $document = $this->args['document']; + $document = new Document($document); + switch (strval($document->getCollection())) { + case Database::SYSTEM_COLLECTION_PROJECTS: + $this->deleteProject($document); + break; + case Database::SYSTEM_COLLECTION_FUNCTIONS: + $this->deleteFunction($document, $projectId); + break; + case Database::SYSTEM_COLLECTION_USERS: + $this->deleteUser($document, $projectId); + break; + case Database::SYSTEM_COLLECTION_COLLECTIONS: + $this->deleteDocuments($document, $projectId); + break; + default: + Console::error('No lazy delete operation available for document of type: '.$document->getCollection()); + break; + } break; - case Database::SYSTEM_COLLECTION_FUNCTIONS: - $this->deleteFunction($document, $projectId); + + case DELETE_TYPE_EXECUTIONS: + $this->deleteExecutionLogs(); break; - case Database::SYSTEM_COLLECTION_USERS: - $this->deleteUser($document, $projectId); + + case DELETE_TYPE_AUDIT: + $this->deleteAuditLogs($this->args['timestamp']); break; - case Database::SYSTEM_COLLECTION_COLLECTIONS: - $this->deleteDocuments($document, $projectId); + + case DELETE_TYPE_ABUSE: + $this->deleteAbuseLogs($this->args['timestamp']); break; + default: - Console::error('No lazy delete operation available for document of type: '.$document->getCollection()); + Console::error('No delete operation for type: '.$type); break; - } + + } } public function tearDown(): void @@ -84,7 +110,7 @@ class DeletesV1 foreach ($tokens as $token) { if (!$this->getProjectDB($projectId)->deleteDocument($token->getId())) { - throw new Exception('Failed to remove token from DB', 500); + throw new Exception('Failed to remove token from DB'); } } @@ -95,6 +121,59 @@ class DeletesV1 ], $this->getProjectDB($projectId)); } + protected function deleteExecutionLogs() + { + $this->deleteForProjectIds(function($projectId) { + if (!($projectDB = $this->getProjectDB($projectId))) { + throw new Exception('Failed to get projectDB for project '.$projectId); + } + + // Delete Executions + $this->deleteByGroup([ + '$collection='.Database::SYSTEM_COLLECTION_EXECUTIONS + ], $projectDB); + }); + } + + protected function deleteAbuseLogs($timestamp) + { + global $register; + if($timestamp == 0) { + throw new Exception('Failed to delete audit logs. No timestamp provided'); + } + + $timeLimit = new TimeLimit("", 0, 1, function () use ($register) { + return $register->get('db'); + }); + + $this->deleteForProjectIds(function($projectId) use ($timeLimit, $timestamp){ + $timeLimit->setNamespace('app_'.$projectId); + $abuse = new Abuse($timeLimit); + + $status = $abuse->cleanup($timestamp); + if (!$status) { + throw new Exception('Failed to delete Abuse logs for project '.$projectId); + } + }); + } + + protected function deleteAuditLogs($timestamp) + { + global $register; + if($timestamp == 0) { + throw new Exception('Failed to delete audit logs. No timestamp provided'); + } + $this->deleteForProjectIds(function($projectId) use ($register, $timestamp){ + $adapter = new AuditAdapter($register->get('db')); + $adapter->setNamespace('app_'.$projectId); + $audit = new Audit($adapter); + $status = $audit->cleanup($timestamp); + if (!$status) { + throw new Exception('Failed to delete Audit logs for project'.$projectId); + } + }); + } + protected function deleteFunction(Document $document, $projectId) { $projectDB = $this->getProjectDB($projectId); @@ -142,6 +221,48 @@ class DeletesV1 Authorization::reset(); } + protected function deleteForProjectIds(callable $callback) + { + $count = 0; + $chunk = 0; + $limit = 50; + $projects = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while($sum === $limit) { + $chunk++; + + Authorization::disable(); + $projects = $this->getConsoleDB()->getCollection([ + 'limit' => $limit, + 'offset' => $count, + 'orderType' => 'ASC', + 'orderCast' => 'string', + 'filters' => [ + '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, + ], + ]); + Authorization::reset(); + + $projectIds = array_map (function ($project) { + return $project->getId(); + }, $projects); + + $sum = count($projects); + + Console::info('Executing delete function for chunk #'.$chunk.'. Found '.$sum.' projects'); + foreach ($projectIds as $projectId) { + $callback($projectId); + $count++; + } + } + + $executionEnd = \microtime(true); + Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds"); + } + protected function deleteByGroup(array $filters, Database $database, callable $callback = null) { $count = 0; @@ -159,7 +280,10 @@ class DeletesV1 $results = $database->getCollection([ 'limit' => $limit, - 'offset' => 0, + 'offset' => $count, + 'orderField' => '$id', + 'orderType' => 'ASC', + 'orderCast' => 'string', 'filters' => $filters, ]); diff --git a/app/workers/mails.php b/app/workers/mails.php index a7f6f6c235..5431ee9225 100644 --- a/app/workers/mails.php +++ b/app/workers/mails.php @@ -7,7 +7,7 @@ require_once __DIR__.'/../init.php'; \cli_set_process_title('Mails V1 Worker'); -echo APP_NAME.' mails worker v1 has started'."\n"; +Console::success(APP_NAME.' mails worker v1 has started'."\n"); class MailsV1 { diff --git a/bin/maintenance b/bin/maintenance new file mode 100644 index 0000000000..099551cb32 --- /dev/null +++ b/bin/maintenance @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php maintenance $@ \ No newline at end of file diff --git a/composer.json b/composer.json index afefca980b..bf9f354acc 100644 --- a/composer.json +++ b/composer.json @@ -35,10 +35,10 @@ "appwrite/php-clamav": "1.0.*", "utopia-php/framework": "0.10.0", - "utopia-php/abuse": "0.2.*", - "utopia-php/audit": "0.3.*", + "utopia-php/abuse": "0.3.*", + "utopia-php/audit": "0.5.*", "utopia-php/cache": "0.2.*", - "utopia-php/cli": "0.7.3", + "utopia-php/cli": "0.8.0", "utopia-php/config": "0.2.*", "utopia-php/locale": "0.3.*", "utopia-php/registry": "0.2.*", diff --git a/composer.lock b/composer.lock index f262565037..500c67d954 100644 --- a/composer.lock +++ b/composer.lock @@ -1112,16 +1112,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.2.2", + "version": "0.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "8e65890a6d7afa9f57992f1eca9fe64508f9822e" + "reference": "23c2eb533bca8f3ef5548ae265398fa7d4d39a1c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/8e65890a6d7afa9f57992f1eca9fe64508f9822e", - "reference": "8e65890a6d7afa9f57992f1eca9fe64508f9822e", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/23c2eb533bca8f3ef5548ae265398fa7d4d39a1c", + "reference": "23c2eb533bca8f3ef5548ae265398fa7d4d39a1c", "shasum": "" }, "require": { @@ -1164,16 +1164,16 @@ }, { "name": "utopia-php/audit", - "version": "0.3.2", + "version": "0.5.1", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "544ecff78788d11f60992a721f102cafc22ab084" + "reference": "154a850170a58667a15e4b65fbabb6cd0b709dd9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/544ecff78788d11f60992a721f102cafc22ab084", - "reference": "544ecff78788d11f60992a721f102cafc22ab084", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/154a850170a58667a15e4b65fbabb6cd0b709dd9", + "reference": "154a850170a58667a15e4b65fbabb6cd0b709dd9", "shasum": "" }, "require": { @@ -1268,16 +1268,16 @@ }, { "name": "utopia-php/cli", - "version": "0.7.3", + "version": "0.8", "source": { "type": "git", "url": "https://github.com/utopia-php/cli.git", - "reference": "0337918242278e0cf98f8dcab2e75b5a3153b856" + "reference": "090c7ae22b53a175d962e8f9601d36350496153c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cli/zipball/0337918242278e0cf98f8dcab2e75b5a3153b856", - "reference": "0337918242278e0cf98f8dcab2e75b5a3153b856", + "url": "https://api.github.com/repos/utopia-php/cli/zipball/090c7ae22b53a175d962e8f9601d36350496153c", + "reference": "090c7ae22b53a175d962e8f9601d36350496153c", "shasum": "" }, "require": { @@ -1875,6 +1875,7 @@ "require-dev": { "phpunit/phpunit": "^7.0" }, + "default-branch": true, "type": "library", "autoload": { "psr-4": { diff --git a/docker-compose.yml b/docker-compose.yml index 701d1e240c..9ac034e4ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -314,6 +314,27 @@ services: - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD + appwrite-maintenance: + entrypoint: maintenance + container_name: appwrite-maintenance + build: + context: . + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + environment: + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_MAINTENANCE_INTERVAL + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + appwrite-schedule: entrypoint: schedule container_name: appwrite-schedule diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index bc712d7244..b2c69337a2 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -6,6 +6,32 @@ use Resque; class Event { + + const DELETE_QUEUE_NAME = 'v1-deletes'; + const DELETE_CLASS_NAME = 'DeletesV1'; + + const AUDITS_QUEUE_NAME = 'v1-audits'; + const AUDITS_CLASS_NAME = 'AuditsV1'; + + const USAGE_QUEUE_NAME = 'v1-usage'; + const USAGE_CLASS_NAME = 'UsageV1'; + + const MAILS_QUEUE_NAME = 'v1-mails'; + const MAILS_CLASS_NAME = 'MailsV1'; + + const FUNCTIONS_QUEUE_NAME = 'v1-functions'; + const FUNCTIONS_CLASS_NAME = 'FunctionsV1'; + + const WEBHOOK_QUEUE_NAME = 'v1-webhooks'; + const WEBHOOK_CLASS_NAME = 'WebhooksV1'; + + const TASK_QUEUE_NAME = 'v1-tasks'; + const TASK_CLASS_NAME = 'TasksV1'; + + const CERTIFICATES_QUEUE_NAME = 'v1-certificates'; + const CERTIFICATES_CLASS_NAME = 'CertificatesV1'; + + /** * @var string */