From 394296f2775bc74804e8355416abcc384c4a1043 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 25 Jul 2022 08:09:55 +0000 Subject: [PATCH] encryption worker and base encryption key rotation flow --- Dockerfile | 2 ++ app/config/collections.php | 11 ++++++ app/controllers/api/projects.php | 10 +----- app/init.php | 2 ++ app/tasks/maintenance.php | 26 +++++++++++++- app/workers/encryption.php | 59 ++++++++++++++++++++++++++++++++ bin/worker-encryption | 10 ++++++ composer.lock | 39 ++++++++------------- docker-compose.yml | 26 ++++++++++++++ src/Appwrite/Event/Encrypt.php | 52 ++++++++++++++++++++++++++++ src/Appwrite/Event/Event.php | 3 ++ 11 files changed, 206 insertions(+), 34 deletions(-) create mode 100644 app/workers/encryption.php create mode 100644 bin/worker-encryption create mode 100644 src/Appwrite/Event/Encrypt.php diff --git a/Dockerfile b/Dockerfile index 29d495ec26..1ffee1be3d 100755 --- a/Dockerfile +++ b/Dockerfile @@ -152,6 +152,7 @@ ENV _APP_SERVER=swoole \ _APP_OPTIONS_ABUSE=enabled \ _APP_OPTIONS_FORCE_HTTPS=disabled \ _APP_OPENSSL_KEY_V1=your-secret-key \ + _APP_KEY_ROTATION_INTERVAL=7776000 \ _APP_STORAGE_LIMIT=10000000 \ _APP_STORAGE_ANTIVIRUS=enabled \ _APP_STORAGE_ANTIVIRUS_HOST=clamav \ @@ -308,6 +309,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-builds && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-messaging && \ + chmod +x /usr/local/bin/worker-encryption && \ chmod +x /usr/local/bin/worker-webhooks # Letsencrypt Permissions diff --git a/app/config/collections.php b/app/config/collections.php index 52d9e93bbf..4cd4d4418e 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -723,6 +723,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'keyRotationDate', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ] ], 'indexes' => [ [ diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 5e1a7d1cd5..83e3e86116 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -92,15 +92,6 @@ App::post('/v1/projects') 'secret' => OpenSSL::secretString(), ]))); - // create new secret for the project - // project should save keyId - // need setFilter to set instance level filter in dbforconsole - // saving keyId in project doesn't make sense, as we only need the secret to - // read and write the project document - - // The problem is we need to know the projectId to set the filter - // but when we get dbForConsole we will never know the project id to work with - $project = $dbForConsole->createDocument('projects', new Document([ '$id' => $projectId, '$read' => ['team:' . $teamId], @@ -110,6 +101,7 @@ App::post('/v1/projects') 'teamId' => $team->getId(), 'description' => $description, 'keyId' => $secret->getId(), + 'keyRotationDate' => time() + App::getEnv('_APP_KEY_ROTATION_INTERVAL', 60 * 60 * 24 * 90), 'logo' => $logo, 'url' => $url, 'version' => APP_VERSION_STABLE, diff --git a/app/init.php b/app/init.php index 039222db67..781fcbfdfb 100644 --- a/app/init.php +++ b/app/init.php @@ -154,6 +154,8 @@ const APP_AUTH_TYPE_SESSION = 'Session'; const APP_AUTH_TYPE_JWT = 'JWT'; const APP_AUTH_TYPE_KEY = 'Key'; const APP_AUTH_TYPE_ADMIN = 'Admin'; +// Encryption rotation types +const APP_ENCRYPTION_TYPE_PROJECT_MASTER_KEY = 'projectMasterKey'; // Response related const MAX_OUTPUT_CHUNK_SIZE = 2 * 1024 * 1024; // 2MB diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index eb50dbc0e2..4c7163e1ec 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -6,6 +6,7 @@ global $register; use Appwrite\Auth\Auth; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; +use Appwrite\Event\Encrypt; use Utopia\App; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -102,7 +103,7 @@ $cli ->trigger(); } - function renewCertificates($dbForConsole) + function renewCertificates(Database $dbForConsole) { $time = date('d-m-Y H:i:s', time()); $certificates = $dbForConsole->find('certificates', [ @@ -127,6 +128,29 @@ $cli } } + function rotateKeys(Database $dbForConsole) + { + $time = date('d-m-Y H:i:s', time()); + $projects = $dbForConsole->find('projects', [ + new Query('keyRotationDate', Query::TYPE_LESSEREQUAL, [\time()]) + ], 200); + + if (\count($projects) > 0) { + Console::info("[{$time}] Found " . \count($projects) . " projects for key rotation, scheduling jobs."); + + $event = new Encrypt(); + foreach ($projects as $project) { + $event + ->setType(APP_ENCRYPTION_TYPE_PROJECT_MASTER_KEY) + ->setProject($project) + ->trigger(); + } + } else { + Console::info("[{$time}] No projects for key rotation."); + } + + } + // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); diff --git a/app/workers/encryption.php b/app/workers/encryption.php new file mode 100644 index 0000000000..582953ac36 --- /dev/null +++ b/app/workers/encryption.php @@ -0,0 +1,59 @@ +args['type'] ?? ''; + + switch($type) { + case APP_ENCRYPTION_TYPE_PROJECT_MASTER_KEY: + $project = new Document($this->args['project'] ?? []); + $this->rotateMasterKeyForProject($project); + break; + default: + Console::error('No encryption operation type: ' . $type); + } + } + + protected function rotateMasterKeyForProject(Document $project): void + { + $projectId = $project->getId(); + $dbForConsole = $this->getConsoleDB(); + $oldKey = $project->getAttribute('keyId', ''); + $secret = Authorization::skip(fn() => $dbForConsole->createDocument('secrets', new Document([ + '$id' => $dbForConsole->getId(), + '$read' => [], + '$write' => [], + '$collection' => 'secrets', + 'secret' => OpenSSL::secretString(), + ]))); + $dbForConsole->updateDocument('projects', $projectId, + $project + ->setAttribute('keyId', $secret->getId()) + ->setAttribute('keyRotationDate', time() + App::getEnv('_APP_KEY_ROTATION_INTERVAL', 60 * 60 * 24 * 90)) + ); + } +} \ No newline at end of file diff --git a/bin/worker-encryption b/bin/worker-encryption new file mode 100644 index 0000000000..f26d066684 --- /dev/null +++ b/bin/worker-encryption @@ -0,0 +1,10 @@ +#!/bin/sh + +if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +INTERVAL=1 QUEUE='v1-encryption' APP_INCLUDE='/usr/src/code/app/workers/encryption.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/composer.lock b/composer.lock index 379a3727a6..2b6c6e6e60 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "380d806f7540199698d12a7abeb13534", + "content-hash": "677b1b47c8567f0b7b05645e2bbc7bc7", "packages": [ { "name": "adhocore/jwt", @@ -2051,16 +2051,16 @@ }, { "name": "utopia-php/database", - "version": "0.18.7", + "version": "0.18.9", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "d542ee433f1a545d926ffaf707bdf952dc18a52e" + "reference": "227b3ca919149b7b0d6556c8effe9ee46ed081e6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/d542ee433f1a545d926ffaf707bdf952dc18a52e", - "reference": "d542ee433f1a545d926ffaf707bdf952dc18a52e", + "url": "https://api.github.com/repos/utopia-php/database/zipball/227b3ca919149b7b0d6556c8effe9ee46ed081e6", + "reference": "227b3ca919149b7b0d6556c8effe9ee46ed081e6", "shasum": "" }, "require": { @@ -2109,9 +2109,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.18.7" + "source": "https://github.com/utopia-php/database/tree/0.18.9" }, - "time": "2022-07-11T10:20:33+00:00" + "time": "2022-07-19T09:42:53+00:00" }, { "name": "utopia-php/domains", @@ -2387,16 +2387,16 @@ }, { "name": "utopia-php/orchestration", - "version": "dev-cli-lib-upgrade", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/orchestration.git", - "reference": "06f2afef516aca900ddb483689ebe6f8e7037d28" + "reference": "94263976413871efb6b16157a7101a81df3b6d78" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/06f2afef516aca900ddb483689ebe6f8e7037d28", - "reference": "06f2afef516aca900ddb483689ebe6f8e7037d28", + "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/94263976413871efb6b16157a7101a81df3b6d78", + "reference": "94263976413871efb6b16157a7101a81df3b6d78", "shasum": "" }, "require": { @@ -2436,9 +2436,9 @@ ], "support": { "issues": "https://github.com/utopia-php/orchestration/issues", - "source": "https://github.com/utopia-php/orchestration/tree/cli-lib-upgrade" + "source": "https://github.com/utopia-php/orchestration/tree/0.6.0" }, - "time": "2022-07-13T14:55:12+00:00" + "time": "2022-07-13T16:47:18+00:00" }, { "name": "utopia-php/preloader", @@ -5346,18 +5346,9 @@ "time": "2022-05-17T05:48:52+00:00" } ], - "aliases": [ - { - "package": "utopia-php/orchestration", - "version": "dev-cli-lib-upgrade", - "alias": "0.4.1", - "alias_normalized": "0.4.1.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/orchestration": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index 20d2c5c2f9..a60bb2a005 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -550,6 +550,32 @@ services: - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + appwrite-worker-encryption: + entrypoint: worker-encryption + <<: *x-logging + container_name: appwrite-worker-encryption + build: + context: . + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + environment: + - _APP_ENV + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + appwrite-maintenance: entrypoint: maintenance <<: *x-logging diff --git a/src/Appwrite/Event/Encrypt.php b/src/Appwrite/Event/Encrypt.php new file mode 100644 index 0000000000..0fed8d57eb --- /dev/null +++ b/src/Appwrite/Event/Encrypt.php @@ -0,0 +1,52 @@ +type = $type; + + return $this; + } + + /** + * Returns the set type for the delete event. + * + * @return string + */ + public function getType(): string + { + return $this->type; + } + + /** + * Executes this event and sends it to the deletes worker. + * + * @return string|bool + * @throws \InvalidArgumentException + */ + public function trigger(): string|bool + { + return Resque::enqueue($this->queue, $this->class, [ + 'project' => $this->project, + 'type' => $this->type, + ]); + } +} diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 222cf59444..44c3452dd5 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -35,6 +35,9 @@ class Event public const MESSAGING_QUEUE_NAME = 'v1-messaging'; public const MESSAGING_CLASS_NAME = 'MessagingV1'; + public const ENCRYPTION_QUEUE_NAME = 'v1-encryption'; + public const ENCRYPTION_CLASS_NAME = 'EncryptionV1'; + protected string $queue = ''; protected string $class = ''; protected string $event = '';