From 2f8b526c01b1749da3ce83f41fc4ab63fe71b201 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 1 Aug 2023 10:53:51 +0300 Subject: [PATCH] Restore.php --- .env | 3 +- docker-compose.yml | 1 + src/Appwrite/Platform/Tasks/Backup.php | 14 +- src/Appwrite/Platform/Tasks/Restore.php | 228 +++++++++++++++++------- 4 files changed, 178 insertions(+), 68 deletions(-) diff --git a/.env b/.env index b7868d805e..cc3073cadc 100644 --- a/.env +++ b/.env @@ -84,5 +84,6 @@ _DO_SPACES_BUCKET_NAME=backups-v1 _DO_SPACES_REGION=fra1 _DO_SPACES_ACCESS_KEY= _DO_SPACES_SECRET_KEY= - +_DO_SPACES_ACCESS_KEY= +_DO_SPACES_SECRET_KEY= _APP_BACKUP_FOLDER=daily \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index dc5cb07dd1..9f120f7308 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -661,6 +661,7 @@ services: - _DO_SPACES_SECRET_KEY - _DO_SPACES_REGION - _APP_BACKUP_FOLDER + - _APP_DB_ROOT_PASS appwrite-usage: entrypoint: usage diff --git a/src/Appwrite/Platform/Tasks/Backup.php b/src/Appwrite/Platform/Tasks/Backup.php index 767dafb3f7..9f8eb5aa3b 100644 --- a/src/Appwrite/Platform/Tasks/Backup.php +++ b/src/Appwrite/Platform/Tasks/Backup.php @@ -36,7 +36,7 @@ class Backup extends Action self::log('--- Backup Start --- '); $start = microtime(true); $type = 'inc'; - //$type = 'full'; + $type = 'full'; switch ($type) { case 'inc': @@ -53,7 +53,7 @@ class Backup extends Action $local->setTransferChunkSize(5 * 1024 * 1024); // > 5MB $backups = $local->getRoot() . '/files'; - $tarFile = $local->getRoot() . '/' . $filename; + $tarFile = $local->getPath($filename); $this->fullBackup( backups: $backups, @@ -78,6 +78,11 @@ class Backup extends Action } self::log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- ' . PHP_EOL . PHP_EOL); + + Console::loop(function () { + self::log('loop'); + }, 2 * 60); + } public function fullBackup(string $backups) @@ -94,7 +99,7 @@ class Backup extends Action $args = [ '--user=root', - '--password=rootsecretpassword', // todo use .env + '--password=' . App::getEnv('_APP_DB_ROOT_PASS'), '--host=mariadb', '--backup', //'--compress', @@ -373,7 +378,8 @@ class Backup extends Action '_DO_SPACES_BUCKET_NAME', '_DO_SPACES_ACCESS_KEY', '_DO_SPACES_SECRET_KEY', - '_DO_SPACES_REGION' + '_DO_SPACES_REGION', + '_APP_DB_ROOT_PASS' ] as $env ) { if (empty(App::getEnv($env))) { diff --git a/src/Appwrite/Platform/Tasks/Restore.php b/src/Appwrite/Platform/Tasks/Restore.php index de10d45ce6..3b83a49b54 100644 --- a/src/Appwrite/Platform/Tasks/Restore.php +++ b/src/Appwrite/Platform/Tasks/Restore.php @@ -6,9 +6,9 @@ use Exception; use Utopia\App; use Utopia\Platform\Action; use Utopia\CLI\Console; +use Utopia\Storage\Device; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Local; -use Utopia\Storage\Storage; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; @@ -28,90 +28,191 @@ class Restore extends Action { $this ->desc('Restore a DB') - ->param('filename', '', new Text(100), 'Backup file name') + ->param('id', '', new Text(100), 'Folder Identifier') ->param('cloud', null, new WhiteList(['true', 'false'], true), 'Take file from cloud?') - ->param('project', null, new WhiteList(['db_fra1_02', 'shimon'], true), 'From _APP_CONNECTIONS_DB_PROJECT') + ->param('project', null, new WhiteList(['db_fra1_02'], true), 'From _APP_CONNECTIONS_DB_PROJECT') ->param('folder', null, new WhiteList(['hourly', 'daily'], true), 'Sub folder') - ->callback(fn ($file, $cloud, $project, $folder) => $this->action($file, $cloud, $project, $folder)); + ->callback(fn ($id, $cloud, $project, $folder) => $this->action($id, $cloud, $project, $folder)); } - /** - * @throws Exception - */ - public function action(string $filename, string $cloud, string $project, string $folder): void + public function action(string $id, string $cloud, string $project, string $folder): void { $this->checkEnvVariables(); - + $filename = $id . '.tar.gz'; Backup::log('--- Restore Start ' . $filename . ' --- '); $start = microtime(true); - $cloud = $cloud === 'true'; - $file = Backup::$backups . '/' . $project . '/' . $folder . '/' . $filename; - $s3 = new DOSpaces('/v1/' . $project . '/' . $folder, App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); - $download = new Local(Backup::$backups . '/downloads'); - if (!file_exists($download->getRoot()) && !mkdir($download->getRoot(), 0755, true)) { - Console::error('Error creating directory: ' . $download->getRoot()); - Console::exit(); - } + $local = new Local(Backup::$backups . '/' . $project . '/' . $folder . '/' . $id); + $files = $local->getRoot() . '/files'; if ($cloud) { - try { - $path = $s3->getPath($filename); + $local = new Local(Backup::$backups . '/downloads/' . $id); + $this->download($project, $folder, $filename, $local); + $files = $local->getRoot() . '/files'; + if (!file_exists($files) && !mkdir($files, 0755, true)) { + Console::error('Error creating directory: ' . $files); + Console::exit(); + } - if (!$s3->exists($path)) { - Console::error('File: ' . $path . ' does not exist on cloud'); - Console::exit(); - } + $file = $local->getPath($filename); - $file = $download->getPath($filename); - Backup::log('Downloading: ' . $file); + $stdout = ''; + $stderr = ''; + $cmd = 'tar -xzf ' . $file . ' -C ' . $files; + Backup::log($cmd); + Console::execute($cmd, '', $stdout, $stderr); + if (!empty($stderr)) { + Console::error($stderr); + Console::exit(); + } - if (!$s3->transfer($path, $file, $download)) { - Console::error('Error Downloading ' . $file); - Console::exit(); - } - } catch (Exception $e) { - Console::error($e->getMessage()); + if (!file_exists($file)) { + Console::error('Restore file not found: ' . $file); Console::exit(); } } - if (!file_exists($file)) { - Console::error('Restore file not found: ' . $file); - Console::exit(); - } - - Backup::stopMysqlContainer($this->containerName); - - $stdout = ''; - $stderr = ''; - //$cmd = 'mv ' . Backup::$mysqlDirectory . '/* ' . ' ' . $original . '/'; - // todo: do we care about original? - $cmd = 'rm -r ' . Backup::$mysqlDirectory . '/*'; - Backup::log($cmd); - Console::execute($cmd, '', $stdout, $stderr); - Backup::log($stdout); - if (!empty($stderr)) { - Console::error($stderr); - Console::exit(); - } - - $stdout = ''; - $stderr = ''; - $cmd = 'tar -xzf ' . $file . ' -C ' . Backup::$mysqlDirectory; - Backup::log($cmd); - Console::execute($cmd, '', $stdout, $stderr); - if (!empty($stderr)) { - Console::error($stderr); - Console::exit(); - } - - Backup::startMysqlContainer($this->containerName); + $this->prepare($files); Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds"); } + public function download(string $project, string $folder, string $filename, Device $local) + { + if (!file_exists($local->getRoot()) && !mkdir($local->getRoot(), 0755, true)) { + Console::error('Error creating directory: ' . $local->getRoot()); + Console::exit(); + } + + $s3 = new DOSpaces($project . '/' . $folder, App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); + + try { + $path = $s3->getPath($filename); + + if (!$s3->exists($path)) { + Console::error('File: ' . $path . ' does not exist on cloud'); + Console::exit(); + } + + $file = $local->getPath($filename); + Backup::log('Downloading: ' . $file); + + if (!$s3->transfer($path, $file, $local)) { + Console::error('Error Downloading ' . $file); + Console::exit(); + } + } catch (Exception $e) { + Console::error($e->getMessage()); + Console::exit(); + } + } + + public function prepare(string $target) + { + if (!file_exists($target)) { + Console::error('$target directory does not exist'); + Console::exit(); + } + + $args = [ + '--user=root', + '--password=' . App::getEnv('_APP_DB_ROOT_PASS'), + '--host=mariadb', + '--prepare', + '--target-dir=' . $target, + ]; + + $stdout = ''; + $stderr = ''; + $cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args); + Backup::log($cmd); + Console::execute($cmd, '', $stdout, $stderr); + if (!empty($stderr)) { + Console::error($stderr); + //Console::exit(); + } + + if (!str_contains($stderr, 'completed OK!')) { + Console::error('Error preparing $target = ' . $target); + Console::exit(); + } + } + +// public function action(string $filename, string $cloud, string $project, string $folder): void +// { +// $this->checkEnvVariables(); +// +// Backup::log('--- Restore Start ' . $filename . ' --- '); +// $start = microtime(true); +// +// $cloud = $cloud === 'true'; +// $file = Backup::$backups . '/' . $project . '/' . $folder . '/' . $filename; +// $s3 = new DOSpaces('/v1/' . $project . '/' . $folder, App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); +// $download = new Local(Backup::$backups . '/downloads'); +// +// if (!file_exists($download->getRoot()) && !mkdir($download->getRoot(), 0755, true)) { +// Console::error('Error creating directory: ' . $download->getRoot()); +// Console::exit(); +// } +// +// if ($cloud) { +// try { +// $path = $s3->getPath($filename); +// +// if (!$s3->exists($path)) { +// Console::error('File: ' . $path . ' does not exist on cloud'); +// Console::exit(); +// } +// +// $file = $download->getPath($filename); +// Backup::log('Downloading: ' . $file); +// +// if (!$s3->transfer($path, $file, $download)) { +// Console::error('Error Downloading ' . $file); +// Console::exit(); +// } +// } catch (Exception $e) { +// Console::error($e->getMessage()); +// Console::exit(); +// } +// } +// +// if (!file_exists($file)) { +// Console::error('Restore file not found: ' . $file); +// Console::exit(); +// } +// +// Backup::stopMysqlContainer($this->containerName); +// +// $stdout = ''; +// $stderr = ''; +// //$cmd = 'mv ' . Backup::$mysqlDirectory . '/* ' . ' ' . $original . '/'; +// // todo: do we care about original? +// $cmd = 'rm -r ' . Backup::$mysqlDirectory . '/*'; +// Backup::log($cmd); +// Console::execute($cmd, '', $stdout, $stderr); +// Backup::log($stdout); +// if (!empty($stderr)) { +// Console::error($stderr); +// Console::exit(); +// } +// +// $stdout = ''; +// $stderr = ''; +// $cmd = 'tar -xzf ' . $file . ' -C ' . Backup::$mysqlDirectory; +// Backup::log($cmd); +// Console::execute($cmd, '', $stdout, $stderr); +// if (!empty($stderr)) { +// Console::error($stderr); +// Console::exit(); +// } +// +// Backup::startMysqlContainer($this->containerName); +// +// Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds"); +// } + public function checkEnvVariables(): void { foreach ( @@ -119,7 +220,8 @@ class Restore extends Action '_DO_SPACES_BUCKET_NAME', '_DO_SPACES_ACCESS_KEY', '_DO_SPACES_SECRET_KEY', - '_DO_SPACES_REGION' + '_DO_SPACES_REGION', + '_APP_DB_ROOT_PASS' ] as $env ) { if (empty(App::getEnv($env))) {