From 90e83955782629c90d2e8c95b03dca2da98de354 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 14 Aug 2023 20:06:00 +0300 Subject: [PATCH] Restore.php --- src/Appwrite/Platform/Tasks/Backup.php | 40 +++--- src/Appwrite/Platform/Tasks/Restore.php | 167 ++++++++++-------------- 2 files changed, 89 insertions(+), 118 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Backup.php b/src/Appwrite/Platform/Tasks/Backup.php index 5699c176bb..b291119de4 100644 --- a/src/Appwrite/Platform/Tasks/Backup.php +++ b/src/Appwrite/Platform/Tasks/Backup.php @@ -31,8 +31,6 @@ class Backup extends Action */ public function __construct() { - $this->checkEnvVariables(); - $this ->desc('Backup a database') ->param('database', null, new Text(20), 'Database name, for example db_fra1_01') @@ -50,6 +48,8 @@ class Backup extends Action */ public function action(string $database, Group $pools): void { + $this->checkEnvVariables(); + $this->database = $database; $this->dsn = $this->getDsn($database); if (is_null($this->dsn)) { @@ -88,8 +88,8 @@ class Backup extends Action } } while ($attempts < $max); - $this->setProcessors(); $this->setContainerId(); + $this->setProcessors(); //sleep(20); @@ -151,7 +151,6 @@ class Backup extends Action '--compress=' . self::COMPRESS_ALGORITHM, '--compress-threads=' . $this->processors, '--parallel=' . intval($this->processors / 2), - //'--rsync', // https://docs.percona.com/percona-xtrabackup/8.0/accelerate-backup-process.html '> ' . $file, '2> ' . $log, ]; @@ -234,22 +233,6 @@ class Backup extends Action return null; } - public function setProcessors() - { - $stdout = ''; - $stderr = ''; - Console::execute('nproc', '', $stdout, $stderr); - if (!empty($stderr)) { - Console::error('Error setting processors: ' . $stderr); - Console::exit(); - } - - $processors = str_replace(PHP_EOL, '', $stdout); - $processors = intval($processors); - $processors = $processors === 0 ? 1 : $processors; - $this->processors = $processors; - } - public function setContainerId() { $stdout = ''; @@ -261,7 +244,6 @@ class Backup extends Action } $containerId = str_replace(PHP_EOL, '', $stdout); - if (empty($containerId)) { Console::error('Xtrabackup Container ID not found'); Console::exit(); @@ -269,4 +251,20 @@ class Backup extends Action $this->xtrabackupContainerId = $containerId; } + + public function setProcessors() + { + $stdout = ''; + $stderr = ''; + Console::execute('docker exec -i ' . $this->xtrabackupContainerId . ' nproc', '', $stdout, $stderr); + if (!empty($stderr)) { + Console::error('Error setting processors: ' . $stderr); + Console::exit(); + } + + $processors = str_replace(PHP_EOL, '', $stdout); + $processors = intval($processors); + $processors = $processors === 0 ? 1 : $processors; + $this->processors = $processors; + } } diff --git a/src/Appwrite/Platform/Tasks/Restore.php b/src/Appwrite/Platform/Tasks/Restore.php index a7a50dc8ed..613bf7c14b 100644 --- a/src/Appwrite/Platform/Tasks/Restore.php +++ b/src/Appwrite/Platform/Tasks/Restore.php @@ -25,10 +25,6 @@ class Restore extends Action public function __construct() { - $this->checkEnvVariables(); - $this->setProcessors(); - $this->setContainerId(); - $this ->desc('Restore a DB') ->param('id', '', new Text(20), 'The backup identification') @@ -44,6 +40,10 @@ class Restore extends Action public function action(string $id, string $cloud, string $database): void { + $this->checkEnvVariables(); + $this->setContainerId(); + $this->setProcessors(); + $this->database = $database; $datadir = self::DATADIR; $this->dsn = $this->getDsn($database); @@ -65,47 +65,45 @@ class Restore extends Action Console::exit(); } - if (file_exists($datadir . '/sys') || file_exists($datadir . '/appwrite')) { - Console::error('Datadir ' . $datadir . ' must be empty!'); - Console::exit(); - } +// if (file_exists($datadir . '/sys') || file_exists($datadir . '/appwrite')) { +// Console::error('Datadir ' . $datadir . ' must be empty!'); +// Console::exit(); +// } $this->log('--- Restore Start ' . $id . ' --- '); - - $filename = $id . '.tar.gz'; + $filename = $id . '.xbstream'; $start = microtime(true); $cloud = $cloud === 'true' || $cloud === '1'; + $destination = new Local(self::BACKUPS_PATH . '/restore/' . $id); + $files = $destination->getRoot() . '/files'; - if ($cloud) { - $local = new Local(self::BACKUPS_PATH . '/downloads/' . $id); - - $files = $local->getRoot() . '/files'; - - if (!file_exists($files) && !mkdir($files, 0755, true)) { - Console::error('Error creating directory: ' . $files); - Console::exit(); - } - - $file = $local->getPath($filename); - - if (!file_exists($file)) { - $this->download($file, $local); - } - - $this->untar($file, $files); - } else { - $local = new Local(self::BACKUPS_PATH . '/' . $database . '/full/' . $id); - $files = $local->getRoot() . '/files'; - } - - if (!file_exists($files)) { - Console::error('Directory not found: ' . $files); + if (file_exists($files)) { + Console::error('Directory exist: ' . $files); Console::exit(); } - $this->decompress($files); + if (!file_exists($files) && !mkdir($files, 0755, true)) { + Console::error('Error creating directory: ' . $files); + Console::exit(); + } + + if ($cloud) { + $stream = $destination->getPath($filename); + if (!file_exists($stream)) { + $this->download($stream, $destination); + } + } else { + $stream = self::BACKUPS_PATH . '/' . $database . '/full/' . $filename; + } + + if (!file_exists($stream)) { + Console::error('File not found: ' . $stream); + Console::exit(); + } + + $this->decompress($stream, $files); $this->prepare($files); - $this->restore($files, $cloud, $datadir); + $this->restore($files, $datadir); $this->log('Restore Finish in ' . (microtime(true) - $start) . ' seconds'); } @@ -133,52 +131,24 @@ class Restore extends Action } } - public function untar(string $file, string $directory) + public function decompress(string $file, string $target) { - $this->log('Untar Start'); + $this->log('Xbstream start'); + + $args = [ + 'xbstream -x < ' . $file, + '--decompress', + '--parallel=' . $this->processors, + '-C ' . $target, + ]; $stdout = ''; $stderr = ''; - $cmd = 'tar -xzf ' . $file . ' -C ' . $directory; + $cmd = 'docker exec -i ' . $this->xtrabackupContainerId . ' ' . implode(' ', $args); + Console::success($cmd); Console::execute($cmd, '', $stdout, $stderr); if (!empty($stderr)) { - Console::error($stderr); - Console::exit(); - } - - if (!file_exists($file)) { - Console::error('Restore file not found: ' . $file); - Console::exit(); - } - } - - public function decompress(string $target) - { - $this->log('Decompress start'); - - $logfile = $target . '/../log.txt'; - - $args = [ - 'xtrabackup', - '--user=' . $this->dsn->getUser(), - '--password=' . $this->dsn->getPassword(), - '--host=' . $this->dsn->getHost(), - '--decompress', - '--strict', - '--remove-original', // Removes *.lz4 compressed files - '--parallel=' . $this->processors, - '--compress-threads=' . $this->processors, - '--target-dir=' . $target, - '2> ' . $logfile, - ]; - - $cmd = 'docker exec ' . $this->xtrabackupContainerId . ' ' . implode(' ', $args); - shell_exec($cmd); - - $stderr = shell_exec('tail -1 ' . $logfile); - - if (!str_contains($stderr, 'completed OK!')) { - Console::error('Decompress failed'); + Console::error('Xbstream Error: ' . $stderr); Console::exit(); } } @@ -192,7 +162,7 @@ class Restore extends Action Console::exit(); } - $logfile = $target . '/../log.txt'; + $logfile = $target . '/../prepare.txt'; $args = [ 'xtrabackup', @@ -211,12 +181,12 @@ class Restore extends Action $stderr = shell_exec('tail -1 ' . $logfile); if (!str_contains($stderr, 'completed OK!')) { - Console::error(date('Y-m-d H:i:s') . ' Prepare failed:' . $stderr); + Console::error('Prepare error: ' . $stderr); Console::exit(); } } - public function restore(string $target, bool $cloud, string $datadir) + public function restore(string $target, string $datadir) { $this->log('Restore start'); @@ -225,14 +195,14 @@ class Restore extends Action Console::exit(); } - $logfile = $target . '/../log.txt'; + $logfile = $target . '/../restore.txt'; $args = [ 'xtrabackup', '--user=' . $this->dsn->getUser(), '--password=' . $this->dsn->getPassword(), '--host=' . $this->dsn->getHost(), - $cloud ? '--move-back' : '--copy-back', + '--move-back', '--strict', '--target-dir=' . $target, '--datadir=' . $datadir, @@ -253,6 +223,9 @@ class Restore extends Action public function checkEnvVariables(): void { + + var_dump(self::getName()); + foreach ( [ '_APP_CONNECTIONS_BACKUPS_STORAGE', @@ -284,22 +257,6 @@ class Restore extends Action return null; } - public function setProcessors() - { - $stdout = ''; - $stderr = ''; - Console::execute('nproc', '', $stdout, $stderr); - if (!empty($stderr)) { - Console::error('Error setting processors: ' . $stderr); - Console::exit(); - } - - $processors = str_replace(PHP_EOL, '', $stdout); - $processors = intval($processors); - $processors = $processors === 0 ? 1 : $processors; - $this->processors = $processors; - } - public function setContainerId() { $stdout = ''; @@ -318,4 +275,20 @@ class Restore extends Action $this->xtrabackupContainerId = $containerId; } + + public function setProcessors() + { + $stdout = ''; + $stderr = ''; + Console::execute('docker exec -i ' . $this->xtrabackupContainerId . ' nproc', '', $stdout, $stderr); + if (!empty($stderr)) { + Console::error('Error setting processors: ' . $stderr); + Console::exit(); + } + + $processors = str_replace(PHP_EOL, '', $stdout); + $processors = intval($processors); + $processors = $processors === 0 ? 1 : $processors; + $this->processors = $processors; + } }