diff --git a/src/Appwrite/Platform/Tasks/Backup.php b/src/Appwrite/Platform/Tasks/Backup.php index 2db4c0b67e..7b0af9d142 100644 --- a/src/Appwrite/Platform/Tasks/Backup.php +++ b/src/Appwrite/Platform/Tasks/Backup.php @@ -16,13 +16,13 @@ use Utopia\Validator\Text; class Backup extends Action { public const BACKUPS_PATH = '/backups'; - public const BACKUP_INTERVAL_SECONDS = 60 * 60 * 1; // 4 hours; - public const COMPRESS_ALGORITHM = 'LZ4'; // faster compression and decompression + public const BACKUP_INTERVAL_SECONDS = 60 * 60 * 4; // 4 hours; + public const COMPRESS_ALGORITHM = 'zstd'; // https://www.percona.com/blog/get-your-backup-to-half-of-its-size-introducing-zstd-support-in-percona-xtrabackup/ protected ?DSN $dsn = null; protected ?string $database = null; protected ?DOSpaces $s3 = null; protected string $xtrabackupContainerId; - protected int $processors = 1; + protected int $processors; /** * @throws Exception @@ -91,7 +91,7 @@ class Backup extends Action $this->setContainerId(); $this->setProcessors(); - sleep(20); + //sleep(20); Console::loop(function () { $this->start(); @@ -102,7 +102,6 @@ class Backup extends Action { $start = microtime(true); $time = date('Y_m_d_H_i_s'); - $this->filename = $time . '.xbstream'; self::log('--- Backup Start ' . $time . ' --- '); @@ -132,6 +131,7 @@ class Backup extends Action Console::exit(); } + $filename = basename($target); $logfile = $target . '/../backup.log'; $args = [ @@ -142,15 +142,15 @@ class Backup extends Action '--port=' . $this->dsn->getPort(), '--backup', '--strict', - '--history="' . $this->database . '|' . pathinfo($this->filename, PATHINFO_FILENAME) . '"', // PERCONA_SCHEMA.xtrabackup_history + '--history="' . $this->database . '|' . pathinfo($filename, PATHINFO_FILENAME) . '"', // PERCONA_SCHEMA.xtrabackup_history '--slave-info', '--safe-slave-backup', '--safe-slave-backup-timeout=300', '--check-privileges', // checks if Percona XtraBackup has all the required privileges. '--target-dir=' . $target, - '--parallel=' . $this->processors, '--compress=' . self::COMPRESS_ALGORITHM, - '--compress-threads=' . $this->processors, + '--compress-threads=' . intval($this->processors / 2), + '--parallel=' . $this->processors, '--rsync', // https://docs.percona.com/percona-xtrabackup/8.0/accelerate-backup-process.html '2> ' . $logfile, ]; @@ -207,23 +207,23 @@ class Backup extends Action Console::exit(); } - try { - $destination = $this->s3->getRoot() . '/' . $filename; + $destination = $this->s3->getRoot() . '/' . $filename; + try { if (!$local->transfer($file, $destination, $this->s3)) { Console::error('Error uploading to ' . $destination); Console::exit(); } - - if (!$this->s3->exists($destination)) { - Console::error('File not found in destination: ' . $destination); - Console::exit(); - } } catch (Exception $e) { Console::error($e->getMessage()); Console::exit(); } + if (!$this->s3->exists($destination)) { + Console::error('File not found in destination: ' . $destination); + Console::exit(); + } + if (!unlink($file)) { Console::error('Error deleting: ' . $file); Console::exit(); diff --git a/src/Appwrite/Platform/Tasks/Restore.php b/src/Appwrite/Platform/Tasks/Restore.php index a132b9e5a8..d6948f5089 100644 --- a/src/Appwrite/Platform/Tasks/Restore.php +++ b/src/Appwrite/Platform/Tasks/Restore.php @@ -47,6 +47,7 @@ class Restore extends Action try { $dsn = new DSN(App::getEnv('_APP_CONNECTIONS_BACKUPS_STORAGE', '')); $this->s3 = new DOSpaces('/' . $database . '/full', $dsn->getUser(), $dsn->getPassword(), $dsn->getPath(), $dsn->getParam('region')); + $this->s3->setTransferChunkSize(40 * 1024 * 1024); // 5MB } catch (\Exception $e) { Console::error($e->getMessage() . 'Invalid DSN.'); Console::exit(); @@ -59,7 +60,7 @@ class Restore extends Action if (file_exists($datadir . '/sys') || file_exists($datadir . '/appwrite')) { Console::error('Datadir ' . $datadir . ' must be empty!'); - Console::exit(); + //Console::exit(); } $this->log('--- Restore Start ' . $id . ' --- '); @@ -97,7 +98,7 @@ class Restore extends Action $this->decompress($files); $this->prepare($files); - $this->restore($files, $cloud, $datadir); + // $this->restore($files, $cloud, $datadir); $this->log('Restore Finish in ' . (microtime(true) - $start) . ' seconds'); } @@ -154,9 +155,9 @@ class Restore extends Action 'xtrabackup', '--decompress', '--strict', - '--remove-original', // Removes *.lz4 compressed files + '--remove-original', '--parallel=' . $this->processors, - '--compress-threads=' . $this->processors, + '--compress-threads=' . intval($this->processors / 2), '--target-dir=' . $target, '2> ' . $logfile, ]; @@ -186,7 +187,7 @@ class Restore extends Action $args = [ 'xtrabackup', '--prepare', - '--parallel=' . intval($this->processors / 2), + '--parallel=' . $this->processors, '--strict', '--target-dir=' . $target, '2> ' . $logfile, @@ -218,7 +219,7 @@ class Restore extends Action 'xtrabackup', $cloud ? '--move-back' : '--copy-back', '--strict', - '--parallel=' . intval($this->processors / 2), + '--parallel=' . $this->processors, '--target-dir=' . $target, '--datadir=' . $datadir, '2> ' . $logfile,