diff --git a/.env b/.env index 570a7466c3..cfa4d1b30a 100644 --- a/.env +++ b/.env @@ -27,7 +27,8 @@ _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword _APP_CONNECTIONS_MAX=3100 _APP_POOL_CLIENTS=14 -_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://root:rootsecretpassword@mariadb:3306/appwrite +_APP_CONNECTIONS_DB_REPLICAS=db_fra1_02=mariadb://root:rootsecretpassword@mariadb:3306/appwrite +_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379 @@ -82,5 +83,3 @@ _APP_CONSOLE_GITHUB_SECRET= _APP_CONSOLE_GITHUB_APP_ID= _DO_SPACES_BUCKET_NAME=backups-v1 _DO_SPACES_REGION=fra1 -_DO_SPACES_ACCESS_KEY= -_DO_SPACES_SECRET_KEY= \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4c9b0e018f..3b78272b79 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -721,7 +721,7 @@ services: - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG mariadb: - image: mysql:8.0.33 # fix issues when upgrading using: mysql_upgrade -u root -p + image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p container_name: appwrite-mariadb <<: *x-logging networks: @@ -770,12 +770,12 @@ services: networks: - appwrite volumes: + - /var/run/docker.sock:/var/run/docker.sock - ./app:/usr/src/code/app - ./src:/usr/src/code/src - appwrite-mariadb:/var/lib/mysql:r - backup-data:/backups:rw - - /var/run/docker.sock:/var/run/docker.sock - - ./dev:/usr/local/dev + depends_on: - redis environment: @@ -792,12 +792,11 @@ services: networks: - appwrite volumes: + - /var/run/docker.sock:/var/run/docker.sock - ./app:/usr/src/code/app - ./src:/usr/src/code/src - backup-data:/backups:rw - restore-data:/varlibmysql:rw - - /var/run/docker.sock:/var/run/docker.sock - - ./dev:/usr/local/dev depends_on: - redis environment: diff --git a/src/Appwrite/Platform/Tasks/Backup.php b/src/Appwrite/Platform/Tasks/Backup.php index 50600f1174..b3cbfe7a44 100644 --- a/src/Appwrite/Platform/Tasks/Backup.php +++ b/src/Appwrite/Platform/Tasks/Backup.php @@ -15,39 +15,44 @@ use Utopia\Validator\Text; class Backup extends Action { + public const BACKUPS_PATH = '/backups'; + public const BACKUP_INTERVAL_SECONDS = 60 * 60 * 4; // 4 hours; + public const COMPRESS_ALGORITHM = 'lz4'; + public const CONFIG_PATH = '/etc/my.cnf'; + public const PROCESSORS = 4; protected ?DSN $dsn = null; protected ?string $database = null; - public const BACKUPS = '/backups'; - public const BACKUP_INTERVAL = 60 * 60 * 4; // 4 hours; - public const COMPRESS_ALGORITHM = 'lz4'; - public const CNF = '/etc/my.cnf'; - public const PROCESSORS = 4; - - public static function getName(): string - { - return 'backup'; - } + protected ?DOSpaces $s3 = null; + /** + * @throws Exception + */ public function __construct() { $this->checkEnvVariables(); $this ->desc('Backup a DB') - ->param('database', null, new Text(20), 'passed from command') + ->param('database', null, new Text(20), 'Database name for example db_fra1_01') ->inject('pools') ->callback(fn(string $database, Group $pools) => $this->action($database, $pools)); } + public static function getName(): string + { + return 'backup'; + } + /** * @throws Exception */ - public function action($database, Group $pools): void + public function action(string $database, Group $pools): void { $this->database = $database; $this->dsn = self::getDsn($database); + $this->s3 = new DOSpaces('/' . $this->database . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); - if (!$this->dsn instanceof DSN) { + if (is_null($this->dsn)) { Console::error('No DSN match'); Console::exit(); } @@ -62,8 +67,7 @@ class Backup extends Action $pools ->get('database_' . $database) ->pop() - ->getResource() - ; + ->getResource(); break; // leave the do-while if successful } catch (Exception $e) { @@ -78,7 +82,7 @@ class Backup extends Action Console::loop(function () { $this->start(); - }, self::BACKUP_INTERVAL); + }, self::BACKUP_INTERVAL_SECONDS); } public function start(): void @@ -90,7 +94,7 @@ class Backup extends Action self::log('--- Creating backup ' . $time . ' --- '); $filename = $time . '.tar.gz'; - $local = new Local(self::BACKUPS . '/' . $this->database . '/full/' . $time); + $local = new Local(self::BACKUPS_PATH . '/' . $this->database . '/full/' . $time); $local->setTransferChunkSize(5 * 1024 * 1024); // 5MB $backups = $local->getRoot() . '/files'; @@ -99,15 +103,13 @@ class Backup extends Action $this->fullBackup($backups); $this->tar($backups, $tarFile); $this->upload($tarFile, $local); - // todo: Do we want to delete the tar file? and remain with the folder? - // todo: Do we want to delete the backup.log? self::log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- ' . PHP_EOL . PHP_EOL); } public function fullBackup(string $target) { - if (!file_exists(self::BACKUPS)) { + if (!file_exists(self::BACKUPS_PATH)) { Console::error('Mount directory does not exist'); Console::exit(); } @@ -120,7 +122,6 @@ class Backup extends Action $logfile = $target . '/../backup.log'; $args = [ - //'--defaults-file=' . $this->cnf, // [ERROR] Failed to open required defaults file: /etc/my.cnf '--user=' . $this->dsn->getUser(), '--password=' . $this->dsn->getPassword(), '--host=' . $this->dsn->getHost(), @@ -136,10 +137,6 @@ class Backup extends Action '--compress=' . self::COMPRESS_ALGORITHM, '--compress-threads=' . self::PROCESSORS, '--rsync', // https://docs.percona.com/percona-xtrabackup/8.0/accelerate-backup-process.html - //'--encrypt-threads=' . $this->processors, - //'--encrypt=AES256', - //'--encrypt-key-file=' . '/encryption_key_file', - //'--no-lock', // https://docs.percona.com/percona-xtrabackup/8.0/xtrabackup-option-reference.html#-no-lock '2> ' . $logfile, ]; @@ -148,21 +145,20 @@ class Backup extends Action shell_exec($cmd); $stderr = shell_exec('tail -1 ' . $logfile); - Backup::log($stderr); + self::log($stderr); if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) { Console::error('Backup failed'); Console::exit(); } - // todo: remove logfile? + unlink($logfile); } public function tar(string $directory, string $file) { $stdout = ''; $stderr = ''; - // Tar from inside the directory for not using --strip-components $cmd = 'cd ' . $directory . ' && tar zcf ' . $file . ' .'; self::log($cmd); Console::execute($cmd, '', $stdout, $stderr); @@ -173,14 +169,14 @@ class Backup extends Action } if (!file_exists($file)) { - Console::error("Can't find tar file: " . $file); + Console::error('Can\'t find tar file: ' . $file); Console::exit(); } $filesize = \filesize($file); - self::log("Tar file size is: " . ceil($filesize / 1024 / 1024) . 'MB'); + self::log('Tar file size is: ' . ceil($filesize / 1024 / 1024) . 'MB'); if ($filesize < (2 * 1024 * 1024)) { - Console::error("File size is very small: " . $file); + Console::error('File size is very small: ' . $file); Console::exit(); } } @@ -188,30 +184,31 @@ class Backup extends Action public function upload(string $file, Device $local) { $filename = basename($file); - $s3 = new DOSpaces('/' . $this->database . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); - if (!$s3->exists('/')) { + if (!$this->s3->exists('/')) { Console::error('Can\'t read s3 root directory'); Console::exit(); } try { self::log('Uploading: ' . $file); - $destination = $s3->getRoot() . '/' . $filename; + $destination = $this->s3->getRoot() . '/' . $filename; - if (!$local->transfer($file, $destination, $s3)) { + if (!$local->transfer($file, $destination, $this->s3)) { Console::error('Error uploading to ' . $destination); Console::exit(); } - if (!$s3->exists($destination)) { - Console::error('File not found on s3 ' . $destination); + if (!$this->s3->exists($destination)) { + Console::error('File not found in destination: ' . $destination); Console::exit(); } } catch (Exception $e) { Console::error($e->getMessage()); Console::exit(); } + + unlink($file); } public static function log(string $message): void @@ -238,15 +235,4 @@ class Backup extends Action } } } - - public static function getDsn(string $database): ?DSN - { - foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_PROJECT')) as $project) { - [$db, $dsn] = explode('=', $project); - if ($db === $database) { - return new DSN($dsn); - } - } - return null; - } } diff --git a/src/Appwrite/Platform/Tasks/Restore.php b/src/Appwrite/Platform/Tasks/Restore.php index 4c77681ada..6d922a2baa 100644 --- a/src/Appwrite/Platform/Tasks/Restore.php +++ b/src/Appwrite/Platform/Tasks/Restore.php @@ -15,14 +15,10 @@ use Utopia\Validator\WhiteList; class Restore extends Action { + public const BACKUPS_PATH = '/backups'; + public const PROCESSORS = 4; protected ?DSN $dsn = null; protected string $database; - public const PROCESSORS = 4; - - public static function getName(): string - { - return 'restore'; - } public function __construct() { @@ -30,17 +26,22 @@ class Restore extends Action $this ->desc('Restore a DB') - ->param('id', '', new Text(20), 'Folder Identifier') - ->param('cloud', null, new WhiteList(['true', 'false'], true), 'Take file from cloud?') - ->param('database', null, new Text(10), 'example: db_fra1_01..') - ->param('datadir', null, new Text(100), 'mysql datadir path') + ->param('id', '', new Text(20), 'The backup identification') + ->param('cloud', null, new WhiteList(['true', 'false'], true), 'Download backup from cloud or use local directory') + ->param('database', null, new Text(10), 'The Database name for example db_fra1_01') + ->param('datadir', null, new Text(100), 'The Path where database would be stored') ->callback(fn ($id, $cloud, $project, $datadir) => $this->action($id, $cloud, $project, $datadir)); } + public static function getName(): string + { + return 'restore'; + } + public function action(string $id, string $cloud, string $database, string $datadir): void { $this->database = $database; - $this->dsn = Backup::getDsn($database); + $this->dsn = self::getDsn($database); if (!$this->dsn instanceof DSN) { Console::error('No dsn match'); @@ -57,22 +58,19 @@ class Restore extends Action Console::exit(); } - //todo: check if the container is stopped - //$this->checkContainerIsStopped(); - $filename = $id . '.tar.gz'; - Backup::log('--- Restore Start ' . $filename . ' --- '); + $this->log('--- Restore Start ' . $filename . ' --- '); $start = microtime(true); $cloud = $cloud === 'true'; if ($cloud) { - $local = new Local(Backup::BACKUPS . '/downloads/' . $id); + $local = new Local(self::BACKUPS_PATH . '/downloads/' . $id); if (file_exists($local->getRoot())) { $stdout = ''; $stderr = ''; $cmd = 'rm -rf ' . $local->getRoot(); - Backup::log($cmd); + $this->log($cmd); Console::execute($cmd, '', $stdout, $stderr); if (!empty($stderr)) { Console::error($stderr); @@ -82,7 +80,7 @@ class Restore extends Action $files = $local->getRoot() . '/files'; - Backup::log('Creating Directory: ' . $files); + $this->log('Creating Directory: ' . $files); if (!mkdir($files, 0755, true)) { Console::error('Error creating directory: ' . $files); Console::exit(); @@ -91,7 +89,7 @@ class Restore extends Action $this->download($filename, $local); $this->untar($local->getPath($filename), $files); } else { - $local = new Local(Backup::BACKUPS . '/' . $database . '/full/' . $id); + $local = new Local(self::BACKUPS_PATH . '/' . $database . '/full/' . $id); $files = $local->getRoot() . '/files'; } @@ -104,7 +102,7 @@ class Restore extends Action $this->prepare($files); $this->restore($files, $cloud, $datadir); - Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds"); + $this->log('Restore Finish in ' . (microtime(true) - $start) . ' seconds'); } public function download(string $filename, Device $local) @@ -120,7 +118,7 @@ class Restore extends Action } $file = $local->getPath($filename); - Backup::log('Downloading: ' . $file); + $this->log('Downloading: ' . $file); if (!$s3->transfer($path, $file, $local)) { Console::error('Error Downloading ' . $file); @@ -137,7 +135,7 @@ class Restore extends Action $stdout = ''; $stderr = ''; $cmd = 'tar -xzf ' . $file . ' -C ' . $directory; - Backup::log($cmd); + $this->log($cmd); Console::execute($cmd, '', $stdout, $stderr); if (!empty($stderr)) { Console::error($stderr); @@ -168,11 +166,11 @@ class Restore extends Action ]; $cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args); - Backup::log($cmd); + $this->log($cmd); shell_exec($cmd); $stderr = shell_exec('tail -1 ' . $logfile); - Backup::log($stderr); + $this->log($stderr); if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) { Console::error('Decompress failed'); @@ -200,11 +198,11 @@ class Restore extends Action ]; $cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args); - Backup::log($cmd); + $this->log($cmd); shell_exec($cmd); $stderr = shell_exec('tail -1 ' . $logfile); - Backup::log($stderr); + $this->log($stderr); if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) { Console::error('Prepare failed'); @@ -234,11 +232,11 @@ class Restore extends Action ]; $cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args); - Backup::log($cmd); + $this->log($cmd); shell_exec($cmd); $stderr = shell_exec('tail -1 ' . $logfile); - Backup::log($stderr); + $this->log($stderr); if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) { Console::error('Restore failed'); @@ -265,27 +263,21 @@ class Restore extends Action } } - public function checkContainerIsStopped(): void + public function log(string $message): void { - $host = $this->dsn->getHost(); - $cmd = 'docker ps --filter "status=running" --filter "name=' . $host . '"'; - Backup::log($cmd); - $stderr = ''; - $stdout = ''; - Console::execute($cmd, '', $stdout, $stderr); - - if (!empty($stderr)) { - Console::error($stderr); - Console::exit(); - } - - $stdout = explode(PHP_EOL, $stdout); - array_shift($stdout); - $info = array_shift($stdout); - - if (!empty($info)) { - Console::error('Please stop container: ' . $host); - //Console::exit(); + if (!empty($message)) { + Console::log(date('Y-m-d H:i:s') . ' ' . $message); } } + + public function getDsn(string $database): ?DSN + { + foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_PROJECT')) as $project) { + [$db, $dsn] = explode('=', $project); + if ($db === $database) { + return new DSN($dsn); + } + } + return null; + } }