From 9fab5c46f1cac2fe8e1cd49704da97dca5a84301 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 7 Aug 2023 15:54:09 +0300 Subject: [PATCH] dsn changes --- .env | 4 +- docker-compose.yml | 10 +- src/Appwrite/Platform/Tasks/Backup.php | 167 ++++++++---------------- src/Appwrite/Platform/Tasks/Restore.php | 145 ++++++++++---------- 4 files changed, 131 insertions(+), 195 deletions(-) diff --git a/.env b/.env index d90b7f04ed..9e3eece255 100644 --- a/.env +++ b/.env @@ -27,7 +27,7 @@ _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword _APP_CONNECTIONS_MAX=3100 _APP_POOL_CLIENTS=14 -_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite +_APP_CONNECTIONS_DB_PROJECT=db_fra1_01=mariadb://root:rootsecretpassword@mariadb:3306/appwrite,db_fra1_02=mariadb://root:rootsecretpassword@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379 @@ -84,3 +84,5 @@ _DO_SPACES_BUCKET_NAME=backups-v1 _DO_SPACES_REGION=fra1 _DO_SPACES_ACCESS_KEY= _DO_SPACES_SECRET_KEY= +_DO_SPACES_ACCESS_KEY= +_DO_SPACES_SECRET_KEY= \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 71230e41b8..141e1a6ba7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -752,8 +752,6 @@ services: image: percona/percona-xtrabackup:latest container_name: appwrite-xtrabackup command: sleep infinity - environment: - - MYSQL_ROOT_PASSWORD=${_APP_DB_ROOT_PASS} volumes: - ./backups:/backups:rw - ./varlibmysql:/varlibmysql:rw @@ -764,6 +762,8 @@ services: appwrite-backup: entrypoint: db-backup + command: + - --database=db_fra1_02 <<: *x-logging container_name: appwrite-backup image: appwrite-dev @@ -772,7 +772,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - - appwrite-mariadb:/var/lib/mysql:rw + - appwrite-mariadb:/var/lib/mysql:r - ./backups:/backups:rw - /var/run/docker.sock:/var/run/docker.sock - ./dev:/usr/local/dev @@ -784,8 +784,6 @@ services: - _DO_SPACES_ACCESS_KEY - _DO_SPACES_SECRET_KEY - _DO_SPACES_REGION - - _APP_BACKUP_FOLDER - - _APP_DB_ROOT_PASS appwrite-restore: command: sleep infinity @@ -808,8 +806,6 @@ services: - _DO_SPACES_ACCESS_KEY - _DO_SPACES_SECRET_KEY - _DO_SPACES_REGION - - _APP_BACKUP_FOLDER - - _APP_DB_ROOT_PASS redis: image: redis:7.0.4-alpine diff --git a/src/Appwrite/Platform/Tasks/Backup.php b/src/Appwrite/Platform/Tasks/Backup.php index ce5416bf39..f58d759609 100644 --- a/src/Appwrite/Platform/Tasks/Backup.php +++ b/src/Appwrite/Platform/Tasks/Backup.php @@ -2,6 +2,8 @@ namespace Appwrite\Platform\Tasks; +use Exception; +use Utopia\DSN\DSN; use Utopia\Platform\Action; use Utopia\App; use Utopia\CLI\Console; @@ -9,11 +11,12 @@ use Utopia\Pools\Group; use Utopia\Storage\Device; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Local; +use Utopia\Validator\Text; class Backup extends Action { - protected string $host = 'mariadb'; - protected string $project; + protected ?DSN $dsn = null; + protected ?string $database = null; public const BACKUPS = '/backups'; //public const BACKUP_INTERVAL = 60 * 60 * 4; // 4 hours; public const BACKUP_INTERVAL = 300; // 4 hours; @@ -29,19 +32,27 @@ class Backup extends Action public function __construct() { $this->checkEnvVariables(); - $this->project = explode('=', App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))[0]; $this ->desc('Backup a DB') + ->param('database', null, new Text(20), 'passed from command') ->inject('pools') - ->callback(fn(Group $pools) => $this->action($pools)); + ->callback(fn(string $database, Group $pools) => $this->action($database, $pools)); } /** - * @throws \Exception + * @throws Exception */ - public function action($pools): void + public function action($database, $pools): void { + $this->database = $database; + $this->dsn = self::getDsn($database); + + if (!$this->dsn instanceof DSN) { + Console::error('No dsn match'); + Console::exit(); + } + $attempts = 0; $max = 10; $sleep = 5; @@ -50,16 +61,16 @@ class Backup extends Action try { $attempts++; $pools - ->get('database_' . $this->project) + ->get('database_' . $database) ->pop() ->getResource() ; break; // leave the do-while if successful - } catch (\Exception $e) { + } catch (Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); if ($attempts >= $max) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + throw new Exception('Failed to connect to database: ' . $e->getMessage()); } sleep($sleep); @@ -73,36 +84,24 @@ class Backup extends Action public function start(): void { - self::log('--- Backup Start --- '); $start = microtime(true); - //$type = 'inc'; - $type = 'full'; + $time = date('Y_m_d_H_i_s'); - switch ($type) { - case 'inc': - $this->incrementalBackup(); - break; - case 'full': - $time = date('Y_m_d_H_i_s'); - self::log('--- Creating backup ' . $time . ' --- '); - $filename = $time . '.tar.gz'; - $local = new Local(self::BACKUPS . '/' . $this->project . '/full/' . $time); - $local->setTransferChunkSize(5 * 1024 * 1024); // > 5MB + self::log('--- Backup Start --- '); + self::log('--- Creating backup ' . $time . ' --- '); - $backups = $local->getRoot() . '/files'; - $tarFile = $local->getPath($filename); + $filename = $time . '.tar.gz'; + $local = new Local(self::BACKUPS . '/' . $this->database . '/full/' . $time); + $local->setTransferChunkSize(5 * 1024 * 1024); // > 5MB - $this->fullBackup($backups); - $this->tar($backups, $tarFile); - $this->upload($tarFile, $local); - // todo: Do we want to delete the tar file? and remain with the folder? - // todo: Do we want to delete the tar log.file? - break; + $backups = $local->getRoot() . '/files'; + $tarFile = $local->getPath($filename); - default: - Console::error('No type detected'); - Console::exit(); - } + $this->fullBackup($backups); + $this->tar($backups, $tarFile); + $this->upload($tarFile, $local); + // todo: Do we want to delete the tar file? and remain with the folder? + // todo: Do we want to delete the backup.log? self::log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- ' . PHP_EOL . PHP_EOL); } @@ -119,16 +118,16 @@ class Backup extends Action Console::exit(); } - $logfile = $target . '/../log.txt'; + $logfile = $target . '/../backup.log'; $args = [ //'--defaults-file=' . $this->cnf, // [ERROR] Failed to open required defaults file: /etc/my.cnf - '--user=root', - '--password=' . App::getEnv('_APP_DB_ROOT_PASS'), - '--host=' . $this->host, + '--user=' . $this->dsn->getUser(), + '--password=' . $this->dsn->getPassword(), + '--host=' . $this->dsn->getHost(), '--backup', '--strict', - '--history=' . $this->project, // logs PERCONA_SCHEMA.xtrabackup_history + '--history=' . $this->database, // logs PERCONA_SCHEMA.xtrabackup_history name attribute '--slave-info', '--safe-slave-backup', '--safe-slave-backup-timeout=300', @@ -190,7 +189,7 @@ class Backup extends Action public function upload(string $file, Device $local) { $filename = basename($file); - $s3 = new DOSpaces('/' . $this->project . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); + $s3 = new DOSpaces('/' . $this->database . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); if (!$s3->exists('/')) { Console::error('Can\'t read s3 root directory'); @@ -210,86 +209,12 @@ class Backup extends Action Console::error('File not found on s3 ' . $destination); Console::exit(); } - } catch (\Exception $e) { + } catch (Exception $e) { Console::error($e->getMessage()); Console::exit(); } } - public function incrementalBackup() - { - $project = $this->project; - //$folder = ceil(time() / 60 * 60); - //$folder = date('Y_m_d'); - //$folder = ceil(date('z') / 7); // day of the year 0-365 - $folder = date('W'); // week of the year 0 - 51 - - $folder = 'v1_' . date('Y') . '_' . $folder; - $local = new Local(self::BACKUPS . '/' . $project . '/inc/' . $folder); - $position = 1; - $target = $local->getRoot() . '/' . $position; - $base = ''; - - if (file_exists($local->getRoot() . '/position')) { - $position = intval(file_get_contents($local->getRoot() . '/position')); - - if (!file_exists($local->getRoot() . '/' . $position . '/xtrabackup_checkpoints')) { - Console::error('Backup ' . $folder . ' is garbage!!!'); - Console::exit(); - } - - $base = $local->getRoot() . '/' . $position; - $position += 1; - $target = $local->getRoot() . '/' . $position; - } - - Console::success($base); - Console::success($target); - - if (!file_exists($target) && !mkdir($target, 0755, true)) { - Console::error('Error creating backup directory: ' . $target); - Console::exit(); - } - - file_put_contents($local->getRoot() . '/position', $position); - - $args = [ - '--user=root', - '--password=rootsecretpassword', - '--backup=1', - '--strict', - '--host=' . $this->host, - '--safe-slave-backup', - '--safe-slave-backup-timeout=300', - '--check-privileges', - //'--no-lock', // https://docs.percona.com/percona-xtrabackup/8.0/xtrabackup-option-reference.html#-no-lock - //'--compress=lz4', - //'--compress-threads=1', - '--target-dir=' . $target - ]; - - if (!empty($base)) { - $args[] = '--incremental-basedir=' . $base; - } - - $stdout = ''; - $stderr = ''; - $cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args); - self::log($cmd); - Console::execute($cmd, '', $stdout, $stderr); - if (!empty($stderr)) { - // Console::error($stderr); - //Console::exit(); - } - - // For some reason they write everything as $stderr - if (!str_contains($stderr, 'completed OK!')) { - /// Todo We need to destroy this directory and all the data inside or move it somewhere - Console::error($stderr); - Console::exit(); - } - } - public static function log(string $message): void { if (!empty($message)) { @@ -306,7 +231,6 @@ class Backup extends Action '_DO_SPACES_ACCESS_KEY', '_DO_SPACES_SECRET_KEY', '_DO_SPACES_REGION', - '_APP_DB_ROOT_PASS' ] as $env ) { if (empty(App::getEnv($env))) { @@ -315,4 +239,15 @@ class Backup extends Action } } } + + public static function getDsn(string $database): ?DSN + { + foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_PROJECT')) as $project) { + list($db, $dsn) = explode('=', $project); + if ($db === $database) { + return new DSN($dsn); + } + } + return null; + } } diff --git a/src/Appwrite/Platform/Tasks/Restore.php b/src/Appwrite/Platform/Tasks/Restore.php index cce9323586..b1bd91608b 100644 --- a/src/Appwrite/Platform/Tasks/Restore.php +++ b/src/Appwrite/Platform/Tasks/Restore.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Tasks; use Exception; use Utopia\App; +use Utopia\DSN\DSN; use Utopia\Platform\Action; use Utopia\CLI\Console; use Utopia\Storage\Device; @@ -14,10 +15,8 @@ use Utopia\Validator\WhiteList; class Restore extends Action { - // todo: Carefully double check this is not a production value!!!!!!!!!!!!!!! - // todo: it will be erased!!!! - protected string $host = 'mariadb'; - protected string $project; + protected ?DSN $dsn = null; + protected string $database; public const PROCESSORS = 4; public static function getName(): string @@ -27,27 +26,32 @@ class Restore extends Action public function __construct() { + $this->checkEnvVariables(); + $this ->desc('Restore a DB') - ->param('id', '', new Text(100), 'Folder Identifier') + ->param('id', '', new Text(20), 'Folder Identifier') ->param('cloud', null, new WhiteList(['true', 'false'], true), 'Take file from cloud?') - ->param('project', null, new WhiteList(['db_fra1_01', 'db_fra1_02', 'db_fra1_03', 'db_fra1_04', 'db_fra1_05', 'db_fra1_06', 'db_fra1_07'], true), 'From _APP_CONNECTIONS_DB_PROJECT') + ->param('database', null, new Text(10), 'example: db_fra1_01..') ->param('datadir', null, new Text(100), 'mysql datadir path') ->callback(fn ($id, $cloud, $project, $datadir) => $this->action($id, $cloud, $project, $datadir)); } - public function action(string $id, string $cloud, string $project, string $datadir): void + public function action(string $id, string $cloud, string $database, string $datadir): void { - // todo? why do we need to pass it as variable? - $datadir = '/varlibmysql'; + $this->database = $database; + $this->dsn = Backup::getDsn($database); - if (!file_exists($datadir)) { - Console::error('Datadir not found: ' . $datadir); + if (!$this->dsn instanceof DSN) { + Console::error('No dsn match'); Console::exit(); } - if (!str_starts_with($datadir, '/')) { - Console::error('datadir must start with /'); + // todo? why do we need to pass it as variable? + //$datadir = '/varlibmysql'; + + if (!file_exists($datadir)) { + Console::error('Datadir not found: ' . $datadir); Console::exit(); } @@ -57,41 +61,40 @@ class Restore extends Action } $this->checkContainerIsStopped(); - $this->checkEnvVariables(); $filename = $id . '.tar.gz'; Backup::log('--- Restore Start ' . $filename . ' --- '); $start = microtime(true); $cloud = $cloud === 'true'; - $local = new Local(Backup::BACKUPS . '/' . $project . '/full/' . $id); - $files = $local->getRoot() . '/files'; - if ($cloud) { $local = new Local(Backup::BACKUPS . '/downloads/' . $id); - $this->download($project, $filename, $local); + + if (file_exists($local->getRoot())) { + $stdout = ''; + $stderr = ''; + $cmd = 'rm -rf ' . $local->getRoot(); + Backup::log($cmd); + Console::execute($cmd, '', $stdout, $stderr); + if (!empty($stderr)) { + Console::error($stderr); + Console::exit(); + } + } + $files = $local->getRoot() . '/files'; - if (!file_exists($files) && !mkdir($files, 0755, true)) { + + Backup::log('Creating Directory: ' . $files); + if (!mkdir($files, 0755, true)) { Console::error('Error creating directory: ' . $files); Console::exit(); } - $file = $local->getPath($filename); - - $stdout = ''; - $stderr = ''; - $cmd = 'tar -xzf ' . $file . ' -C ' . $files; - Backup::log($cmd); - Console::execute($cmd, '', $stdout, $stderr); - if (!empty($stderr)) { - Console::error($stderr); - Console::exit(); - } - - if (!file_exists($file)) { - Console::error('Restore file not found: ' . $file); - Console::exit(); - } + $this->download($filename, $local); + $this->untar($local->getPath($filename), $files); + } else { + $local = new Local(Backup::BACKUPS . '/' . $database . '/full/' . $id); + $files = $local->getRoot() . '/files'; } $this->decompress($files); @@ -101,26 +104,9 @@ class Restore extends Action Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds"); } - public function download(string $project, string $filename, Device $local) + public function download(string $filename, Device $local) { - if (file_exists($local->getRoot())) { - $stdout = ''; - $stderr = ''; - $cmd = 'rm -rf ' . $local->getRoot(); - Backup::log($cmd); - Console::execute($cmd, '', $stdout, $stderr); - if (!empty($stderr)) { - Console::error($stderr); - Console::exit(); - } - } - - if (!file_exists($local->getRoot()) && !mkdir($local->getRoot(), 0755, true)) { - Console::error('Error creating directory: ' . $local->getRoot()); - Console::exit(); - } - - $s3 = new DOSpaces($project . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); + $s3 = new DOSpaces($this->database . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); try { $path = $s3->getPath($filename); @@ -143,6 +129,24 @@ class Restore extends Action } } + public function untar(string $file, string $directory) + { + $stdout = ''; + $stderr = ''; + $cmd = 'tar -xzf ' . $file . ' -C ' . $directory; + Backup::log($cmd); + Console::execute($cmd, '', $stdout, $stderr); + if (!empty($stderr)) { + Console::error($stderr); + Console::exit(); + } + + if (!file_exists($file)) { + Console::error('Restore file not found: ' . $file); + Console::exit(); + } + } + public function decompress(string $target) { if (!file_exists($target)) { @@ -150,15 +154,15 @@ class Restore extends Action Console::exit(); } - $logfile = $target . '/../log.txt'; + $logfile = $target . '/../decompress.log'; $args = [ - '--user=root', - '--password=' . App::getEnv('_APP_DB_ROOT_PASS'), - '--host=' . $this->host, + '--user=' . $this->dsn->getUser(), + '--password=' . $this->dsn->getPassword(), + '--host=' . $this->dsn->getHost(), '--decompress', '--strict', - '--remove-original', // Removes *.lz4 + '--remove-original', // Removes *.lz4 compressed files '--parallel=' . self::PROCESSORS, '--compress-threads=' . self::PROCESSORS, '--target-dir=' . $target, @@ -185,12 +189,12 @@ class Restore extends Action Console::exit(); } - $logfile = $target . '/../log.txt'; + $logfile = $target . '/../prepare.log'; $args = [ - '--user=root', - '--password=' . App::getEnv('_APP_DB_ROOT_PASS'), - '--host=' . $this->host, + '--user=' . $this->dsn->getUser(), + '--password=' . $this->dsn->getPassword(), + '--host=' . $this->dsn->getHost(), '--prepare', '--strict', '--target-dir=' . $target, @@ -217,12 +221,12 @@ class Restore extends Action Console::exit(); } - $logfile = $target . '/../log.txt'; + $logfile = $target . '/../restore.log'; $args = [ - '--user=root', - '--password=' . App::getEnv('_APP_DB_ROOT_PASS'), - '--host=' . $this->host, + '--user=' . $this->dsn->getUser(), + '--password=' . $this->dsn->getPassword(), + '--host=' . $this->dsn->getHost(), $cloud ? '--move-back' : '--copy-back', '--strict', '--target-dir=' . $target, @@ -254,7 +258,6 @@ class Restore extends Action '_DO_SPACES_ACCESS_KEY', '_DO_SPACES_SECRET_KEY', '_DO_SPACES_REGION', - '_APP_DB_ROOT_PASS' ] as $env ) { if (empty(App::getEnv($env))) { @@ -266,7 +269,8 @@ class Restore extends Action public function checkContainerIsStopped(): void { - $cmd = 'docker ps --filter "status=running" --filter "name=' . $this->host . '"'; + $host = $this->dsn->getHost(); + $cmd = 'docker ps --filter "status=running" --filter "name=' . $host . '"'; Backup::log($cmd); $stderr = ''; $stdout = ''; @@ -282,9 +286,8 @@ class Restore extends Action $info = array_shift($stdout); if (!empty($info)) { - Console::error('Please stop container: ' . $this->host); + Console::error('Please stop container: ' . $host); //Console::exit(); } } - }