diff --git a/.env b/.env index 7516d161f3..cc3073cadc 100644 --- a/.env +++ b/.env @@ -85,4 +85,5 @@ _DO_SPACES_REGION=fra1 _DO_SPACES_ACCESS_KEY= _DO_SPACES_SECRET_KEY= _DO_SPACES_ACCESS_KEY= -_DO_SPACES_SECRET_KEY= \ No newline at end of file +_DO_SPACES_SECRET_KEY= +_APP_BACKUP_FOLDER=daily \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index ba42350a0f..c43eeddb16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -634,7 +634,6 @@ services: - ./src:/usr/src/code/src - appwrite-mariadb:/var/lib/mysql:rw - ./backups:/backups:rw - - ./extracts:/extracts:rw - /var/run/docker.sock:/var/run/docker.sock depends_on: - redis @@ -644,6 +643,7 @@ services: - _DO_SPACES_ACCESS_KEY - _DO_SPACES_SECRET_KEY - _DO_SPACES_REGION + - _APP_BACKUP_FOLDER appwrite-usage: entrypoint: usage diff --git a/src/Appwrite/Platform/Tasks/Backup.php b/src/Appwrite/Platform/Tasks/Backup.php index 129b648137..88b964a3d6 100644 --- a/src/Appwrite/Platform/Tasks/Backup.php +++ b/src/Appwrite/Platform/Tasks/Backup.php @@ -7,11 +7,10 @@ use Utopia\App; use Utopia\CLI\Console; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Storage; -use Utopia\Validator\Hostname; class Backup extends Action { - protected string $directory = '/var/lib/mysql'; + public static string $mysqlDirectory = '/var/lib/mysql'; public static string $backups = '/backups'; // Mounted volume protected string $containerName = 'appwrite-mariadb'; @@ -22,53 +21,30 @@ class Backup extends Action public function __construct() { - //docker compose exec appwrite backup $this ->desc('Backup a DB') - ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain.', true) - ->callback(fn($domain) => $this->action($domain)); + ->callback(fn() => $this->action()); } /** * @throws \Exception */ - public function action(string $domain): void + public function action(): void { + $this->checkEnvVariables(); + + $folder = App::getEnv('_APP_BACKUP_FOLDER'); + $start = microtime(true); $time = date('Y-m-d_H:i:s'); - $this->log('--- Backup Start --- '); - - if (empty(App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))) { - Console::error('Can\'t read _APP_CONNECTIONS_DB_PROJECT'); - Console::exit(); - } - - if (empty(App::getEnv('_DO_SPACES_BUCKET_NAME'))) { - Console::error('Can\'t read _DO_SPACES_BUCKET_NAME'); - Console::exit(); - } - - if (empty(App::getEnv('_DO_SPACES_ACCESS_KEY'))) { - Console::error('Can\'t read _DO_SPACES_ACCESS_KEY'); - Console::exit(); - } - - if (empty(App::getEnv('_DO_SPACES_SECRET_KEY'))) { - Console::error('Can\'t read _DO_SPACES_SECRET_KEY'); - Console::exit(); - } - - if (empty(App::getEnv('_DO_SPACES_REGION'))) { - Console::error('Can\'t read _DO_SPACES_REGION'); - Console::exit(); - } + self::log('--- Backup Start --- '); Storage::setDevice('files', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'))); $device = Storage::getDevice('files'); // Todo: do we want to have the backup ready on the mounted dir? or to abort? if (!$device->exists('/')) { - Console::error('Can\'t read from DO aborting'); + Console::error('Can\'t read from DO '); Console::exit(); } @@ -79,7 +55,7 @@ class Backup extends Action Console::exit(); } - $backups = self::$backups . '/' . $dsn; + $backups = self::$backups . '/' . $dsn . '/' . $folder; if (!file_exists($backups) && !mkdir($backups, 0755, true)) { Console::error('Error creating directory: ' . $backups); @@ -89,10 +65,10 @@ class Backup extends Action $stdout = ''; $stderr = ''; $cmd = 'docker stop ' . $this->containerName; - $this->log($cmd); + self::log($cmd); Console::execute($cmd, '', $stdout, $stderr); - $this->log($stdout); if (!empty($stderr)) { + self::log($stdout); Console::error($stderr); Console::exit(); } @@ -110,10 +86,10 @@ class Backup extends Action $stdout = ''; $stderr = ''; - $cmd = 'cd ' . $this->directory . ' && tar zcf ' . $file . ' .'; - $this->log($cmd); + $cmd = 'cd ' . self::$mysqlDirectory . ' && tar zcf ' . $file . ' .'; + self::log($cmd); Console::execute($cmd, '', $stdout, $stderr); - $this->log($stdout); + self::log($stdout); if (!empty($stderr)) { Console::error($stderr); Console::exit(); @@ -122,50 +98,57 @@ class Backup extends Action $stdout = ''; $stderr = ''; $cmd = 'docker start ' . $this->containerName; - $this->log($cmd); + self::log($cmd); Console::execute($cmd, '', $stdout, $stderr); - $this->log($stdout); if (!empty($stderr)) { + self::log($stdout); Console::error($stderr); Console::exit(); } try { - $folder = App::getEnv('_APP_BACKUP_FOLDER', 'hourly'); - $path = '/' . $dsn . '/' . $folder . '/' . $filename; - $this->log('Uploading ' . $path); + self::log('Uploading ' . $path); if (!$device->upload($file, $path)) { Console::error('Error uploading to ' . $path); Console::exit(); } - -// $isDaily = true; -// -// if ($isDaily) { -// $dailyPath = '/' . $dsn . '/daily/' . $filename; -// $this->log('Moving ' . $dailyPath); -// if (!$device->move($hourlyPath, $dailyPath)) { -// Console::error('Error moving to hourly to ' . $dailyPath); -// Console::exit(); -// } -// } } catch (\Exception $e) { Console::error($e->getMessage()); Console::exit(); } - $this->log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- ' . PHP_EOL . PHP_EOL); + self::log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- ' . PHP_EOL . PHP_EOL); Console::loop(function () { - Console::log('Hello'); + self::log('loop'); }, 100); } - public function log(string $message): void + public static function log(string $message): void { if (!empty($message)) { Console::log(date('Y-m-d H:i:s') . ' ' . $message); } } + + public function checkEnvVariables(): void + { + foreach ( + [ + '_APP_BACKUP_FOLDER', + '_APP_CONNECTIONS_DB_PROJECT', + '_DO_SPACES_BUCKET_NAME', + '_DO_SPACES_ACCESS_KEY', + '_DO_SPACES_SECRET_KEY', + '_DO_SPACES_REGION' + ] as $env + ) { + if (empty(App::getEnv($env))) { + Console::error('Can\'t read ' . $env); + Console::exit(); + } + } + } + } diff --git a/src/Appwrite/Platform/Tasks/Restore.php b/src/Appwrite/Platform/Tasks/Restore.php index 19f6d6744f..6281ff16cc 100644 --- a/src/Appwrite/Platform/Tasks/Restore.php +++ b/src/Appwrite/Platform/Tasks/Restore.php @@ -8,15 +8,15 @@ use Utopia\CLI\Console; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; -use Utopia\Validator\Boolean; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -// docker compose exec appwrite-backup db-restore --file=2023-07-17_11:36:12.tar.gz +// docker compose exec appwrite-backup db-restore --cloud=false --filename=2023-07-18_12:39:59.tar.gz class Restore extends Action { - protected string $extract = '/extracts'; + // todo: double check this is not a production value!!!!!!!!!!!!!!! + // todo: it will be erased!!!! protected string $containerName = 'appwrite-mariadb'; public static function getName(): string @@ -28,51 +28,70 @@ class Restore extends Action { $this ->desc('Restore a DB') - ->param('file', '', new Text(100), 'Backup file name') - ->param('cloud', 'false', new Boolean(true), 'Take file from cloud?') - ->callback(fn ($file, $cloud) => $this->action($file, $cloud)); + ->param('filename', '', new Text(100), 'Backup file name') + ->param('cloud', null, new WhiteList(['true', 'false'], true), 'Take file from cloud?') + ->param('project', null, new WhiteList(['db_fra1_02'], true), 'From _APP_CONNECTIONS_DB_PROJECT') + ->param('folder', null, new WhiteList(['hourly', 'daily'], true), 'Sub folder') + ->callback(fn ($file, $cloud, $project, $folder) => $this->action($file, $cloud, $project, $folder)); } /** * @throws \Exception */ - public function action(string $file, bool $cloud): void + public function action(string $filename, string $cloud, string $project, string $folder): void { + $this->checkEnvVariables(); - var_dump($cloud); + Backup::log('--- Restore Start ' . $filename . ' --- '); + $start = microtime(true); + + $cloud = $cloud === 'true'; + $file = Backup::$backups . '/' . $project . '/' . $folder . '/' . $filename; + $extract = Backup::$backups . '/extract'; + $original = $extract . '/original-' . time(); + + Backup::log('Creating directory ' . $original); + if (!file_exists($original) && !mkdir($original, 0755, true)) { + Console::error('Error creating directory: ' . $original); + Console::exit(); + } + +// $tarDirectory = $extract . '/' . str_replace(['.', '-', '_', ':', 'tar', 'gz'], '', $filename); +// Backup::log('Creating directory ' . $tarDirectory); +// if (!file_exists($tarDirectory) && !mkdir($tarDirectory, 0755, true)) { +// Console::error('Error creating directory: ' . $tarDirectory); +// Console::exit(); +// } if ($cloud) { - Storage::setDevice('files', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'))); + Storage::setDevice('s3', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'))); Storage::setDevice('mount', new Local(Backup::$backups)); - $device = Storage::getDevice('files'); + $device = Storage::getDevice('s3'); $mount = Storage::getDevice('mount'); try { $folder = App::getEnv('_APP_BACKUP_FOLDER', 'hourly'); - $dsn = explode('=', App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))[0]; - $path = '/' . $dsn . '/' . $folder . '/' . $file; + $path = '/' . $project . '/' . $folder . '/' . $filename; - $this->log('Downloading from Space ' . $path); + if (!$device->exists($path)) { + Console::error('File: ' . $path . ' does not exist on cloud'); + Console::exit(); + } - $x = $device->read($path); - - $mount->write('/bla-' . $file, $device->read($path)); + while ($data = $device->read($path)) { + $mount->write('file-' . $file, $data); + } + Backup::log('Downloading from s3 ' . $path); + $mount->write('file-' . $file, $device->read($path)); + // $mount->get('file-' . $file, $device->read($path)); } catch (\Exception $e) { Console::error($e->getMessage()); Console::exit(); } } - - exit; - - $file = Backup::$backups . '/' . $file; - $start = microtime(true); - - $this->log('--- Restore Start ' . $file . ' --- '); - if (!file_exists($file) || empty($file)) { Console::error('Restore file not found: ' . $file); Console::exit(); @@ -81,50 +100,79 @@ class Restore extends Action $stdout = ''; $stderr = ''; $cmd = 'docker stop ' . $this->containerName; - $this->log($cmd); + Backup::log($cmd); Console::execute($cmd, '', $stdout, $stderr); - $this->log($stdout); if (!empty($stderr)) { + Backup::log($stdout); Console::error($stderr); Console::exit(); } - $extract = $this->extract . '/' . time(); - $this->log('Creating extract directory ' . $extract); - if (!file_exists($extract) && !mkdir($extract, 0755, true)) { - Console::error('Error creating directory: ' . $extract); - Console::exit(); - } $stdout = ''; $stderr = ''; - $cmd = 'tar -xzf ' . $file . ' -C ' . $this->extract; - $this->log($cmd); + $cmd = 'mv ' . Backup::$mysqlDirectory . '/* ' . ' ' . $original . '/'; + Backup::log($cmd); Console::execute($cmd, '', $stdout, $stderr); - $this->log($stdout); + Backup::log($stdout); if (!empty($stderr)) { Console::error($stderr); Console::exit(); } + + $stdout = ''; + $stderr = ''; + // $cmd = 'tar -xzf ' . $file . ' -C ' . $tarDirectory; + $cmd = 'tar -xzf ' . $file . ' -C ' . Backup::$mysqlDirectory; + + Backup::log($cmd); + Console::execute($cmd, '', $stdout, $stderr); + if (!empty($stderr)) { + Backup::log($stdout); + Console::error($stderr); + Console::exit(); + } + +// $stdout = ''; +// $stderr = ''; +// $cmd = 'mv ' . $tarDirectory . ' ' . Backup::$mysqlDirectory; +// Backup::log($cmd); +// Console::execute($cmd, '', $stdout, $stderr); +// Backup::log($stdout); +// if (!empty($stderr)) { +// Console::error($stderr); +// Console::exit(); +// } + $stdout = ''; $stderr = ''; $cmd = 'docker start ' . $this->containerName; - $this->log($cmd); + Backup::log($cmd); Console::execute($cmd, '', $stdout, $stderr); - $this->log($stdout); + Backup::log($stdout); if (!empty($stderr)) { Console::error($stderr); Console::exit(); } - $this->log("Restore Finish in " . (microtime(true) - $start) . " seconds"); + Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds"); } - public function log(string $message): void + public function checkEnvVariables(): void { - if (!empty($message)) { - Console::log(date('Y-m-d H:i:s') . ' ' . $message); + foreach ( + [ + '_DO_SPACES_BUCKET_NAME', + '_DO_SPACES_ACCESS_KEY', + '_DO_SPACES_SECRET_KEY', + '_DO_SPACES_REGION' + ] as $env + ) { + if (empty(App::getEnv($env))) { + Console::error('Can\'t read ' . $env); + Console::exit(); + } } } }