From 186b92d5afec801381cc1d675e6718957f0ce5be Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 17 Jul 2023 20:38:31 +0300 Subject: [PATCH] Restore.php + name Changes --- .env | 2 + bin/db-backup | 2 +- bin/db-restore | 2 +- docker-compose.yml | 2 +- src/Appwrite/Platform/Services/Tasks.php | 7 +- .../Tasks/{DbBackup.php => Backup.php} | 96 +++++++++---- src/Appwrite/Platform/Tasks/DbRestore.php | 66 --------- src/Appwrite/Platform/Tasks/Restore.php | 130 ++++++++++++++++++ 8 files changed, 205 insertions(+), 102 deletions(-) rename src/Appwrite/Platform/Tasks/{DbBackup.php => Backup.php} (58%) delete mode 100644 src/Appwrite/Platform/Tasks/DbRestore.php create mode 100644 src/Appwrite/Platform/Tasks/Restore.php diff --git a/.env b/.env index 037f6286a1..7516d161f3 100644 --- a/.env +++ b/.env @@ -83,4 +83,6 @@ _APP_CONSOLE_GITHUB_APP_ID= _DO_SPACES_BUCKET_NAME=backups-v1 _DO_SPACES_REGION=fra1 _DO_SPACES_ACCESS_KEY= +_DO_SPACES_SECRET_KEY= +_DO_SPACES_ACCESS_KEY= _DO_SPACES_SECRET_KEY= \ No newline at end of file diff --git a/bin/db-backup b/bin/db-backup index b56614e986..97d6dfb156 100644 --- a/bin/db-backup +++ b/bin/db-backup @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php db-backup $@ \ No newline at end of file +php /usr/src/code/app/cli.php backup $@ \ No newline at end of file diff --git a/bin/db-restore b/bin/db-restore index 4b5f24b6d1..6935f47778 100644 --- a/bin/db-restore +++ b/bin/db-restore @@ -1,3 +1,3 @@ #!/bin/sh -php /usr/src/code/app/cli.php db-restore $@ \ No newline at end of file +php /usr/src/code/app/cli.php restore $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index dff6f9ea6a..ba42350a0f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -634,7 +634,7 @@ services: - ./src:/usr/src/code/src - appwrite-mariadb:/var/lib/mysql:rw - ./backups:/backups:rw - # - ./extracts:/extracts:rw + - ./extracts:/extracts:rw - /var/run/docker.sock:/var/run/docker.sock depends_on: - redis diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index fbf189c580..436fa5eb8e 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -3,8 +3,7 @@ namespace Appwrite\Platform\Services; use Appwrite\Platform\Tasks\Backup; -use Appwrite\Platform\Tasks\DbBackup; -use Appwrite\Platform\Tasks\DbRestore; +use Appwrite\Platform\Tasks\Restore; use Utopia\Platform\Service; use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; @@ -51,8 +50,8 @@ class Tasks extends Service ->addAction(CalcUsersStats::getName(), new CalcUsersStats()) ->addAction(CalcTierStats::getName(), new CalcTierStats()) ->addAction(PatchDeleteProjectCollections::getName(), new PatchDeleteProjectCollections()) - ->addAction(DbBackup::getName(), new DbBackup()) - ->addAction(DbRestore::getName(), new DbRestore()) + ->addAction(Backup::getName(), new Backup()) + ->addAction(Restore::getName(), new Restore()) ; } } diff --git a/src/Appwrite/Platform/Tasks/DbBackup.php b/src/Appwrite/Platform/Tasks/Backup.php similarity index 58% rename from src/Appwrite/Platform/Tasks/DbBackup.php rename to src/Appwrite/Platform/Tasks/Backup.php index 68f390c84f..129b648137 100644 --- a/src/Appwrite/Platform/Tasks/DbBackup.php +++ b/src/Appwrite/Platform/Tasks/Backup.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Tasks; -use Appwrite\Extend\Exception; use Utopia\Platform\Action; use Utopia\App; use Utopia\CLI\Console; @@ -10,15 +9,15 @@ use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Storage; use Utopia\Validator\Hostname; -class DbBackup extends Action +class Backup extends Action { protected string $directory = '/var/lib/mysql'; - public static string $backups = '/backups'; // This is the mounted volume + public static string $backups = '/backups'; // Mounted volume protected string $containerName = 'appwrite-mariadb'; public static function getName(): string { - return 'db-backup'; + return 'backup'; } public function __construct() @@ -27,7 +26,7 @@ class DbBackup extends Action $this ->desc('Backup a DB') ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain.', true) - ->callback(fn ($domain) => $this->action($domain)); + ->callback(fn($domain) => $this->action($domain)); } /** @@ -35,6 +34,10 @@ class DbBackup extends Action */ public function action(string $domain): void { + $start = microtime(true); + $time = date('Y-m-d_H:i:s'); + $this->log('--- Backup Start --- '); + if (empty(App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))) { Console::error('Can\'t read _APP_CONNECTIONS_DB_PROJECT'); Console::exit(); @@ -60,29 +63,42 @@ class DbBackup extends Action Console::exit(); } - $time = date('Y-m-d_H:i:s'); - Console::log('--- Backup Start --- ' . $time); + Storage::setDevice('files', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'))); + $device = Storage::getDevice('files'); -// Console::log('creating directory:' . $this->backups); -// if (!file_exists($this->backups) && !mkdir($this->backups, 0755, true)) { -// Console::error('Error creating directory: ' . $this->backups); -// Console::exit(); -// } + // Todo: do we want to have the backup ready on the mounted dir? or to abort? + if (!$device->exists('/')) { + Console::error('Can\'t read from DO aborting'); + Console::exit(); + } + + $dsn = explode('=', App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))[0]; + + if (!file_exists(self::$backups)) { + Console::error('Mount directory does not exist'); + Console::exit(); + } + + $backups = self::$backups . '/' . $dsn; + + if (!file_exists($backups) && !mkdir($backups, 0755, true)) { + Console::error('Error creating directory: ' . $backups); + Console::exit(); + } $stdout = ''; $stderr = ''; $cmd = 'docker stop ' . $this->containerName; - Console::log($cmd); - $code = Console::execute($cmd, '', $stdout, $stderr); + $this->log($cmd); + Console::execute($cmd, '', $stdout, $stderr); + $this->log($stdout); if (!empty($stderr)) { Console::error($stderr); Console::exit(); } - Console::log($stdout); - $filename = $time . '.tar.gz'; - $file = self::$backups . '/' . $filename; + $file = $backups . '/' . $filename; // $cmd = 'tar czf ' . $file . ' --absolute-names ' . $this->directory; // @@ -95,9 +111,9 @@ class DbBackup extends Action $stdout = ''; $stderr = ''; $cmd = 'cd ' . $this->directory . ' && tar zcf ' . $file . ' .'; + $this->log($cmd); Console::execute($cmd, '', $stdout, $stderr); - Console::log($cmd); - Console::log($stdout); + $this->log($stdout); if (!empty($stderr)) { Console::error($stderr); Console::exit(); @@ -106,28 +122,50 @@ class DbBackup extends Action $stdout = ''; $stderr = ''; $cmd = 'docker start ' . $this->containerName; + $this->log($cmd); Console::execute($cmd, '', $stdout, $stderr); - Console::log($stdout); + $this->log($stdout); if (!empty($stderr)) { Console::error($stderr); Console::exit(); } - $dsn = explode('=', App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))[0]; - - Storage::setDevice('files', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'))); - $device = Storage::getDevice('files'); try { - $result = $device->upload($file, '/' . $dsn . '/daily/' . $filename); + $folder = App::getEnv('_APP_BACKUP_FOLDER', 'hourly'); + + $path = '/' . $dsn . '/' . $folder . '/' . $filename; + $this->log('Uploading ' . $path); + if (!$device->upload($file, $path)) { + Console::error('Error uploading to ' . $path); + Console::exit(); + } + +// $isDaily = true; +// +// if ($isDaily) { +// $dailyPath = '/' . $dsn . '/daily/' . $filename; +// $this->log('Moving ' . $dailyPath); +// if (!$device->move($hourlyPath, $dailyPath)) { +// Console::error('Error moving to hourly to ' . $dailyPath); +// Console::exit(); +// } +// } } catch (\Exception $e) { Console::error($e->getMessage()); Console::exit(); } - Console::log('-- Backup End -- '); + $this->log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- ' . PHP_EOL . PHP_EOL); -// Console::loop(function () { -// Console::log('Hello'); -// }, 100); + Console::loop(function () { + Console::log('Hello'); + }, 100); + } + + public function log(string $message): void + { + if (!empty($message)) { + Console::log(date('Y-m-d H:i:s') . ' ' . $message); + } } } diff --git a/src/Appwrite/Platform/Tasks/DbRestore.php b/src/Appwrite/Platform/Tasks/DbRestore.php deleted file mode 100644 index 78d6944552..0000000000 --- a/src/Appwrite/Platform/Tasks/DbRestore.php +++ /dev/null @@ -1,66 +0,0 @@ -desc('Restore a DB') - ->param('file', '', new Text(100), 'Backup file name') - ->callback(fn ($file) => $this->action($file)); - } - - /** - * @throws \Exception - */ - public function action(string $file): void - { - $start = microtime(true); - Console::log('Started' . $file); - - $file = DbBackup::$backups . '/' . $file; - - if (!file_exists($file) || empty($file)) { - Console::error('File not found: ' . $file); - Console::exit(); - } - - // Todo: shut down target container - - $extract = $this->extract . '/' . time(); - if (!file_exists($extract) && !mkdir($extract, 0755, true)) { - Console::error('Error creating directory: ' . $extract); - Console::exit(); - } - - $stdout = ''; - $stderr = ''; - $cmd = 'tar -xzf ' . $file . ' -C ' . $this->extract; - Console::log($cmd); - $code = Console::execute($cmd, '', $stdout, $stderr); - if (!empty($stderr)) { - Console::error($stderr); - Console::exit(); - } - - Console::log($stdout); - // Todo: start down target container - - Console::log("Restore Finish in " . (microtime(true) - $start) . " seconds"); - } -} diff --git a/src/Appwrite/Platform/Tasks/Restore.php b/src/Appwrite/Platform/Tasks/Restore.php new file mode 100644 index 0000000000..19f6d6744f --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Restore.php @@ -0,0 +1,130 @@ +desc('Restore a DB') + ->param('file', '', new Text(100), 'Backup file name') + ->param('cloud', 'false', new Boolean(true), 'Take file from cloud?') + ->callback(fn ($file, $cloud) => $this->action($file, $cloud)); + } + + /** + * @throws \Exception + */ + public function action(string $file, bool $cloud): void + { + + var_dump($cloud); + + if ($cloud) { + Storage::setDevice('files', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'))); + Storage::setDevice('mount', new Local(Backup::$backups)); + + $device = Storage::getDevice('files'); + $mount = Storage::getDevice('mount'); + + try { + $folder = App::getEnv('_APP_BACKUP_FOLDER', 'hourly'); + $dsn = explode('=', App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))[0]; + $path = '/' . $dsn . '/' . $folder . '/' . $file; + + $this->log('Downloading from Space ' . $path); + + $x = $device->read($path); + + $mount->write('/bla-' . $file, $device->read($path)); + + } catch (\Exception $e) { + Console::error($e->getMessage()); + Console::exit(); + } + } + + + exit; + + $file = Backup::$backups . '/' . $file; + $start = microtime(true); + + $this->log('--- Restore Start ' . $file . ' --- '); + + if (!file_exists($file) || empty($file)) { + Console::error('Restore file not found: ' . $file); + Console::exit(); + } + + $stdout = ''; + $stderr = ''; + $cmd = 'docker stop ' . $this->containerName; + $this->log($cmd); + Console::execute($cmd, '', $stdout, $stderr); + $this->log($stdout); + if (!empty($stderr)) { + Console::error($stderr); + Console::exit(); + } + + $extract = $this->extract . '/' . time(); + $this->log('Creating extract directory ' . $extract); + if (!file_exists($extract) && !mkdir($extract, 0755, true)) { + Console::error('Error creating directory: ' . $extract); + Console::exit(); + } + + $stdout = ''; + $stderr = ''; + $cmd = 'tar -xzf ' . $file . ' -C ' . $this->extract; + $this->log($cmd); + Console::execute($cmd, '', $stdout, $stderr); + $this->log($stdout); + if (!empty($stderr)) { + Console::error($stderr); + Console::exit(); + } + + $stdout = ''; + $stderr = ''; + $cmd = 'docker start ' . $this->containerName; + $this->log($cmd); + Console::execute($cmd, '', $stdout, $stderr); + $this->log($stdout); + if (!empty($stderr)) { + Console::error($stderr); + Console::exit(); + } + + $this->log("Restore Finish in " . (microtime(true) - $start) . " seconds"); + } + + public function log(string $message): void + { + if (!empty($message)) { + Console::log(date('Y-m-d H:i:s') . ' ' . $message); + } + } +}