diff --git a/.gitignore b/.gitignore index 9537c2900f..977b94520a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,6 @@ debug/ app/sdks dev/yasd_init.php .phpunit.result.cache -/backups/ +/backup-data/ +/restore-data/ /varlibmysql/ \ No newline at end of file diff --git a/composer.lock b/composer.lock index 245f01d413..bcf66029c2 100644 --- a/composer.lock +++ b/composer.lock @@ -2626,12 +2626,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "ae190865088491d7f8aa3fc63e6906efe6e42295" + "reference": "25f6b80b3063b85d7c9483be85f7b6ffea4c1edd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/ae190865088491d7f8aa3fc63e6906efe6e42295", - "reference": "ae190865088491d7f8aa3fc63e6906efe6e42295", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/25f6b80b3063b85d7c9483be85f7b6ffea4c1edd", + "reference": "25f6b80b3063b85d7c9483be85f7b6ffea4c1edd", "shasum": "" }, "require": { @@ -2643,7 +2643,8 @@ "ext-zlib": "*", "ext-zstd": "*", "php": ">=8.0", - "utopia-php/framework": "0.*.*" + "utopia-php/framework": "0.*.*", + "utopia-php/system": "0.*.*" }, "require-dev": { "laravel/pint": "1.2.*", @@ -2672,7 +2673,7 @@ "issues": "https://github.com/utopia-php/storage/issues", "source": "https://github.com/utopia-php/storage/tree/feat-transfer" }, - "time": "2023-07-25T07:40:02+00:00" + "time": "2023-08-08T02:58:34+00:00" }, { "name": "utopia-php/swoole", diff --git a/docker-compose.yml b/docker-compose.yml index 3b78272b79..597d7f68e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -721,7 +721,7 @@ services: - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG mariadb: - image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p + image: mysql:8.0.33 # fix issues when upgrading using: mysql_upgrade -u root -p container_name: appwrite-mariadb <<: *x-logging networks: @@ -753,8 +753,8 @@ services: container_name: appwrite-xtrabackup command: sleep infinity volumes: - - backup-data:/backups:rw - - restore-data:/varlibmysql:rw + - ./backup-data:/backups:rw + - ./restore-data:/varlibmysql:rw - appwrite-mariadb:/var/lib/mysql:r networks: - appwrite @@ -774,12 +774,13 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - appwrite-mariadb:/var/lib/mysql:r - - backup-data:/backups:rw + - ./backup-data:/backups:rw depends_on: - redis environment: - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_REPLICAS - _DO_SPACES_BUCKET_NAME - _DO_SPACES_ACCESS_KEY - _DO_SPACES_SECRET_KEY @@ -795,12 +796,13 @@ services: - /var/run/docker.sock:/var/run/docker.sock - ./app:/usr/src/code/app - ./src:/usr/src/code/src - - backup-data:/backups:rw - - restore-data:/varlibmysql:rw + - ./backup-data:/backups:rw + - ./restore-data:/varlibmysql:rw depends_on: - redis environment: - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_REPLICAS - _DO_SPACES_BUCKET_NAME - _DO_SPACES_ACCESS_KEY - _DO_SPACES_SECRET_KEY @@ -968,6 +970,6 @@ volumes: appwrite-config: appwrite-functions: appwrite-builds: - backup-data: - restore-data: + #backup-data: + #restore-data: # appwrite-chronograf: diff --git a/src/Appwrite/Platform/Tasks/Backup.php b/src/Appwrite/Platform/Tasks/Backup.php index b3cbfe7a44..e02fed7b93 100644 --- a/src/Appwrite/Platform/Tasks/Backup.php +++ b/src/Appwrite/Platform/Tasks/Backup.php @@ -49,7 +49,7 @@ class Backup extends Action public function action(string $database, Group $pools): void { $this->database = $database; - $this->dsn = self::getDsn($database); + $this->dsn = $this->getDsn($database); $this->s3 = new DOSpaces('/' . $this->database . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); if (is_null($this->dsn)) { @@ -223,6 +223,7 @@ class Backup extends Action foreach ( [ '_APP_CONNECTIONS_DB_PROJECT', + '_APP_CONNECTIONS_DB_REPLICAS', '_DO_SPACES_BUCKET_NAME', '_DO_SPACES_ACCESS_KEY', '_DO_SPACES_SECRET_KEY', @@ -235,4 +236,15 @@ class Backup extends Action } } } + + public function getDsn(string $database): ?DSN + { + foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_REPLICAS')) as $project) { + [$db, $dsn] = explode('=', $project); + if ($db === $database) { + return new DSN($dsn); + } + } + return null; + } } diff --git a/src/Appwrite/Platform/Tasks/Restore.php b/src/Appwrite/Platform/Tasks/Restore.php index 6d922a2baa..a0b843966e 100644 --- a/src/Appwrite/Platform/Tasks/Restore.php +++ b/src/Appwrite/Platform/Tasks/Restore.php @@ -41,7 +41,7 @@ class Restore extends Action public function action(string $id, string $cloud, string $database, string $datadir): void { $this->database = $database; - $this->dsn = self::getDsn($database); + $this->dsn = $this->getDsn($database); if (!$this->dsn instanceof DSN) { Console::error('No dsn match'); @@ -66,28 +66,20 @@ class Restore extends Action if ($cloud) { $local = new Local(self::BACKUPS_PATH . '/downloads/' . $id); - if (file_exists($local->getRoot())) { - $stdout = ''; - $stderr = ''; - $cmd = 'rm -rf ' . $local->getRoot(); - $this->log($cmd); - Console::execute($cmd, '', $stdout, $stderr); - if (!empty($stderr)) { - Console::error($stderr); - Console::exit(); - } - } - $files = $local->getRoot() . '/files'; - $this->log('Creating Directory: ' . $files); - if (!mkdir($files, 0755, true)) { + if (!file_exists($files) && !mkdir($files, 0755, true)) { Console::error('Error creating directory: ' . $files); Console::exit(); } - $this->download($filename, $local); - $this->untar($local->getPath($filename), $files); + $file = $local->getPath($filename); + + if (!file_exists($file)) { + $this->download($file, $local); + } + + $this->untar($file, $files); } else { $local = new Local(self::BACKUPS_PATH . '/' . $database . '/full/' . $id); $files = $local->getRoot() . '/files'; @@ -105,8 +97,9 @@ class Restore extends Action $this->log('Restore Finish in ' . (microtime(true) - $start) . ' seconds'); } - public function download(string $filename, Device $local) + public function download(string $file, Device $local) { + $filename = basename($file); $s3 = new DOSpaces($this->database . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); try { @@ -117,7 +110,6 @@ class Restore extends Action Console::exit(); } - $file = $local->getPath($filename); $this->log('Downloading: ' . $file); if (!$s3->transfer($path, $file, $local)) { @@ -150,7 +142,7 @@ class Restore extends Action public function decompress(string $target) { - $logfile = $target . '/../decompress.log'; + $logfile = $target . '/../log.txt'; $args = [ '--user=' . $this->dsn->getUser(), @@ -185,7 +177,7 @@ class Restore extends Action Console::exit(); } - $logfile = $target . '/../prepare.log'; + $logfile = $target . '/../log.txt'; $args = [ '--user=' . $this->dsn->getUser(), @@ -217,7 +209,7 @@ class Restore extends Action Console::exit(); } - $logfile = $target . '/../restore.log'; + $logfile = $target . '/../log.txt'; $args = [ '--user=' . $this->dsn->getUser(), @@ -243,6 +235,8 @@ class Restore extends Action Console::exit(); } + unlink($logfile); + // todo: Do we need to chown -R mysql:mysql /var/lib/mysql? } @@ -250,6 +244,7 @@ class Restore extends Action { foreach ( [ + '_APP_CONNECTIONS_DB_REPLICAS', '_DO_SPACES_BUCKET_NAME', '_DO_SPACES_ACCESS_KEY', '_DO_SPACES_SECRET_KEY', @@ -272,7 +267,7 @@ class Restore extends Action public function getDsn(string $database): ?DSN { - foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_PROJECT')) as $project) { + foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_REPLICAS')) as $project) { [$db, $dsn] = explode('=', $project); if ($db === $database) { return new DSN($dsn);