diff --git a/composer.lock b/composer.lock index ac4e996add..d970e04897 100644 --- a/composer.lock +++ b/composer.lock @@ -2626,12 +2626,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "2ee3666511b3da604e7e4d66263f1dbea9f6fb04" + "reference": "ae190865088491d7f8aa3fc63e6906efe6e42295" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/2ee3666511b3da604e7e4d66263f1dbea9f6fb04", - "reference": "2ee3666511b3da604e7e4d66263f1dbea9f6fb04", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/ae190865088491d7f8aa3fc63e6906efe6e42295", + "reference": "ae190865088491d7f8aa3fc63e6906efe6e42295", "shasum": "" }, "require": { @@ -2672,7 +2672,7 @@ "issues": "https://github.com/utopia-php/storage/issues", "source": "https://github.com/utopia-php/storage/tree/feat-transfer" }, - "time": "2023-07-19T07:14:15+00:00" + "time": "2023-07-25T07:40:02+00:00" }, { "name": "utopia-php/swoole", @@ -3655,16 +3655,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.22.1", + "version": "1.23.0", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "65c39594fbd8c67abfc68bb323f86447bab79cc0" + "reference": "a2b24135c35852b348894320d47b3902a94bc494" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/65c39594fbd8c67abfc68bb323f86447bab79cc0", - "reference": "65c39594fbd8c67abfc68bb323f86447bab79cc0", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/a2b24135c35852b348894320d47b3902a94bc494", + "reference": "a2b24135c35852b348894320d47b3902a94bc494", "shasum": "" }, "require": { @@ -3696,9 +3696,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.22.1" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.23.0" }, - "time": "2023-06-29T20:46:06+00:00" + "time": "2023-07-23T22:17:56+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/docker-compose.yml b/docker-compose.yml index d516d28bc3..66b9f00355 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -746,13 +746,14 @@ services: - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG mariadb: - image: mysql:8.0.30 # fix issues when upgrading using: mysql_upgrade -u root -p + image: mysql:8.0.33 # fix issues when upgrading using: mysql_upgrade -u root -p container_name: appwrite-mariadb <<: *x-logging networks: - appwrite volumes: - appwrite-mariadb:/var/lib/mysql:rw + - ./mysql-init.sql:/mysql-init.sql:rw ports: - "3306:3306" environment: diff --git a/src/Appwrite/Platform/Tasks/Backup.php b/src/Appwrite/Platform/Tasks/Backup.php index a7f002f6be..962a29c143 100644 --- a/src/Appwrite/Platform/Tasks/Backup.php +++ b/src/Appwrite/Platform/Tasks/Backup.php @@ -43,7 +43,7 @@ class Backup extends Action $local = new Local(self::$backups . '/' . $project . '/' . $folder); $source = $local->getRoot() . '/' . $filename; - $local->setTransferChunkSize(10 * 1024 * 1024); // > 5MB + $local->setTransferChunkSize(5 * 1024 * 1024); // > 5MB // $source = '/backups/shmuel.tar.gz'; // 1452521974 // $destination = '/shmuel/' . $time . '.tar.gz'; @@ -70,27 +70,11 @@ class Backup extends Action Console::exit(); } - $stdout = ''; - $stderr = ''; - $cmd = 'docker stop ' . $this->containerName; - self::log($cmd); - Console::execute($cmd, '', $stdout, $stderr); - if (!empty($stderr)) { - self::log($stdout); - Console::error($stderr); - Console::exit(); - } - - // $cmd = 'tar czf ' . $file . ' --absolute-names ' . $this->directory; - // - // $cmd = ' - // cd /var/lib/mysql - // tar czf ' . $file . ' --absolute-names *'; - // Extract::: - // tar --strip-components=4 -xf 2023-07-11_12:40:47.tar.gz -C ./a2 + self::stopMysqlContainer($this->containerName); $stdout = ''; $stderr = ''; + // Tar from inside the mysql directory for not using --strip-components $cmd = 'cd ' . self::$mysqlDirectory . ' && tar zcf ' . $source . ' .'; self::log($cmd); Console::execute($cmd, '', $stdout, $stderr); @@ -107,30 +91,21 @@ class Backup extends Action } $filesize = \filesize($source); - self::log("Tar file size is :" . ceil($filesize / 1024 / 1024) . 'MB'); + self::log("Tar file size is: " . ceil($filesize / 1024 / 1024) . 'MB'); if ($filesize < (2 * 1024)) { Console::error("File size is very small: " . $source); Console::exit(); } - $stdout = ''; - $stderr = ''; - $cmd = 'docker start ' . $this->containerName; - self::log($cmd); - Console::execute($cmd, '', $stdout, $stderr); - if (!empty($stderr)) { - self::log($stdout); - Console::error($stderr); - Console::exit(); - } + self::startMysqlContainer($this->containerName); try { - self::log('Start transferring ' . $source); + self::log('Uploading: ' . $source); $destination = $s3->getRoot() . '/' . $filename; if (!$local->transfer($source, $destination, $s3)) { - Console::error('Error transferring to ' . $destination); + Console::error('Error uploading to ' . $destination); Console::exit(); } @@ -175,4 +150,70 @@ class Backup extends Action } } } + + public static function startMysqlContainer($name): void + { + $stdout = ''; + $stderr = ''; + $cmd = 'docker start ' . $name; + Backup::log($cmd); + Console::execute($cmd, '', $stdout, $stderr); + if (!empty($stderr)) { + Backup::log($stdout); + Console::error($stderr); + Console::exit(); + } + + sleep(5); // maybe change to while? + + $cmd = 'docker ps --filter "status=running" --filter "name=' . $name . '"'; + Backup::log($cmd); + Console::execute($cmd, '', $stdout, $stderr); + + if (!empty($stderr)) { + Console::error($stderr); + Console::exit(); + } + + $stdout = explode(PHP_EOL, $stdout); + array_shift($stdout); + $info = array_shift($stdout); + if (empty($info)) { + Console::error($name . ' container could not start check logs!'); + Console::exit(); + } + } + + public static function stopMysqlContainer($name): void + { + $stdout = ''; + $stderr = ''; + $cmd = 'docker stop ' . $name; + self::log($cmd); + Console::execute($cmd, '', $stdout, $stderr); + var_dump($stdout); + if (!empty($stderr)) { + self::log($stdout); + Console::error($stderr); + Console::exit(); + } + + sleep(5); // maybe change to while? + + $cmd = 'docker ps --filter "status=running" --filter "name=' . $name . '"'; + Console::execute($cmd, '', $stdout, $stderr); + + if (!empty($stderr)) { + Console::error($stderr); + Console::exit(); + } + + $stdout = explode(PHP_EOL, $stdout); + array_shift($stdout); // remove headers row + $info = array_shift($stdout); + if (!empty($info)) { + Console::error($name . ' container could not stop check logs!'); + Console::exit(); + } + } } diff --git a/src/Appwrite/Platform/Tasks/Restore.php b/src/Appwrite/Platform/Tasks/Restore.php index 658736afb2..de10d45ce6 100644 --- a/src/Appwrite/Platform/Tasks/Restore.php +++ b/src/Appwrite/Platform/Tasks/Restore.php @@ -30,7 +30,7 @@ class Restore extends Action ->desc('Restore a DB') ->param('filename', '', new Text(100), 'Backup file name') ->param('cloud', null, new WhiteList(['true', 'false'], true), 'Take file from cloud?') - ->param('project', null, new WhiteList(['db_fra1_02'], true), 'From _APP_CONNECTIONS_DB_PROJECT') + ->param('project', null, new WhiteList(['db_fra1_02', 'shimon'], true), 'From _APP_CONNECTIONS_DB_PROJECT') ->param('folder', null, new WhiteList(['hourly', 'daily'], true), 'Sub folder') ->callback(fn ($file, $cloud, $project, $folder) => $this->action($file, $cloud, $project, $folder)); } @@ -47,17 +47,9 @@ class Restore extends Action $cloud = $cloud === 'true'; $file = Backup::$backups . '/' . $project . '/' . $folder . '/' . $filename; - $extract = Backup::$backups . '/extract'; - $original = $extract . '/original-' . time(); $s3 = new DOSpaces('/v1/' . $project . '/' . $folder, App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); $download = new Local(Backup::$backups . '/downloads'); - Backup::log('Creating directory ' . $original); - if (!file_exists($original) && !mkdir($original, 0755, true)) { - Console::error('Error creating directory: ' . $original); - Console::exit(); - } - if (!file_exists($download->getRoot()) && !mkdir($download->getRoot(), 0755, true)) { Console::error('Error creating directory: ' . $download->getRoot()); Console::exit(); @@ -73,10 +65,10 @@ class Restore extends Action } $file = $download->getPath($filename); - Backup::log('Transferring ' . $path . ' => ' . $file); + Backup::log('Downloading: ' . $file); if (!$s3->transfer($path, $file, $download)) { - Console::error('Error transferring ' . $file); + Console::error('Error Downloading ' . $file); Console::exit(); } } catch (Exception $e) { @@ -85,26 +77,16 @@ class Restore extends Action } } - if (!file_exists($file) || empty($file)) { + if (!file_exists($file)) { Console::error('Restore file not found: ' . $file); Console::exit(); } - $stdout = ''; - $stderr = ''; - $cmd = 'docker stop ' . $this->containerName; - Backup::log($cmd); - Console::execute($cmd, '', $stdout, $stderr); - if (!empty($stderr)) { - Backup::log($stdout); - Console::error($stderr); - Console::exit(); - } - + Backup::stopMysqlContainer($this->containerName); $stdout = ''; $stderr = ''; - $cmd = 'mv ' . Backup::$mysqlDirectory . '/* ' . ' ' . $original . '/'; + //$cmd = 'mv ' . Backup::$mysqlDirectory . '/* ' . ' ' . $original . '/'; // todo: do we care about original? $cmd = 'rm -r ' . Backup::$mysqlDirectory . '/*'; Backup::log($cmd); @@ -121,21 +103,11 @@ class Restore extends Action Backup::log($cmd); Console::execute($cmd, '', $stdout, $stderr); if (!empty($stderr)) { - Backup::log($stdout); Console::error($stderr); Console::exit(); } - $stdout = ''; - $stderr = ''; - $cmd = 'docker start ' . $this->containerName; - Backup::log($cmd); - Console::execute($cmd, '', $stdout, $stderr); - if (!empty($stderr)) { - Backup::log($stdout); - Console::error($stderr); - Console::exit(); - } + Backup::startMysqlContainer($this->containerName); Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds"); }