From 970c35b31a4794bac582e5d7fa70e20ef2c75d45 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 23 Jul 2023 19:05:35 +0300 Subject: [PATCH] Transfer big data --- composer.json | 2 +- composer.lock | 38 ++++++++------ docker-compose.yml | 4 +- src/Appwrite/Platform/Tasks/Backup.php | 70 +++++++++++++++++-------- src/Appwrite/Platform/Tasks/Restore.php | 63 ++++++++-------------- 5 files changed, 96 insertions(+), 81 deletions(-) diff --git a/composer.json b/composer.json index fdaabc9897..62976b4413 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.5.*", "utopia-php/registry": "0.5.*", - "utopia-php/storage": "0.13.*", + "utopia-php/storage": "dev-feat-transfer as 0.13.3", "utopia-php/swoole": "0.5.*", "utopia-php/websocket": "0.1.0", "resque/php-resque": "1.3.6", diff --git a/composer.lock b/composer.lock index bcf4c542cb..ac4e996add 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "329498d12fabaa764c4178881cb07788", + "content-hash": "ff72579b1906a3ee181ec1209198c13f", "packages": [ { "name": "adhocore/jwt", @@ -2622,26 +2622,31 @@ }, { "name": "utopia-php/storage", - "version": "0.13.2", + "version": "dev-feat-transfer", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "ad1c00f24ca56e73888acc2af3deee4919b1194b" + "reference": "2ee3666511b3da604e7e4d66263f1dbea9f6fb04" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/ad1c00f24ca56e73888acc2af3deee4919b1194b", - "reference": "ad1c00f24ca56e73888acc2af3deee4919b1194b", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/2ee3666511b3da604e7e4d66263f1dbea9f6fb04", + "reference": "2ee3666511b3da604e7e4d66263f1dbea9f6fb04", "shasum": "" }, "require": { + "ext-brotli": "*", "ext-fileinfo": "*", + "ext-lz4": "*", + "ext-snappy": "*", + "ext-xz": "*", "ext-zlib": "*", "ext-zstd": "*", "php": ">=8.0", "utopia-php/framework": "0.*.*" }, "require-dev": { + "laravel/pint": "1.2.*", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.0.1" }, @@ -2655,12 +2660,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "A simple Storage library to manage application storage", "keywords": [ "framework", @@ -2671,9 +2670,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.13.2" + "source": "https://github.com/utopia-php/storage/tree/feat-transfer" }, - "time": "2022-12-20T11:11:35+00:00" + "time": "2023-07-19T07:14:15+00:00" }, { "name": "utopia-php/swoole", @@ -5521,9 +5520,18 @@ "time": "2023-06-08T12:52:13+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/storage", + "version": "dev-feat-transfer", + "alias": "0.13.3", + "alias_normalized": "0.13.3.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/storage": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index c43eeddb16..d516d28bc3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -635,6 +635,8 @@ services: - appwrite-mariadb:/var/lib/mysql:rw - ./backups:/backups:rw - /var/run/docker.sock:/var/run/docker.sock + - ./dev:/usr/local/dev + - ./vendor/utopia-php/storage:/usr/src/code/vendor/utopia-php/storage depends_on: - redis environment: @@ -744,7 +746,7 @@ services: - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG mariadb: - image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p + image: mysql:8.0.30 # fix issues when upgrading using: mysql_upgrade -u root -p container_name: appwrite-mariadb <<: *x-logging networks: diff --git a/src/Appwrite/Platform/Tasks/Backup.php b/src/Appwrite/Platform/Tasks/Backup.php index 88b964a3d6..a7f002f6be 100644 --- a/src/Appwrite/Platform/Tasks/Backup.php +++ b/src/Appwrite/Platform/Tasks/Backup.php @@ -6,6 +6,7 @@ use Utopia\Platform\Action; use Utopia\App; use Utopia\CLI\Console; use Utopia\Storage\Device\DOSpaces; +use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; class Backup extends Action @@ -31,34 +32,41 @@ class Backup extends Action */ public function action(): void { + self::log('--- Backup Start --- '); $this->checkEnvVariables(); - $folder = App::getEnv('_APP_BACKUP_FOLDER'); - $start = microtime(true); - $time = date('Y-m-d_H:i:s'); - self::log('--- Backup Start --- '); + $filename = date('Ymd_His') . '.tar.gz'; + $folder = App::getEnv('_APP_BACKUP_FOLDER'); + $project = explode('=', App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))[0]; + $s3 = new DOSpaces('/v1/' . $project . '/' . $folder, App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); + $local = new Local(self::$backups . '/' . $project . '/' . $folder); + $source = $local->getRoot() . '/' . $filename; - Storage::setDevice('files', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'))); - $device = Storage::getDevice('files'); + $local->setTransferChunkSize(10 * 1024 * 1024); // > 5MB - // Todo: do we want to have the backup ready on the mounted dir? or to abort? - if (!$device->exists('/')) { +// $source = '/backups/shmuel.tar.gz'; // 1452521974 +// $destination = '/shmuel/' . $time . '.tar.gz'; +// +// $local->transfer($source, $destination, $s3); +// +// if ($s3->exists($destination)) { +// Console::success('Uploaded successfully !!! ' . $destination); +// Console::exit(); +// } + + if (!$s3->exists('/')) { Console::error('Can\'t read from DO '); Console::exit(); } - $dsn = explode('=', App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))[0]; - if (!file_exists(self::$backups)) { Console::error('Mount directory does not exist'); Console::exit(); } - $backups = self::$backups . '/' . $dsn . '/' . $folder; - - if (!file_exists($backups) && !mkdir($backups, 0755, true)) { - Console::error('Error creating directory: ' . $backups); + if (!file_exists($local->getRoot()) && !mkdir($local->getRoot(), 0755, true)) { + Console::error('Error creating directory: ' . $local->getRoot()); Console::exit(); } @@ -73,9 +81,6 @@ class Backup extends Action Console::exit(); } - $filename = $time . '.tar.gz'; - $file = $backups . '/' . $filename; - // $cmd = 'tar czf ' . $file . ' --absolute-names ' . $this->directory; // // $cmd = ' @@ -86,7 +91,7 @@ class Backup extends Action $stdout = ''; $stderr = ''; - $cmd = 'cd ' . self::$mysqlDirectory . ' && tar zcf ' . $file . ' .'; + $cmd = 'cd ' . self::$mysqlDirectory . ' && tar zcf ' . $source . ' .'; self::log($cmd); Console::execute($cmd, '', $stdout, $stderr); self::log($stdout); @@ -95,6 +100,19 @@ class Backup extends Action Console::exit(); } + + if (!file_exists($source)) { + Console::error("Can't find tar file: " . $source); + Console::exit(); + } + + $filesize = \filesize($source); + self::log("Tar file size is :" . ceil($filesize / 1024 / 1024) . 'MB'); + if ($filesize < (2 * 1024)) { + Console::error("File size is very small: " . $source); + Console::exit(); + } + $stdout = ''; $stderr = ''; $cmd = 'docker start ' . $this->containerName; @@ -107,10 +125,17 @@ class Backup extends Action } try { - $path = '/' . $dsn . '/' . $folder . '/' . $filename; - self::log('Uploading ' . $path); - if (!$device->upload($file, $path)) { - Console::error('Error uploading to ' . $path); + self::log('Start transferring ' . $source); + + $destination = $s3->getRoot() . '/' . $filename; + + if (!$local->transfer($source, $destination, $s3)) { + Console::error('Error transferring to ' . $destination); + Console::exit(); + } + + if (!$s3->exists($destination)) { + Console::error('File not found on s3 ' . $destination); Console::exit(); } } catch (\Exception $e) { @@ -150,5 +175,4 @@ class Backup extends Action } } } - } diff --git a/src/Appwrite/Platform/Tasks/Restore.php b/src/Appwrite/Platform/Tasks/Restore.php index 6281ff16cc..658736afb2 100644 --- a/src/Appwrite/Platform/Tasks/Restore.php +++ b/src/Appwrite/Platform/Tasks/Restore.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Tasks; +use Exception; use Utopia\App; use Utopia\Platform\Action; use Utopia\CLI\Console; @@ -11,11 +12,10 @@ use Utopia\Storage\Storage; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -// docker compose exec appwrite-backup db-restore --cloud=false --filename=2023-07-18_12:39:59.tar.gz - class Restore extends Action { - // todo: double check this is not a production value!!!!!!!!!!!!!!! + // docker compose exec appwrite-backup db-restore --cloud=false --filename=2023-07-19_09:25:11.tar.gz --project=db_fra1_02 --folder=daily + // todo: Carefully double check this is not a production value!!!!!!!!!!!!!!! // todo: it will be erased!!!! protected string $containerName = 'appwrite-mariadb'; @@ -36,7 +36,7 @@ class Restore extends Action } /** - * @throws \Exception + * @throws Exception */ public function action(string $filename, string $cloud, string $project, string $folder): void { @@ -49,6 +49,8 @@ class Restore extends Action $file = Backup::$backups . '/' . $project . '/' . $folder . '/' . $filename; $extract = Backup::$backups . '/extract'; $original = $extract . '/original-' . time(); + $s3 = new DOSpaces('/v1/' . $project . '/' . $folder, App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')); + $download = new Local(Backup::$backups . '/downloads'); Backup::log('Creating directory ' . $original); if (!file_exists($original) && !mkdir($original, 0755, true)) { @@ -56,37 +58,28 @@ class Restore extends Action Console::exit(); } -// $tarDirectory = $extract . '/' . str_replace(['.', '-', '_', ':', 'tar', 'gz'], '', $filename); -// Backup::log('Creating directory ' . $tarDirectory); -// if (!file_exists($tarDirectory) && !mkdir($tarDirectory, 0755, true)) { -// Console::error('Error creating directory: ' . $tarDirectory); -// Console::exit(); -// } + if (!file_exists($download->getRoot()) && !mkdir($download->getRoot(), 0755, true)) { + Console::error('Error creating directory: ' . $download->getRoot()); + Console::exit(); + } if ($cloud) { - Storage::setDevice('s3', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'))); - Storage::setDevice('mount', new Local(Backup::$backups)); - - $device = Storage::getDevice('s3'); - $mount = Storage::getDevice('mount'); - try { - $folder = App::getEnv('_APP_BACKUP_FOLDER', 'hourly'); - $path = '/' . $project . '/' . $folder . '/' . $filename; + $path = $s3->getPath($filename); - if (!$device->exists($path)) { + if (!$s3->exists($path)) { Console::error('File: ' . $path . ' does not exist on cloud'); Console::exit(); } - while ($data = $device->read($path)) { - $mount->write('file-' . $file, $data); - } + $file = $download->getPath($filename); + Backup::log('Transferring ' . $path . ' => ' . $file); - Backup::log('Downloading from s3 ' . $path); - $mount->write('file-' . $file, $device->read($path)); - // $mount->get('file-' . $file, $device->read($path)); - } catch (\Exception $e) { + if (!$s3->transfer($path, $file, $download)) { + Console::error('Error transferring ' . $file); + Console::exit(); + } + } catch (Exception $e) { Console::error($e->getMessage()); Console::exit(); } @@ -112,6 +105,8 @@ class Restore extends Action $stdout = ''; $stderr = ''; $cmd = 'mv ' . Backup::$mysqlDirectory . '/* ' . ' ' . $original . '/'; + // todo: do we care about original? + $cmd = 'rm -r ' . Backup::$mysqlDirectory . '/*'; Backup::log($cmd); Console::execute($cmd, '', $stdout, $stderr); Backup::log($stdout); @@ -120,12 +115,9 @@ class Restore extends Action Console::exit(); } - $stdout = ''; $stderr = ''; - // $cmd = 'tar -xzf ' . $file . ' -C ' . $tarDirectory; $cmd = 'tar -xzf ' . $file . ' -C ' . Backup::$mysqlDirectory; - Backup::log($cmd); Console::execute($cmd, '', $stdout, $stderr); if (!empty($stderr)) { @@ -134,24 +126,13 @@ class Restore extends Action Console::exit(); } -// $stdout = ''; -// $stderr = ''; -// $cmd = 'mv ' . $tarDirectory . ' ' . Backup::$mysqlDirectory; -// Backup::log($cmd); -// Console::execute($cmd, '', $stdout, $stderr); -// Backup::log($stdout); -// if (!empty($stderr)) { -// Console::error($stderr); -// Console::exit(); -// } - $stdout = ''; $stderr = ''; $cmd = 'docker start ' . $this->containerName; Backup::log($cmd); Console::execute($cmd, '', $stdout, $stderr); - Backup::log($stdout); if (!empty($stderr)) { + Backup::log($stdout); Console::error($stderr); Console::exit(); }