From 748abb29cf324c416bb6afe12eeef4c5d6d320ba Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 12 Jul 2023 14:55:09 +0300 Subject: [PATCH] structure --- .gitignore | 1 + Dockerfile | 8 ++ bin/db-backup | 3 + bin/db-restore | 3 + composer.lock | 26 +++--- docker-compose.yml | 19 ++++ src/Appwrite/Platform/Services/Tasks.php | 5 ++ src/Appwrite/Platform/Tasks/DbBackup.php | 101 ++++++++++++++++++++++ src/Appwrite/Platform/Tasks/DbRestore.php | 73 ++++++++++++++++ 9 files changed, 226 insertions(+), 13 deletions(-) create mode 100644 bin/db-backup create mode 100644 bin/db-restore create mode 100644 src/Appwrite/Platform/Tasks/DbBackup.php create mode 100644 src/Appwrite/Platform/Tasks/DbRestore.php diff --git a/.gitignore b/.gitignore index a0e910f79b..0301e5d92e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ debug/ app/sdks dev/yasd_init.php .phpunit.result.cache +/backups/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f40d6475d1..2a810f07aa 100755 --- a/Dockerfile +++ b/Dockerfile @@ -123,6 +123,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/calc-tier-stats && \ chmod +x /usr/local/bin/patch-delete-project-collections && \ chmod +x /usr/local/bin/maintenance && \ + chmod +x /usr/local/bin/db-backup && \ + chmod +x /usr/local/bin/db-restore && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ @@ -159,6 +161,12 @@ RUN echo "default_socket_timeout=-1" >> /usr/local/etc/php/conf.d/appwrite.ini RUN echo "opcache.jit_buffer_size=100M" >> /usr/local/etc/php/conf.d/appwrite.ini RUN echo "opcache.jit=1235" >> /usr/local/etc/php/conf.d/appwrite.ini +RUN \ + apk update \ + && apk add --no-cache \ + docker \ + && rm -rf /var/cache/apk/* + EXPOSE 80 CMD [ "php", "app/http.php", "-dopcache.preload=opcache.preload=/usr/src/code/app/preload.php" ] \ No newline at end of file diff --git a/bin/db-backup b/bin/db-backup new file mode 100644 index 0000000000..b56614e986 --- /dev/null +++ b/bin/db-backup @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php db-backup $@ \ No newline at end of file diff --git a/bin/db-restore b/bin/db-restore new file mode 100644 index 0000000000..4b5f24b6d1 --- /dev/null +++ b/bin/db-restore @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php db-restore $@ \ No newline at end of file diff --git a/composer.lock b/composer.lock index aed1b97dd5..bcf4c542cb 100644 --- a/composer.lock +++ b/composer.lock @@ -3253,16 +3253,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.15.5", + "version": "v4.16.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "11e2663a5bc9db5d714eedb4277ee300403b4a9e" + "reference": "19526a33fb561ef417e822e85f08a00db4059c17" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/11e2663a5bc9db5d714eedb4277ee300403b4a9e", - "reference": "11e2663a5bc9db5d714eedb4277ee300403b4a9e", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/19526a33fb561ef417e822e85f08a00db4059c17", + "reference": "19526a33fb561ef417e822e85f08a00db4059c17", "shasum": "" }, "require": { @@ -3303,9 +3303,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.5" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.16.0" }, - "time": "2023-05-19T20:20:00+00:00" + "time": "2023-06-25T14:52:30+00:00" }, { "name": "phar-io/manifest", @@ -3656,16 +3656,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.22.0", + "version": "1.22.1", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "ec58baf7b3c7f1c81b3b00617c953249fb8cf30c" + "reference": "65c39594fbd8c67abfc68bb323f86447bab79cc0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/ec58baf7b3c7f1c81b3b00617c953249fb8cf30c", - "reference": "ec58baf7b3c7f1c81b3b00617c953249fb8cf30c", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/65c39594fbd8c67abfc68bb323f86447bab79cc0", + "reference": "65c39594fbd8c67abfc68bb323f86447bab79cc0", "shasum": "" }, "require": { @@ -3697,9 +3697,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.22.0" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.22.1" }, - "time": "2023-06-01T12:35:21+00:00" + "time": "2023-06-29T20:46:06+00:00" }, { "name": "phpunit/php-code-coverage", @@ -5547,5 +5547,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/docker-compose.yml b/docker-compose.yml index d36b175c0c..acf15c6d70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -622,6 +622,25 @@ services: - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES + appwrite-backup: + entrypoint: db-backup + <<: *x-logging + container_name: appwrite-backup + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + - appwrite-mariadb:/var/lib/mysql:rw + - ./backups:/backups:rw + # - ./extracts:/extracts:rw + - /var/run/docker.sock:/var/run/docker.sock + depends_on: + - redis + environment: + - _APP_CONNECTIONS_DB_PROJECT + appwrite-usage: entrypoint: usage <<: *x-logging diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 10c573da42..fbf189c580 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -2,6 +2,9 @@ namespace Appwrite\Platform\Services; +use Appwrite\Platform\Tasks\Backup; +use Appwrite\Platform\Tasks\DbBackup; +use Appwrite\Platform\Tasks\DbRestore; use Utopia\Platform\Service; use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; @@ -48,6 +51,8 @@ class Tasks extends Service ->addAction(CalcUsersStats::getName(), new CalcUsersStats()) ->addAction(CalcTierStats::getName(), new CalcTierStats()) ->addAction(PatchDeleteProjectCollections::getName(), new PatchDeleteProjectCollections()) + ->addAction(DbBackup::getName(), new DbBackup()) + ->addAction(DbRestore::getName(), new DbRestore()) ; } } diff --git a/src/Appwrite/Platform/Tasks/DbBackup.php b/src/Appwrite/Platform/Tasks/DbBackup.php new file mode 100644 index 0000000000..709b47fea5 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/DbBackup.php @@ -0,0 +1,101 @@ +desc('Backup a DB') + ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain.', true) + ->callback(fn ($domain) => $this->action($domain)); + } + + /** + * @throws \Exception + */ + public function action(string $domain): void + { + + if (empty(App::getEnv('_APP_CONNECTIONS_DB_PROJECT', ''))) { + Console::error('Can\'t read .env variables'); + Console::exit(); + } + + $time = date('Y-m-d_H:i:s'); + Console::log('Backup started:' . $time); + +// Console::log('creating directory:' . $this->backups); +// if (!file_exists($this->backups) && !mkdir($this->backups, 0755, true)) { +// Console::error('Error creating directory: ' . $this->backups); +// Console::exit(); +// } + + $stdout = ''; + $stderr = ''; + $cmd = 'docker stop ' . $this->containerName; + Console::log($cmd); + $code = Console::execute($cmd, '', $stdout, $stderr); + if (!empty($stderr)) { + Console::error($stderr); + Console::exit(); + } + + Console::log($stdout); + + $filename = $time . '.tar.gz'; + $file = self::$backups . '/' . $filename; + + // $cmd = 'tar czf ' . $file . ' --absolute-names ' . $this->directory; + // + // $cmd = ' + // cd /var/lib/mysql + // tar czf ' . $file . ' --absolute-names *'; + // Extract::: + // tar --strip-components=4 -xf 2023-07-11_12:40:47.tar.gz -C ./a2 + + $stdout = ''; + $stderr = ''; + $cmd = 'cd ' . $this->directory . ' && tar zcf ' . $file . ' .'; + $code = Console::execute($cmd, '', $stdout, $stderr); + Console::log($cmd); + Console::log($stdout); + if (!empty($stderr)) { + Console::error($stderr); + Console::exit(); + } + + $stdout = ''; + $stderr = ''; + $cmd = 'docker start ' . $this->containerName; + + $code = Console::execute($cmd, '', $stdout, $stderr); + + Console::log($stdout); + if (!empty($stderr)) { + Console::error($stderr); + Console::exit(); + } + + Console::loop(function () { + Console::log('Hello'); + }, 1); + + } +} diff --git a/src/Appwrite/Platform/Tasks/DbRestore.php b/src/Appwrite/Platform/Tasks/DbRestore.php new file mode 100644 index 0000000000..9550a97bfc --- /dev/null +++ b/src/Appwrite/Platform/Tasks/DbRestore.php @@ -0,0 +1,73 @@ +desc('Restore a DB') + ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain.', true) + ->callback(fn ($domain) => $this->action($domain)); + } + + /** + * @throws \Exception + */ + public function action(string $domain): void + { + $filename = '2023-07-12_09:37:47.tar.gz'; + + Console::log('Restoring backup' . $filename); + + $file = DbBackup::$backups . '/' . $filename; + + if (!file_exists($file)) { + Console::error('File not found: ' . $file); + Console::exit(); + } + + // Todo: shut down target container + + if (!file_exists($this->extract) && !mkdir($this->extract, 0755, true)) { + Console::error('Error creating directory: ' . $this->extract); + Console::exit(); + } + + $stdout = ''; + $stderr = ''; + + $start = microtime(true); + Console::log($start); + + $cmd = 'tar -xzf ' . $file . ' -C ' . $this->extract; + Console::log($cmd); + $code = Console::execute($cmd, '', $stdout, $stderr); + + Console::log('time end ' . (microtime(true) - $start)); + + if (!empty($stderr)) { + Console::error($stderr); + Console::exit(); + } + + Console::log($stdout); +// // Todo: start down target container + + Console::log("Restore Finish"); + } +}