Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Transfer big data

Browse Source
This commit is contained in:
fogelito
2023-07-25 18:38:26 +03:00
parent 970c35b31a
commit 55a6732444
4 changed files with 92 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
composer.lock
Generated
+10 -10
View File
@@ -2626,12 +2626,12 @@
"source": {
"type": "git",
"url": "https://github.com/utopia-php/storage.git",
"reference": "2ee3666511b3da604e7e4d66263f1dbea9f6fb04"
"reference": "ae190865088491d7f8aa3fc63e6906efe6e42295"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/2ee3666511b3da604e7e4d66263f1dbea9f6fb04",
"reference": "2ee3666511b3da604e7e4d66263f1dbea9f6fb04",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/ae190865088491d7f8aa3fc63e6906efe6e42295",
"reference": "ae190865088491d7f8aa3fc63e6906efe6e42295",
"shasum": ""
},
"require": {
@@ -2672,7 +2672,7 @@
"issues": "https://github.com/utopia-php/storage/issues",
"source": "https://github.com/utopia-php/storage/tree/feat-transfer"
},
"time": "2023-07-19T07:14:15+00:00"
"time": "2023-07-25T07:40:02+00:00"
},
{
"name": "utopia-php/swoole",
@@ -3655,16 +3655,16 @@
},
{
"name": "phpstan/phpdoc-parser",
"version": "1.22.1",
"version": "1.23.0",
"source": {
"type": "git",
"url": "https://github.com/phpstan/phpdoc-parser.git",
"reference": "65c39594fbd8c67abfc68bb323f86447bab79cc0"
"reference": "a2b24135c35852b348894320d47b3902a94bc494"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/65c39594fbd8c67abfc68bb323f86447bab79cc0",
"reference": "65c39594fbd8c67abfc68bb323f86447bab79cc0",
"url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/a2b24135c35852b348894320d47b3902a94bc494",
"reference": "a2b24135c35852b348894320d47b3902a94bc494",
"shasum": ""
},
"require": {
@@ -3696,9 +3696,9 @@
"description": "PHPDoc parser with support for nullable, intersection and generic types",
"support": {
"issues": "https://github.com/phpstan/phpdoc-parser/issues",
"source": "https://github.com/phpstan/phpdoc-parser/tree/1.22.1"
"source": "https://github.com/phpstan/phpdoc-parser/tree/1.23.0"
},
"time": "2023-06-29T20:46:06+00:00"
"time": "2023-07-23T22:17:56+00:00"
},
{
"name": "phpunit/php-code-coverage",
docker-compose.yml
+2 -1
View File
@@ -746,13 +746,14 @@ services:
- OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG
mariadb:
image: mysql:8.0.30 # fix issues when upgrading using: mysql_upgrade -u root -p
image: mysql:8.0.33 # fix issues when upgrading using: mysql_upgrade -u root -p
container_name: appwrite-mariadb
<<: *x-logging
networks:
- appwrite
volumes:
- appwrite-mariadb:/var/lib/mysql:rw
- ./mysql-init.sql:/mysql-init.sql:rw
ports:
- "3306:3306"
environment:
src/Appwrite/Platform/Tasks/Backup.php
+73 -32
View File
@@ -43,7 +43,7 @@ class Backup extends Action
$local = new Local(self::$backups . '/' . $project . '/' . $folder);
$source = $local->getRoot() . '/' . $filename;
$local->setTransferChunkSize(10 * 1024 * 1024); // > 5MB
$local->setTransferChunkSize(5 * 1024 * 1024); // > 5MB
// $source = '/backups/shmuel.tar.gz'; // 1452521974
// $destination = '/shmuel/' . $time . '.tar.gz';
@@ -70,27 +70,11 @@ class Backup extends Action
Console::exit();
}
$stdout = '';
$stderr = '';
$cmd = 'docker stop ' . $this->containerName;
self::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
self::log($stdout);
Console::error($stderr);
Console::exit();
}
// $cmd = 'tar czf ' . $file . ' --absolute-names ' . $this->directory;
//
// $cmd = '
// cd /var/lib/mysql
// tar czf ' . $file . ' --absolute-names *';
// Extract:::
// tar --strip-components=4 -xf 2023-07-11_12:40:47.tar.gz -C ./a2
self::stopMysqlContainer($this->containerName);
$stdout = '';
$stderr = '';
// Tar from inside the mysql directory for not using --strip-components
$cmd = 'cd ' . self::$mysqlDirectory . ' && tar zcf ' . $source . ' .';
self::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
@@ -107,30 +91,21 @@ class Backup extends Action
}
$filesize = \filesize($source);
self::log("Tar file size is :" . ceil($filesize / 1024 / 1024) . 'MB');
self::log("Tar file size is: " . ceil($filesize / 1024 / 1024) . 'MB');
if ($filesize < (2 * 1024)) {
Console::error("File size is very small: " . $source);
Console::exit();
}
$stdout = '';
$stderr = '';
$cmd = 'docker start ' . $this->containerName;
self::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
self::log($stdout);
Console::error($stderr);
Console::exit();
}
self::startMysqlContainer($this->containerName);
try {
self::log('Start transferring ' . $source);
self::log('Uploading: ' . $source);
$destination = $s3->getRoot() . '/' . $filename;
if (!$local->transfer($source, $destination, $s3)) {
Console::error('Error transferring to ' . $destination);
Console::error('Error uploading to ' . $destination);
Console::exit();
}
@@ -175,4 +150,70 @@ class Backup extends Action
}
}
}
public static function startMysqlContainer($name): void
{
$stdout = '';
$stderr = '';
$cmd = 'docker start ' . $name;
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Backup::log($stdout);
Console::error($stderr);
Console::exit();
}
sleep(5); // maybe change to while?
$cmd = 'docker ps --filter "status=running" --filter "name=' . $name . '"';
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
Console::exit();
}
$stdout = explode(PHP_EOL, $stdout);
array_shift($stdout);
$info = array_shift($stdout);
if (empty($info)) {
Console::error($name . ' container could not start check logs!');
Console::exit();
}
}
public static function stopMysqlContainer($name): void
{
$stdout = '';
$stderr = '';
$cmd = 'docker stop ' . $name;
self::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
var_dump($stdout);
if (!empty($stderr)) {
self::log($stdout);
Console::error($stderr);
Console::exit();
}
sleep(5); // maybe change to while?
$cmd = 'docker ps --filter "status=running" --filter "name=' . $name . '"';
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
Console::exit();
}
$stdout = explode(PHP_EOL, $stdout);
array_shift($stdout); // remove headers row
$info = array_shift($stdout);
if (!empty($info)) {
Console::error($name . ' container could not stop check logs!');
Console::exit();
}
}
}
src/Appwrite/Platform/Tasks/Restore.php
+7 -35
View File
@@ -30,7 +30,7 @@ class Restore extends Action
->desc('Restore a DB')
->param('filename', '', new Text(100), 'Backup file name')
->param('cloud', null, new WhiteList(['true', 'false'], true), 'Take file from cloud?')
->param('project', null, new WhiteList(['db_fra1_02'], true), 'From _APP_CONNECTIONS_DB_PROJECT')
->param('project', null, new WhiteList(['db_fra1_02', 'shimon'], true), 'From _APP_CONNECTIONS_DB_PROJECT')
->param('folder', null, new WhiteList(['hourly', 'daily'], true), 'Sub folder')
->callback(fn ($file, $cloud, $project, $folder) => $this->action($file, $cloud, $project, $folder));
}
@@ -47,17 +47,9 @@ class Restore extends Action
$cloud = $cloud === 'true';
$file = Backup::$backups . '/' . $project . '/' . $folder . '/' . $filename;
$extract = Backup::$backups . '/extract';
$original = $extract . '/original-' . time();
$s3 = new DOSpaces('/v1/' . $project . '/' . $folder, App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
$download = new Local(Backup::$backups . '/downloads');
Backup::log('Creating directory ' . $original);
if (!file_exists($original) && !mkdir($original, 0755, true)) {
Console::error('Error creating directory: ' . $original);
Console::exit();
}
if (!file_exists($download->getRoot()) && !mkdir($download->getRoot(), 0755, true)) {
Console::error('Error creating directory: ' . $download->getRoot());
Console::exit();
@@ -73,10 +65,10 @@ class Restore extends Action
}
$file = $download->getPath($filename);
Backup::log('Transferring ' . $path . ' => ' . $file);
Backup::log('Downloading: ' . $file);
if (!$s3->transfer($path, $file, $download)) {
Console::error('Error transferring ' . $file);
Console::error('Error Downloading ' . $file);
Console::exit();
}
} catch (Exception $e) {
@@ -85,26 +77,16 @@ class Restore extends Action
}
}
if (!file_exists($file) || empty($file)) {
if (!file_exists($file)) {
Console::error('Restore file not found: ' . $file);
Console::exit();
}
$stdout = '';
$stderr = '';
$cmd = 'docker stop ' . $this->containerName;
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Backup::log($stdout);
Console::error($stderr);
Console::exit();
}
Backup::stopMysqlContainer($this->containerName);
$stdout = '';
$stderr = '';
$cmd = 'mv ' . Backup::$mysqlDirectory . '/* ' . ' ' . $original . '/';
//$cmd = 'mv ' . Backup::$mysqlDirectory . '/* ' . ' ' . $original . '/';
// todo: do we care about original?
$cmd = 'rm -r ' . Backup::$mysqlDirectory . '/*';
Backup::log($cmd);
@@ -121,21 +103,11 @@ class Restore extends Action
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Backup::log($stdout);
Console::error($stderr);
Console::exit();
}
$stdout = '';
$stderr = '';
$cmd = 'docker start ' . $this->containerName;
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Backup::log($stdout);
Console::error($stderr);
Console::exit();
}
Backup::startMysqlContainer($this->containerName);
Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds");
}