Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Restore.php

Browse Source
This commit is contained in:
fogelito
2023-08-01 10:53:51 +03:00
parent c1cf6acc7a
commit 2f8b526c01
4 changed files with 178 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+2 -1
View File
@@ -84,5 +84,6 @@ _DO_SPACES_BUCKET_NAME=backups-v1
_DO_SPACES_REGION=fra1
_DO_SPACES_ACCESS_KEY=
_DO_SPACES_SECRET_KEY=
_DO_SPACES_ACCESS_KEY=
_DO_SPACES_SECRET_KEY=
_APP_BACKUP_FOLDER=daily
docker-compose.yml
+1
View File
@@ -661,6 +661,7 @@ services:
- _DO_SPACES_SECRET_KEY
- _DO_SPACES_REGION
- _APP_BACKUP_FOLDER
- _APP_DB_ROOT_PASS
appwrite-usage:
entrypoint: usage
src/Appwrite/Platform/Tasks/Backup.php
+10 -4
View File
@@ -36,7 +36,7 @@ class Backup extends Action
self::log('--- Backup Start --- ');
$start = microtime(true);
$type = 'inc';
//$type = 'full';
$type = 'full';
switch ($type) {
case 'inc':
@@ -53,7 +53,7 @@ class Backup extends Action
$local->setTransferChunkSize(5 * 1024 * 1024); // > 5MB
$backups = $local->getRoot() . '/files';
$tarFile = $local->getRoot() . '/' . $filename;
$tarFile = $local->getPath($filename);
$this->fullBackup(
backups: $backups,
@@ -78,6 +78,11 @@ class Backup extends Action
}
self::log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- ' . PHP_EOL . PHP_EOL);
Console::loop(function () {
self::log('loop');
}, 2 * 60);
}
public function fullBackup(string $backups)
@@ -94,7 +99,7 @@ class Backup extends Action
$args = [
'--user=root',
'--password=rootsecretpassword', // todo use .env
'--password=' . App::getEnv('_APP_DB_ROOT_PASS'),
'--host=mariadb',
'--backup',
//'--compress',
@@ -373,7 +378,8 @@ class Backup extends Action
'_DO_SPACES_BUCKET_NAME',
'_DO_SPACES_ACCESS_KEY',
'_DO_SPACES_SECRET_KEY',
'_DO_SPACES_REGION'
'_DO_SPACES_REGION',
'_APP_DB_ROOT_PASS'
] as $env
) {
if (empty(App::getEnv($env))) {
src/Appwrite/Platform/Tasks/Restore.php
+165 -63
View File
@@ -6,9 +6,9 @@ use Exception;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Storage\Device;
use Utopia\Storage\Device\DOSpaces;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
@@ -28,90 +28,191 @@ class Restore extends Action
{
$this
->desc('Restore a DB')
->param('filename', '', new Text(100), 'Backup file name')
->param('id', '', new Text(100), 'Folder Identifier')
->param('cloud', null, new WhiteList(['true', 'false'], true), 'Take file from cloud?')
->param('project', null, new WhiteList(['db_fra1_02', 'shimon'], true), 'From _APP_CONNECTIONS_DB_PROJECT')
->param('project', null, new WhiteList(['db_fra1_02'], true), 'From _APP_CONNECTIONS_DB_PROJECT')
->param('folder', null, new WhiteList(['hourly', 'daily'], true), 'Sub folder')
->callback(fn ($file, $cloud, $project, $folder) => $this->action($file, $cloud, $project, $folder));
->callback(fn ($id, $cloud, $project, $folder) => $this->action($id, $cloud, $project, $folder));
}
/**
* @throws Exception
*/
public function action(string $filename, string $cloud, string $project, string $folder): void
public function action(string $id, string $cloud, string $project, string $folder): void
{
$this->checkEnvVariables();
$filename = $id . '.tar.gz';
Backup::log('--- Restore Start ' . $filename . ' --- ');
$start = microtime(true);
$cloud = $cloud === 'true';
$file = Backup::$backups . '/' . $project . '/' . $folder . '/' . $filename;
$s3 = new DOSpaces('/v1/' . $project . '/' . $folder, App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
$download = new Local(Backup::$backups . '/downloads');
if (!file_exists($download->getRoot()) && !mkdir($download->getRoot(), 0755, true)) {
Console::error('Error creating directory: ' . $download->getRoot());
Console::exit();
}
$local = new Local(Backup::$backups . '/' . $project . '/' . $folder . '/' . $id);
$files = $local->getRoot() . '/files';
if ($cloud) {
try {
$path = $s3->getPath($filename);
$local = new Local(Backup::$backups . '/downloads/' . $id);
$this->download($project, $folder, $filename, $local);
$files = $local->getRoot() . '/files';
if (!file_exists($files) && !mkdir($files, 0755, true)) {
Console::error('Error creating directory: ' . $files);
Console::exit();
}
if (!$s3->exists($path)) {
Console::error('File: ' . $path . ' does not exist on cloud');
Console::exit();
}
$file = $local->getPath($filename);
$file = $download->getPath($filename);
Backup::log('Downloading: ' . $file);
$stdout = '';
$stderr = '';
$cmd = 'tar -xzf ' . $file . ' -C ' . $files;
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
Console::exit();
}
if (!$s3->transfer($path, $file, $download)) {
Console::error('Error Downloading ' . $file);
Console::exit();
}
} catch (Exception $e) {
Console::error($e->getMessage());
if (!file_exists($file)) {
Console::error('Restore file not found: ' . $file);
Console::exit();
}
}
if (!file_exists($file)) {
Console::error('Restore file not found: ' . $file);
Console::exit();
}
Backup::stopMysqlContainer($this->containerName);
$stdout = '';
$stderr = '';
//$cmd = 'mv ' . Backup::$mysqlDirectory . '/* ' . ' ' . $original . '/';
// todo: do we care about original?
$cmd = 'rm -r ' . Backup::$mysqlDirectory . '/*';
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
Backup::log($stdout);
if (!empty($stderr)) {
Console::error($stderr);
Console::exit();
}
$stdout = '';
$stderr = '';
$cmd = 'tar -xzf ' . $file . ' -C ' . Backup::$mysqlDirectory;
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
Console::exit();
}
Backup::startMysqlContainer($this->containerName);
$this->prepare($files);
Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds");
}
public function download(string $project, string $folder, string $filename, Device $local)
{
if (!file_exists($local->getRoot()) && !mkdir($local->getRoot(), 0755, true)) {
Console::error('Error creating directory: ' . $local->getRoot());
Console::exit();
}
$s3 = new DOSpaces($project . '/' . $folder, App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
try {
$path = $s3->getPath($filename);
if (!$s3->exists($path)) {
Console::error('File: ' . $path . ' does not exist on cloud');
Console::exit();
}
$file = $local->getPath($filename);
Backup::log('Downloading: ' . $file);
if (!$s3->transfer($path, $file, $local)) {
Console::error('Error Downloading ' . $file);
Console::exit();
}
} catch (Exception $e) {
Console::error($e->getMessage());
Console::exit();
}
}
public function prepare(string $target)
{
if (!file_exists($target)) {
Console::error('$target directory does not exist');
Console::exit();
}
$args = [
'--user=root',
'--password=' . App::getEnv('_APP_DB_ROOT_PASS'),
'--host=mariadb',
'--prepare',
'--target-dir=' . $target,
];
$stdout = '';
$stderr = '';
$cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
//Console::exit();
}
if (!str_contains($stderr, 'completed OK!')) {
Console::error('Error preparing $target = ' . $target);
Console::exit();
}
}
// public function action(string $filename, string $cloud, string $project, string $folder): void
// {
// $this->checkEnvVariables();
//
// Backup::log('--- Restore Start ' . $filename . ' --- ');
// $start = microtime(true);
//
// $cloud = $cloud === 'true';
// $file = Backup::$backups . '/' . $project . '/' . $folder . '/' . $filename;
// $s3 = new DOSpaces('/v1/' . $project . '/' . $folder, App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
// $download = new Local(Backup::$backups . '/downloads');
//
// if (!file_exists($download->getRoot()) && !mkdir($download->getRoot(), 0755, true)) {
// Console::error('Error creating directory: ' . $download->getRoot());
// Console::exit();
// }
//
// if ($cloud) {
// try {
// $path = $s3->getPath($filename);
//
// if (!$s3->exists($path)) {
// Console::error('File: ' . $path . ' does not exist on cloud');
// Console::exit();
// }
//
// $file = $download->getPath($filename);
// Backup::log('Downloading: ' . $file);
//
// if (!$s3->transfer($path, $file, $download)) {
// Console::error('Error Downloading ' . $file);
// Console::exit();
// }
// } catch (Exception $e) {
// Console::error($e->getMessage());
// Console::exit();
// }
// }
//
// if (!file_exists($file)) {
// Console::error('Restore file not found: ' . $file);
// Console::exit();
// }
//
// Backup::stopMysqlContainer($this->containerName);
//
// $stdout = '';
// $stderr = '';
// //$cmd = 'mv ' . Backup::$mysqlDirectory . '/* ' . ' ' . $original . '/';
// // todo: do we care about original?
// $cmd = 'rm -r ' . Backup::$mysqlDirectory . '/*';
// Backup::log($cmd);
// Console::execute($cmd, '', $stdout, $stderr);
// Backup::log($stdout);
// if (!empty($stderr)) {
// Console::error($stderr);
// Console::exit();
// }
//
// $stdout = '';
// $stderr = '';
// $cmd = 'tar -xzf ' . $file . ' -C ' . Backup::$mysqlDirectory;
// Backup::log($cmd);
// Console::execute($cmd, '', $stdout, $stderr);
// if (!empty($stderr)) {
// Console::error($stderr);
// Console::exit();
// }
//
// Backup::startMysqlContainer($this->containerName);
//
// Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds");
// }
public function checkEnvVariables(): void
{
foreach (
@@ -119,7 +220,8 @@ class Restore extends Action
'_DO_SPACES_BUCKET_NAME',
'_DO_SPACES_ACCESS_KEY',
'_DO_SPACES_SECRET_KEY',
'_DO_SPACES_REGION'
'_DO_SPACES_REGION',
'_APP_DB_ROOT_PASS'
] as $env
) {
if (empty(App::getEnv($env))) {