Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Restore.php

Browse Source
This commit is contained in:
fogelito
2023-08-02 19:10:45 +03:00
parent 53df65a3c4
commit e4f653be13
4 changed files with 157 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2 -1
View File
@@ -11,4 +11,5 @@ debug/
app/sdks
dev/yasd_init.php
.phpunit.result.cache
/backups/
/backups/
/shimon/
docker-compose.yml
+67 -41
View File
@@ -622,47 +622,6 @@ services:
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES
xtrabackup:
image: percona/percona-xtrabackup:latest
container_name: appwrite-xtrabackup
command: sleep infinity
#command: bash -c "sleep 86400"
environment:
- MYSQL_ROOT_PASSWORD=${_APP_DB_ROOT_PASS}
volumes:
#- xtrabackup_data:/backups
- ./backups:/backups:rw
- appwrite-mariadb:/var/lib/mysql:r
networks:
- appwrite
user: 'root:root'
appwrite-backup:
entrypoint: db-backup
<<: *x-logging
container_name: appwrite-backup
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- appwrite-mariadb:/var/lib/mysql:rw
- ./backups:/backups:rw
- /var/run/docker.sock:/var/run/docker.sock
- ./dev:/usr/local/dev
- ./vendor/utopia-php/storage:/usr/src/code/vendor/utopia-php/storage
depends_on:
- redis
environment:
- _APP_CONNECTIONS_DB_PROJECT
- _DO_SPACES_BUCKET_NAME
- _DO_SPACES_ACCESS_KEY
- _DO_SPACES_SECRET_KEY
- _DO_SPACES_REGION
- _APP_BACKUP_FOLDER
- _APP_DB_ROOT_PASS
appwrite-usage:
entrypoint: usage
<<: *x-logging
@@ -789,6 +748,73 @@ services:
# - SMARTHOST_HOST=smtp
# - SMARTHOST_PORT=587
xtrabackup:
image: percona/percona-xtrabackup:latest
container_name: appwrite-xtrabackup
command: sleep infinity
#command: bash -c "sleep 86400"
environment:
- MYSQL_ROOT_PASSWORD=${_APP_DB_ROOT_PASS}
volumes:
#- xtrabackup_data:/backups
- ./backups:/backups:rw
- appwrite-mariadb:/var/lib/mysql:r
networks:
- appwrite
user: 'root:root'
appwrite-backup:
entrypoint: db-backup
<<: *x-logging
container_name: appwrite-backup
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- appwrite-mariadb:/var/lib/mysql:rw
- ./backups:/backups:rw
- /var/run/docker.sock:/var/run/docker.sock
- ./dev:/usr/local/dev
- ./vendor/utopia-php/storage:/usr/src/code/vendor/utopia-php/storage
depends_on:
- redis
environment:
- _APP_CONNECTIONS_DB_PROJECT
- _DO_SPACES_BUCKET_NAME
- _DO_SPACES_ACCESS_KEY
- _DO_SPACES_SECRET_KEY
- _DO_SPACES_REGION
- _APP_BACKUP_FOLDER
- _APP_DB_ROOT_PASS
appwrite-restore:
command: sleep infinity
container_name: appwrite-backup-restore
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./backups:/backups:rw
- ./shimon:/var/lib/shimon:rw
- /var/run/docker.sock:/var/run/docker.sock
- ./dev:/usr/local/dev
- ./vendor/utopia-php/storage:/usr/src/code/vendor/utopia-php/storage
depends_on:
- redis
environment:
- _APP_CONNECTIONS_DB_PROJECT
- _DO_SPACES_BUCKET_NAME
- _DO_SPACES_ACCESS_KEY
- _DO_SPACES_SECRET_KEY
- _DO_SPACES_REGION
- _APP_BACKUP_FOLDER
- _APP_DB_ROOT_PASS
redis:
image: redis:7.0.4-alpine
<<: *x-logging
src/Appwrite/Platform/Tasks/Backup.php
+17 -16
View File
@@ -17,6 +17,7 @@ class Backup extends Action
protected string $host = 'mariadb';
protected int $processors = 4;
protected string $cnf = '/etc/my.cnf';
protected string $compressAlgorithm = 'lz4';
protected string $project;
public static function getName(): string
@@ -90,46 +91,45 @@ class Backup extends Action
Console::exit();
}
$logfile = $target . '/../log.txt';
$args = [
// '--defaults-file=' . $this->cnf, // [ERROR] Failed to open required defaults file: /etc/my.cnf
//'--defaults-file=' . $this->cnf, // [ERROR] Failed to open required defaults file: /etc/my.cnf
'--user=root',
'--password=' . App::getEnv('_APP_DB_ROOT_PASS'),
'--host=' . $this->host,
'--backup',
'--strict',
'--history=' . $this->project, // logs PERCONA_SCHEMA.xtrabackup_history
'--slave-info',
'--safe-slave-backup',
'--safe-slave-backup-timeout=300',
'--check-privileges', // checks if Percona XtraBackup has all the required privileges.
'--target-dir=' . $target,
'--compress=lz4',
'--parallel=' . $this->processors,
'--compress=' . $this->compressAlgorithm,
'--compress-threads=' . $this->processors,
'--rsync', // https://docs.percona.com/percona-xtrabackup/8.0/accelerate-backup-process.htm
//'--encrypt-threads=' . $this->processors,
//'--encrypt=AES256',
//'--encrypt-key-file=' . '/encryption_key_file',
//'--no-lock', // https://docs.percona.com/percona-xtrabackup/8.0/xtrabackup-option-reference.html#-no-lock
'2> ' . $logfile,
];
$stdout = '';
$stderr = '';
$cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
self::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
//self::log($stdout);
if (!empty($stderr)) {
Console::error($stderr);
//Console::exit();
}
shell_exec($cmd);
if (!str_contains($stderr, 'completed OK!')) {
$stderr = shell_exec('tail -1 ' . $logfile);
Backup::log($stderr);
if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
Console::error('Backup failed');
Console::exit();
}
if (!file_exists($target . '/xtrabackup_checkpoints')) {
Console::error('Backup failed missing files');
Console::exit();
}
// todo: remove logfile?
}
public function tar(string $directory, string $file)
@@ -165,7 +165,7 @@ class Backup extends Action
$s3 = new DOSpaces('/' . $this->project . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
if (!$s3->exists('/')) {
Console::error('Can\'t read from DO ');
Console::error('Can\'t read s3 root directory');
Console::exit();
}
@@ -229,6 +229,7 @@ class Backup extends Action
'--user=root',
'--password=rootsecretpassword',
'--backup=1',
'--strict',
'--host=' . $this->host,
'--safe-slave-backup',
'--safe-slave-backup-timeout=300',
src/Appwrite/Platform/Tasks/Restore.php
+71 -20
View File
@@ -19,6 +19,8 @@ class Restore extends Action
// todo: it will be erased!!!!
//protected string $containerName = 'appwrite-mariadb';
protected string $host = 'mariadb';
protected string $project;
protected int $processors = 4;
public static function getName(): string
@@ -33,11 +35,19 @@ class Restore extends Action
->param('id', '', new Text(100), 'Folder Identifier')
->param('cloud', null, new WhiteList(['true', 'false'], true), 'Take file from cloud?')
->param('project', null, new WhiteList(['db_fra1_02'], true), 'From _APP_CONNECTIONS_DB_PROJECT')
->callback(fn ($id, $cloud, $project) => $this->action($id, $cloud, $project));
->param('datadir', null, new Text(100), 'mysql datadir path')
->callback(fn ($id, $cloud, $project, $datadir) => $this->action($id, $cloud, $project, $datadir));
}
public function action(string $id, string $cloud, string $project): void
public function action(string $id, string $cloud, string $project, string $datadir): void
{
$datadir = '/backups/var_lib_mysql';
if (file_exists($datadir . '/sys')) {
Console::error('Datadir ' . $datadir . ' must be empty!');
Console::exit();
}
$this->checkEnvVariables();
$filename = $id . '.tar.gz';
Backup::log('--- Restore Start ' . $filename . ' --- ');
@@ -76,6 +86,7 @@ class Restore extends Action
$this->decompress($files);
$this->prepare($files);
$this->restore($files, $cloud, $datadir);
Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds");
}
@@ -117,28 +128,30 @@ class Restore extends Action
Console::exit();
}
$logfile = $target . '/../log.txt';
$args = [
'--user=root',
'--password=' . App::getEnv('_APP_DB_ROOT_PASS'),
'--host=' . $this->host,
'--decompress',
'--strict',
'--remove-original', // Removes *.lz4
'--parallel=' . $this->processors,
'--compress-threads=' . $this->processors,
'--target-dir=' . $target,
'2> ' . $logfile,
];
$stdout = '';
$stderr = '';
$cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
//Console::exit();
}
shell_exec($cmd);
if (!str_contains($stderr, 'completed OK!')) {
Console::error('Error decompressing: ' . $target);
$stderr = shell_exec('tail -1 ' . $logfile);
Backup::log($stderr);
if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
Console::error('Decompress failed');
Console::exit();
}
}
@@ -150,30 +163,68 @@ class Restore extends Action
Console::exit();
}
$logfile = $target . '/../log.txt';
$args = [
'--user=root',
'--password=' . App::getEnv('_APP_DB_ROOT_PASS'),
'--host=' . $this->host,
'--prepare',
'--strict',
'--target-dir=' . $target,
'2> ' . $logfile,
];
$stdout = '';
$stderr = '';
$cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
//Console::exit();
}
shell_exec($cmd);
if (!str_contains($stderr, 'completed OK!')) {
Console::error('Error preparing: ' . $target);
$stderr = shell_exec('tail -1 ' . $logfile);
Backup::log($stderr);
if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
Console::error('Prepare failed');
Console::exit();
}
}
public function restore(string $target, bool $cloud, string $datadir)
{
if (!file_exists($target)) {
Console::error('restore error directory not found: ' . $target);
Console::exit();
}
$logfile = $target . '/../log.txt';
$args = [
'--user=root',
'--password=' . App::getEnv('_APP_DB_ROOT_PASS'),
'--host=' . $this->host,
$cloud ? '--move-back' : '--copy-back',
'--strict',
'--target-dir=' . $target,
'--datadir=' . $datadir,
'--parallel=' . $this->processors,
'2> ' . $logfile,
];
$cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
Backup::log($cmd);
shell_exec($cmd);
$stderr = shell_exec('tail -1 ' . $logfile);
Backup::log($stderr);
if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
Console::error('Restore failed');
Console::exit();
}
// todo: Do we need to chown -R mysql:mysql /var/lib/mysql?
}
// public function action(string $filename, string $cloud, string $project, string $folder): void
// {
// $this->checkEnvVariables();