Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

addressing comments part 1

Browse Source
This commit is contained in:
fogelito
2023-08-08 08:46:13 +03:00
parent 50aba6a1c2
commit cfc178b046
4 changed files with 80 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+2 -3
View File
@@ -27,7 +27,8 @@ _APP_DB_PASS=password
_APP_DB_ROOT_PASS=rootsecretpassword
_APP_CONNECTIONS_MAX=3100
_APP_POOL_CLIENTS=14
_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://root:rootsecretpassword@mariadb:3306/appwrite
_APP_CONNECTIONS_DB_REPLICAS=db_fra1_02=mariadb://root:rootsecretpassword@mariadb:3306/appwrite
_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
_APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379
@@ -82,5 +83,3 @@ _APP_CONSOLE_GITHUB_SECRET=
_APP_CONSOLE_GITHUB_APP_ID=
_DO_SPACES_BUCKET_NAME=backups-v1
_DO_SPACES_REGION=fra1
_DO_SPACES_ACCESS_KEY=
_DO_SPACES_SECRET_KEY=
docker-compose.yml
+4 -5
View File
@@ -721,7 +721,7 @@ services:
- OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG
mariadb:
image: mysql:8.0.33 # fix issues when upgrading using: mysql_upgrade -u root -p
image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p
container_name: appwrite-mariadb
<<: *x-logging
networks:
@@ -770,12 +770,12 @@ services:
networks:
- appwrite
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- appwrite-mariadb:/var/lib/mysql:r
- backup-data:/backups:rw
- /var/run/docker.sock:/var/run/docker.sock
- ./dev:/usr/local/dev
depends_on:
- redis
environment:
@@ -792,12 +792,11 @@ services:
networks:
- appwrite
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- backup-data:/backups:rw
- restore-data:/varlibmysql:rw
- /var/run/docker.sock:/var/run/docker.sock
- ./dev:/usr/local/dev
depends_on:
- redis
environment:
src/Appwrite/Platform/Tasks/Backup.php
+34 -48
View File
@@ -15,39 +15,44 @@ use Utopia\Validator\Text;
class Backup extends Action
{
public const BACKUPS_PATH = '/backups';
public const BACKUP_INTERVAL_SECONDS = 60 * 60 * 4; // 4 hours;
public const COMPRESS_ALGORITHM = 'lz4';
public const CONFIG_PATH = '/etc/my.cnf';
public const PROCESSORS = 4;
protected ?DSN $dsn = null;
protected ?string $database = null;
public const BACKUPS = '/backups';
public const BACKUP_INTERVAL = 60 * 60 * 4; // 4 hours;
public const COMPRESS_ALGORITHM = 'lz4';
public const CNF = '/etc/my.cnf';
public const PROCESSORS = 4;
public static function getName(): string
{
return 'backup';
}
protected ?DOSpaces $s3 = null;
/**
* @throws Exception
*/
public function __construct()
{
$this->checkEnvVariables();
$this
->desc('Backup a DB')
->param('database', null, new Text(20), 'passed from command')
->param('database', null, new Text(20), 'Database name for example db_fra1_01')
->inject('pools')
->callback(fn(string $database, Group $pools) => $this->action($database, $pools));
}
public static function getName(): string
{
return 'backup';
}
/**
* @throws Exception
*/
public function action($database, Group $pools): void
public function action(string $database, Group $pools): void
{
$this->database = $database;
$this->dsn = self::getDsn($database);
$this->s3 = new DOSpaces('/' . $this->database . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
if (!$this->dsn instanceof DSN) {
if (is_null($this->dsn)) {
Console::error('No DSN match');
Console::exit();
}
@@ -62,8 +67,7 @@ class Backup extends Action
$pools
->get('database_' . $database)
->pop()
->getResource()
;
->getResource();
break; // leave the do-while if successful
} catch (Exception $e) {
@@ -78,7 +82,7 @@ class Backup extends Action
Console::loop(function () {
$this->start();
}, self::BACKUP_INTERVAL);
}, self::BACKUP_INTERVAL_SECONDS);
}
public function start(): void
@@ -90,7 +94,7 @@ class Backup extends Action
self::log('--- Creating backup ' . $time . ' --- ');
$filename = $time . '.tar.gz';
$local = new Local(self::BACKUPS . '/' . $this->database . '/full/' . $time);
$local = new Local(self::BACKUPS_PATH . '/' . $this->database . '/full/' . $time);
$local->setTransferChunkSize(5 * 1024 * 1024); // 5MB
$backups = $local->getRoot() . '/files';
@@ -99,15 +103,13 @@ class Backup extends Action
$this->fullBackup($backups);
$this->tar($backups, $tarFile);
$this->upload($tarFile, $local);
// todo: Do we want to delete the tar file? and remain with the folder?
// todo: Do we want to delete the backup.log?
self::log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- ' . PHP_EOL . PHP_EOL);
}
public function fullBackup(string $target)
{
if (!file_exists(self::BACKUPS)) {
if (!file_exists(self::BACKUPS_PATH)) {
Console::error('Mount directory does not exist');
Console::exit();
}
@@ -120,7 +122,6 @@ class Backup extends Action
$logfile = $target . '/../backup.log';
$args = [
//'--defaults-file=' . $this->cnf, // [ERROR] Failed to open required defaults file: /etc/my.cnf
'--user=' . $this->dsn->getUser(),
'--password=' . $this->dsn->getPassword(),
'--host=' . $this->dsn->getHost(),
@@ -136,10 +137,6 @@ class Backup extends Action
'--compress=' . self::COMPRESS_ALGORITHM,
'--compress-threads=' . self::PROCESSORS,
'--rsync', // https://docs.percona.com/percona-xtrabackup/8.0/accelerate-backup-process.html
//'--encrypt-threads=' . $this->processors,
//'--encrypt=AES256',
//'--encrypt-key-file=' . '/encryption_key_file',
//'--no-lock', // https://docs.percona.com/percona-xtrabackup/8.0/xtrabackup-option-reference.html#-no-lock
'2> ' . $logfile,
];
@@ -148,21 +145,20 @@ class Backup extends Action
shell_exec($cmd);
$stderr = shell_exec('tail -1 ' . $logfile);
Backup::log($stderr);
self::log($stderr);
if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
Console::error('Backup failed');
Console::exit();
}
// todo: remove logfile?
unlink($logfile);
}
public function tar(string $directory, string $file)
{
$stdout = '';
$stderr = '';
// Tar from inside the directory for not using --strip-components
$cmd = 'cd ' . $directory . ' && tar zcf ' . $file . ' .';
self::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
@@ -173,14 +169,14 @@ class Backup extends Action
}
if (!file_exists($file)) {
Console::error("Can't find tar file: " . $file);
Console::error('Can\'t find tar file: ' . $file);
Console::exit();
}
$filesize = \filesize($file);
self::log("Tar file size is: " . ceil($filesize / 1024 / 1024) . 'MB');
self::log('Tar file size is: ' . ceil($filesize / 1024 / 1024) . 'MB');
if ($filesize < (2 * 1024 * 1024)) {
Console::error("File size is very small: " . $file);
Console::error('File size is very small: ' . $file);
Console::exit();
}
}
@@ -188,30 +184,31 @@ class Backup extends Action
public function upload(string $file, Device $local)
{
$filename = basename($file);
$s3 = new DOSpaces('/' . $this->database . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
if (!$s3->exists('/')) {
if (!$this->s3->exists('/')) {
Console::error('Can\'t read s3 root directory');
Console::exit();
}
try {
self::log('Uploading: ' . $file);
$destination = $s3->getRoot() . '/' . $filename;
$destination = $this->s3->getRoot() . '/' . $filename;
if (!$local->transfer($file, $destination, $s3)) {
if (!$local->transfer($file, $destination, $this->s3)) {
Console::error('Error uploading to ' . $destination);
Console::exit();
}
if (!$s3->exists($destination)) {
Console::error('File not found on s3 ' . $destination);
if (!$this->s3->exists($destination)) {
Console::error('File not found in destination: ' . $destination);
Console::exit();
}
} catch (Exception $e) {
Console::error($e->getMessage());
Console::exit();
}
unlink($file);
}
public static function log(string $message): void
@@ -238,15 +235,4 @@ class Backup extends Action
}
}
}
public static function getDsn(string $database): ?DSN
{
foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_PROJECT')) as $project) {
[$db, $dsn] = explode('=', $project);
if ($db === $database) {
return new DSN($dsn);
}
}
return null;
}
}
src/Appwrite/Platform/Tasks/Restore.php
+40 -48
View File
@@ -15,14 +15,10 @@ use Utopia\Validator\WhiteList;
class Restore extends Action
{
public const BACKUPS_PATH = '/backups';
public const PROCESSORS = 4;
protected ?DSN $dsn = null;
protected string $database;
public const PROCESSORS = 4;
public static function getName(): string
{
return 'restore';
}
public function __construct()
{
@@ -30,17 +26,22 @@ class Restore extends Action
$this
->desc('Restore a DB')
->param('id', '', new Text(20), 'Folder Identifier')
->param('cloud', null, new WhiteList(['true', 'false'], true), 'Take file from cloud?')
->param('database', null, new Text(10), 'example: db_fra1_01..')
->param('datadir', null, new Text(100), 'mysql datadir path')
->param('id', '', new Text(20), 'The backup identification')
->param('cloud', null, new WhiteList(['true', 'false'], true), 'Download backup from cloud or use local directory')
->param('database', null, new Text(10), 'The Database name for example db_fra1_01')
->param('datadir', null, new Text(100), 'The Path where database would be stored')
->callback(fn ($id, $cloud, $project, $datadir) => $this->action($id, $cloud, $project, $datadir));
}
public static function getName(): string
{
return 'restore';
}
public function action(string $id, string $cloud, string $database, string $datadir): void
{
$this->database = $database;
$this->dsn = Backup::getDsn($database);
$this->dsn = self::getDsn($database);
if (!$this->dsn instanceof DSN) {
Console::error('No dsn match');
@@ -57,22 +58,19 @@ class Restore extends Action
Console::exit();
}
//todo: check if the container is stopped
//$this->checkContainerIsStopped();
$filename = $id . '.tar.gz';
Backup::log('--- Restore Start ' . $filename . ' --- ');
$this->log('--- Restore Start ' . $filename . ' --- ');
$start = microtime(true);
$cloud = $cloud === 'true';
if ($cloud) {
$local = new Local(Backup::BACKUPS . '/downloads/' . $id);
$local = new Local(self::BACKUPS_PATH . '/downloads/' . $id);
if (file_exists($local->getRoot())) {
$stdout = '';
$stderr = '';
$cmd = 'rm -rf ' . $local->getRoot();
Backup::log($cmd);
$this->log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
@@ -82,7 +80,7 @@ class Restore extends Action
$files = $local->getRoot() . '/files';
Backup::log('Creating Directory: ' . $files);
$this->log('Creating Directory: ' . $files);
if (!mkdir($files, 0755, true)) {
Console::error('Error creating directory: ' . $files);
Console::exit();
@@ -91,7 +89,7 @@ class Restore extends Action
$this->download($filename, $local);
$this->untar($local->getPath($filename), $files);
} else {
$local = new Local(Backup::BACKUPS . '/' . $database . '/full/' . $id);
$local = new Local(self::BACKUPS_PATH . '/' . $database . '/full/' . $id);
$files = $local->getRoot() . '/files';
}
@@ -104,7 +102,7 @@ class Restore extends Action
$this->prepare($files);
$this->restore($files, $cloud, $datadir);
Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds");
$this->log('Restore Finish in ' . (microtime(true) - $start) . ' seconds');
}
public function download(string $filename, Device $local)
@@ -120,7 +118,7 @@ class Restore extends Action
}
$file = $local->getPath($filename);
Backup::log('Downloading: ' . $file);
$this->log('Downloading: ' . $file);
if (!$s3->transfer($path, $file, $local)) {
Console::error('Error Downloading ' . $file);
@@ -137,7 +135,7 @@ class Restore extends Action
$stdout = '';
$stderr = '';
$cmd = 'tar -xzf ' . $file . ' -C ' . $directory;
Backup::log($cmd);
$this->log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
@@ -168,11 +166,11 @@ class Restore extends Action
];
$cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
Backup::log($cmd);
$this->log($cmd);
shell_exec($cmd);
$stderr = shell_exec('tail -1 ' . $logfile);
Backup::log($stderr);
$this->log($stderr);
if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
Console::error('Decompress failed');
@@ -200,11 +198,11 @@ class Restore extends Action
];
$cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
Backup::log($cmd);
$this->log($cmd);
shell_exec($cmd);
$stderr = shell_exec('tail -1 ' . $logfile);
Backup::log($stderr);
$this->log($stderr);
if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
Console::error('Prepare failed');
@@ -234,11 +232,11 @@ class Restore extends Action
];
$cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
Backup::log($cmd);
$this->log($cmd);
shell_exec($cmd);
$stderr = shell_exec('tail -1 ' . $logfile);
Backup::log($stderr);
$this->log($stderr);
if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
Console::error('Restore failed');
@@ -265,27 +263,21 @@ class Restore extends Action
}
}
public function checkContainerIsStopped(): void
public function log(string $message): void
{
$host = $this->dsn->getHost();
$cmd = 'docker ps --filter "status=running" --filter "name=' . $host . '"';
Backup::log($cmd);
$stderr = '';
$stdout = '';
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
Console::exit();
}
$stdout = explode(PHP_EOL, $stdout);
array_shift($stdout);
$info = array_shift($stdout);
if (!empty($info)) {
Console::error('Please stop container: ' . $host);
//Console::exit();
if (!empty($message)) {
Console::log(date('Y-m-d H:i:s') . ' ' . $message);
}
}
public function getDsn(string $database): ?DSN
{
foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_PROJECT')) as $project) {
[$db, $dsn] = explode('=', $project);
if ($db === $database) {
return new DSN($dsn);
}
}
return null;
}
}