Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

compression change

Browse Source
This commit is contained in:
fogelito
2023-08-20 16:21:51 +03:00
parent b4f5adea59
commit cccf8e0204
2 changed files with 22 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Appwrite/Platform/Tasks/Backup.php
+15 -15
View File
@@ -16,13 +16,13 @@ use Utopia\Validator\Text;
class Backup extends Action
{
public const BACKUPS_PATH = '/backups';
public const BACKUP_INTERVAL_SECONDS = 60 * 60 * 1; // 4 hours;
public const COMPRESS_ALGORITHM = 'LZ4'; // faster compression and decompression
public const BACKUP_INTERVAL_SECONDS = 60 * 60 * 4; // 4 hours;
public const COMPRESS_ALGORITHM = 'zstd'; // https://www.percona.com/blog/get-your-backup-to-half-of-its-size-introducing-zstd-support-in-percona-xtrabackup/
protected ?DSN $dsn = null;
protected ?string $database = null;
protected ?DOSpaces $s3 = null;
protected string $xtrabackupContainerId;
protected int $processors = 1;
protected int $processors;
/**
* @throws Exception
@@ -91,7 +91,7 @@ class Backup extends Action
$this->setContainerId();
$this->setProcessors();
sleep(20);
//sleep(20);
Console::loop(function () {
$this->start();
@@ -102,7 +102,6 @@ class Backup extends Action
{
$start = microtime(true);
$time = date('Y_m_d_H_i_s');
$this->filename = $time . '.xbstream';
self::log('--- Backup Start ' . $time . ' --- ');
@@ -132,6 +131,7 @@ class Backup extends Action
Console::exit();
}
$filename = basename($target);
$logfile = $target . '/../backup.log';
$args = [
@@ -142,15 +142,15 @@ class Backup extends Action
'--port=' . $this->dsn->getPort(),
'--backup',
'--strict',
'--history="' . $this->database . '|' . pathinfo($this->filename, PATHINFO_FILENAME) . '"', // PERCONA_SCHEMA.xtrabackup_history
'--history="' . $this->database . '|' . pathinfo($filename, PATHINFO_FILENAME) . '"', // PERCONA_SCHEMA.xtrabackup_history
'--slave-info',
'--safe-slave-backup',
'--safe-slave-backup-timeout=300',
'--check-privileges', // checks if Percona XtraBackup has all the required privileges.
'--target-dir=' . $target,
'--parallel=' . $this->processors,
'--compress=' . self::COMPRESS_ALGORITHM,
'--compress-threads=' . $this->processors,
'--compress-threads=' . intval($this->processors / 2),
'--parallel=' . $this->processors,
'--rsync', // https://docs.percona.com/percona-xtrabackup/8.0/accelerate-backup-process.html
'2> ' . $logfile,
];
@@ -207,23 +207,23 @@ class Backup extends Action
Console::exit();
}
try {
$destination = $this->s3->getRoot() . '/' . $filename;
$destination = $this->s3->getRoot() . '/' . $filename;
try {
if (!$local->transfer($file, $destination, $this->s3)) {
Console::error('Error uploading to ' . $destination);
Console::exit();
}
if (!$this->s3->exists($destination)) {
Console::error('File not found in destination: ' . $destination);
Console::exit();
}
} catch (Exception $e) {
Console::error($e->getMessage());
Console::exit();
}
if (!$this->s3->exists($destination)) {
Console::error('File not found in destination: ' . $destination);
Console::exit();
}
if (!unlink($file)) {
Console::error('Error deleting: ' . $file);
Console::exit();
src/Appwrite/Platform/Tasks/Restore.php
+7 -6
View File
@@ -47,6 +47,7 @@ class Restore extends Action
try {
$dsn = new DSN(App::getEnv('_APP_CONNECTIONS_BACKUPS_STORAGE', ''));
$this->s3 = new DOSpaces('/' . $database . '/full', $dsn->getUser(), $dsn->getPassword(), $dsn->getPath(), $dsn->getParam('region'));
$this->s3->setTransferChunkSize(40 * 1024 * 1024); // 5MB
} catch (\Exception $e) {
Console::error($e->getMessage() . 'Invalid DSN.');
Console::exit();
@@ -59,7 +60,7 @@ class Restore extends Action
if (file_exists($datadir . '/sys') || file_exists($datadir . '/appwrite')) {
Console::error('Datadir ' . $datadir . ' must be empty!');
Console::exit();
//Console::exit();
}
$this->log('--- Restore Start ' . $id . ' --- ');
@@ -97,7 +98,7 @@ class Restore extends Action
$this->decompress($files);
$this->prepare($files);
$this->restore($files, $cloud, $datadir);
// $this->restore($files, $cloud, $datadir);
$this->log('Restore Finish in ' . (microtime(true) - $start) . ' seconds');
}
@@ -154,9 +155,9 @@ class Restore extends Action
'xtrabackup',
'--decompress',
'--strict',
'--remove-original', // Removes *.lz4 compressed files
'--remove-original',
'--parallel=' . $this->processors,
'--compress-threads=' . $this->processors,
'--compress-threads=' . intval($this->processors / 2),
'--target-dir=' . $target,
'2> ' . $logfile,
];
@@ -186,7 +187,7 @@ class Restore extends Action
$args = [
'xtrabackup',
'--prepare',
'--parallel=' . intval($this->processors / 2),
'--parallel=' . $this->processors,
'--strict',
'--target-dir=' . $target,
'2> ' . $logfile,
@@ -218,7 +219,7 @@ class Restore extends Action
'xtrabackup',
$cloud ? '--move-back' : '--copy-back',
'--strict',
'--parallel=' . intval($this->processors / 2),
'--parallel=' . $this->processors,
'--target-dir=' . $target,
'--datadir=' . $datadir,
'2> ' . $logfile,