Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

dsn changes

Browse Source
This commit is contained in:
fogelito
2023-08-07 15:54:09 +03:00
parent d730b41b5f
commit 9fab5c46f1
4 changed files with 131 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+3 -1
View File
@@ -27,7 +27,7 @@ _APP_DB_PASS=password
_APP_DB_ROOT_PASS=rootsecretpassword
_APP_CONNECTIONS_MAX=3100
_APP_POOL_CLIENTS=14
_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_DB_PROJECT=db_fra1_01=mariadb://root:rootsecretpassword@mariadb:3306/appwrite,db_fra1_02=mariadb://root:rootsecretpassword@mariadb:3306/appwrite
_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
_APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379
@@ -84,3 +84,5 @@ _DO_SPACES_BUCKET_NAME=backups-v1
_DO_SPACES_REGION=fra1
_DO_SPACES_ACCESS_KEY=
_DO_SPACES_SECRET_KEY=
_DO_SPACES_ACCESS_KEY=
_DO_SPACES_SECRET_KEY=
docker-compose.yml
+3 -7
View File
@@ -752,8 +752,6 @@ services:
image: percona/percona-xtrabackup:latest
container_name: appwrite-xtrabackup
command: sleep infinity
environment:
- MYSQL_ROOT_PASSWORD=${_APP_DB_ROOT_PASS}
volumes:
- ./backups:/backups:rw
- ./varlibmysql:/varlibmysql:rw
@@ -764,6 +762,8 @@ services:
appwrite-backup:
entrypoint: db-backup
command:
- --database=db_fra1_02
<<: *x-logging
container_name: appwrite-backup
image: appwrite-dev
@@ -772,7 +772,7 @@ services:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- appwrite-mariadb:/var/lib/mysql:rw
- appwrite-mariadb:/var/lib/mysql:r
- ./backups:/backups:rw
- /var/run/docker.sock:/var/run/docker.sock
- ./dev:/usr/local/dev
@@ -784,8 +784,6 @@ services:
- _DO_SPACES_ACCESS_KEY
- _DO_SPACES_SECRET_KEY
- _DO_SPACES_REGION
- _APP_BACKUP_FOLDER
- _APP_DB_ROOT_PASS
appwrite-restore:
command: sleep infinity
@@ -808,8 +806,6 @@ services:
- _DO_SPACES_ACCESS_KEY
- _DO_SPACES_SECRET_KEY
- _DO_SPACES_REGION
- _APP_BACKUP_FOLDER
- _APP_DB_ROOT_PASS
redis:
image: redis:7.0.4-alpine
src/Appwrite/Platform/Tasks/Backup.php
+51 -116
View File
@@ -2,6 +2,8 @@
namespace Appwrite\Platform\Tasks;
use Exception;
use Utopia\DSN\DSN;
use Utopia\Platform\Action;
use Utopia\App;
use Utopia\CLI\Console;
@@ -9,11 +11,12 @@ use Utopia\Pools\Group;
use Utopia\Storage\Device;
use Utopia\Storage\Device\DOSpaces;
use Utopia\Storage\Device\Local;
use Utopia\Validator\Text;
class Backup extends Action
{
protected string $host = 'mariadb';
protected string $project;
protected ?DSN $dsn = null;
protected ?string $database = null;
public const BACKUPS = '/backups';
//public const BACKUP_INTERVAL = 60 * 60 * 4; // 4 hours;
public const BACKUP_INTERVAL = 300; // 4 hours;
@@ -29,19 +32,27 @@ class Backup extends Action
public function __construct()
{
$this->checkEnvVariables();
$this->project = explode('=', App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))[0];
$this
->desc('Backup a DB')
->param('database', null, new Text(20), 'passed from command')
->inject('pools')
->callback(fn(Group $pools) => $this->action($pools));
->callback(fn(string $database, Group $pools) => $this->action($database, $pools));
}
/**
* @throws \Exception
* @throws Exception
*/
public function action($pools): void
public function action($database, $pools): void
{
$this->database = $database;
$this->dsn = self::getDsn($database);
if (!$this->dsn instanceof DSN) {
Console::error('No dsn match');
Console::exit();
}
$attempts = 0;
$max = 10;
$sleep = 5;
@@ -50,16 +61,16 @@ class Backup extends Action
try {
$attempts++;
$pools
->get('database_' . $this->project)
->get('database_' . $database)
->pop()
->getResource()
;
break; // leave the do-while if successful
} catch (\Exception $e) {
} catch (Exception $e) {
Console::warning("Database not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('Failed to connect to database: ' . $e->getMessage());
throw new Exception('Failed to connect to database: ' . $e->getMessage());
}
sleep($sleep);
@@ -73,36 +84,24 @@ class Backup extends Action
public function start(): void
{
self::log('--- Backup Start --- ');
$start = microtime(true);
//$type = 'inc';
$type = 'full';
$time = date('Y_m_d_H_i_s');
switch ($type) {
case 'inc':
$this->incrementalBackup();
break;
case 'full':
$time = date('Y_m_d_H_i_s');
self::log('--- Creating backup ' . $time . ' --- ');
$filename = $time . '.tar.gz';
$local = new Local(self::BACKUPS . '/' . $this->project . '/full/' . $time);
$local->setTransferChunkSize(5 * 1024 * 1024); // > 5MB
self::log('--- Backup Start --- ');
self::log('--- Creating backup ' . $time . ' --- ');
$backups = $local->getRoot() . '/files';
$tarFile = $local->getPath($filename);
$filename = $time . '.tar.gz';
$local = new Local(self::BACKUPS . '/' . $this->database . '/full/' . $time);
$local->setTransferChunkSize(5 * 1024 * 1024); // > 5MB
$this->fullBackup($backups);
$this->tar($backups, $tarFile);
$this->upload($tarFile, $local);
// todo: Do we want to delete the tar file? and remain with the folder?
// todo: Do we want to delete the tar log.file?
break;
$backups = $local->getRoot() . '/files';
$tarFile = $local->getPath($filename);
default:
Console::error('No type detected');
Console::exit();
}
$this->fullBackup($backups);
$this->tar($backups, $tarFile);
$this->upload($tarFile, $local);
// todo: Do we want to delete the tar file? and remain with the folder?
// todo: Do we want to delete the backup.log?
self::log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- ' . PHP_EOL . PHP_EOL);
}
@@ -119,16 +118,16 @@ class Backup extends Action
Console::exit();
}
$logfile = $target . '/../log.txt';
$logfile = $target . '/../backup.log';
$args = [
//'--defaults-file=' . $this->cnf, // [ERROR] Failed to open required defaults file: /etc/my.cnf
'--user=root',
'--password=' . App::getEnv('_APP_DB_ROOT_PASS'),
'--host=' . $this->host,
'--user=' . $this->dsn->getUser(),
'--password=' . $this->dsn->getPassword(),
'--host=' . $this->dsn->getHost(),
'--backup',
'--strict',
'--history=' . $this->project, // logs PERCONA_SCHEMA.xtrabackup_history
'--history=' . $this->database, // logs PERCONA_SCHEMA.xtrabackup_history name attribute
'--slave-info',
'--safe-slave-backup',
'--safe-slave-backup-timeout=300',
@@ -190,7 +189,7 @@ class Backup extends Action
public function upload(string $file, Device $local)
{
$filename = basename($file);
$s3 = new DOSpaces('/' . $this->project . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
$s3 = new DOSpaces('/' . $this->database . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
if (!$s3->exists('/')) {
Console::error('Can\'t read s3 root directory');
@@ -210,86 +209,12 @@ class Backup extends Action
Console::error('File not found on s3 ' . $destination);
Console::exit();
}
} catch (\Exception $e) {
} catch (Exception $e) {
Console::error($e->getMessage());
Console::exit();
}
}
public function incrementalBackup()
{
$project = $this->project;
//$folder = ceil(time() / 60 * 60);
//$folder = date('Y_m_d');
//$folder = ceil(date('z') / 7); // day of the year 0-365
$folder = date('W'); // week of the year 0 - 51
$folder = 'v1_' . date('Y') . '_' . $folder;
$local = new Local(self::BACKUPS . '/' . $project . '/inc/' . $folder);
$position = 1;
$target = $local->getRoot() . '/' . $position;
$base = '';
if (file_exists($local->getRoot() . '/position')) {
$position = intval(file_get_contents($local->getRoot() . '/position'));
if (!file_exists($local->getRoot() . '/' . $position . '/xtrabackup_checkpoints')) {
Console::error('Backup ' . $folder . ' is garbage!!!');
Console::exit();
}
$base = $local->getRoot() . '/' . $position;
$position += 1;
$target = $local->getRoot() . '/' . $position;
}
Console::success($base);
Console::success($target);
if (!file_exists($target) && !mkdir($target, 0755, true)) {
Console::error('Error creating backup directory: ' . $target);
Console::exit();
}
file_put_contents($local->getRoot() . '/position', $position);
$args = [
'--user=root',
'--password=rootsecretpassword',
'--backup=1',
'--strict',
'--host=' . $this->host,
'--safe-slave-backup',
'--safe-slave-backup-timeout=300',
'--check-privileges',
//'--no-lock', // https://docs.percona.com/percona-xtrabackup/8.0/xtrabackup-option-reference.html#-no-lock
//'--compress=lz4',
//'--compress-threads=1',
'--target-dir=' . $target
];
if (!empty($base)) {
$args[] = '--incremental-basedir=' . $base;
}
$stdout = '';
$stderr = '';
$cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
self::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
// Console::error($stderr);
//Console::exit();
}
// For some reason they write everything as $stderr
if (!str_contains($stderr, 'completed OK!')) {
/// Todo We need to destroy this directory and all the data inside or move it somewhere
Console::error($stderr);
Console::exit();
}
}
public static function log(string $message): void
{
if (!empty($message)) {
@@ -306,7 +231,6 @@ class Backup extends Action
'_DO_SPACES_ACCESS_KEY',
'_DO_SPACES_SECRET_KEY',
'_DO_SPACES_REGION',
'_APP_DB_ROOT_PASS'
] as $env
) {
if (empty(App::getEnv($env))) {
@@ -315,4 +239,15 @@ class Backup extends Action
}
}
}
public static function getDsn(string $database): ?DSN
{
foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_PROJECT')) as $project) {
list($db, $dsn) = explode('=', $project);
if ($db === $database) {
return new DSN($dsn);
}
}
return null;
}
}
src/Appwrite/Platform/Tasks/Restore.php
+74 -71
View File
@@ -4,6 +4,7 @@ namespace Appwrite\Platform\Tasks;
use Exception;
use Utopia\App;
use Utopia\DSN\DSN;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Storage\Device;
@@ -14,10 +15,8 @@ use Utopia\Validator\WhiteList;
class Restore extends Action
{
// todo: Carefully double check this is not a production value!!!!!!!!!!!!!!!
// todo: it will be erased!!!!
protected string $host = 'mariadb';
protected string $project;
protected ?DSN $dsn = null;
protected string $database;
public const PROCESSORS = 4;
public static function getName(): string
@@ -27,27 +26,32 @@ class Restore extends Action
public function __construct()
{
$this->checkEnvVariables();
$this
->desc('Restore a DB')
->param('id', '', new Text(100), 'Folder Identifier')
->param('id', '', new Text(20), 'Folder Identifier')
->param('cloud', null, new WhiteList(['true', 'false'], true), 'Take file from cloud?')
->param('project', null, new WhiteList(['db_fra1_01', 'db_fra1_02', 'db_fra1_03', 'db_fra1_04', 'db_fra1_05', 'db_fra1_06', 'db_fra1_07'], true), 'From _APP_CONNECTIONS_DB_PROJECT')
->param('database', null, new Text(10), 'example: db_fra1_01..')
->param('datadir', null, new Text(100), 'mysql datadir path')
->callback(fn ($id, $cloud, $project, $datadir) => $this->action($id, $cloud, $project, $datadir));
}
public function action(string $id, string $cloud, string $project, string $datadir): void
public function action(string $id, string $cloud, string $database, string $datadir): void
{
// todo? why do we need to pass it as variable?
$datadir = '/varlibmysql';
$this->database = $database;
$this->dsn = Backup::getDsn($database);
if (!file_exists($datadir)) {
Console::error('Datadir not found: ' . $datadir);
if (!$this->dsn instanceof DSN) {
Console::error('No dsn match');
Console::exit();
}
if (!str_starts_with($datadir, '/')) {
Console::error('datadir must start with /');
// todo? why do we need to pass it as variable?
//$datadir = '/varlibmysql';
if (!file_exists($datadir)) {
Console::error('Datadir not found: ' . $datadir);
Console::exit();
}
@@ -57,41 +61,40 @@ class Restore extends Action
}
$this->checkContainerIsStopped();
$this->checkEnvVariables();
$filename = $id . '.tar.gz';
Backup::log('--- Restore Start ' . $filename . ' --- ');
$start = microtime(true);
$cloud = $cloud === 'true';
$local = new Local(Backup::BACKUPS . '/' . $project . '/full/' . $id);
$files = $local->getRoot() . '/files';
if ($cloud) {
$local = new Local(Backup::BACKUPS . '/downloads/' . $id);
$this->download($project, $filename, $local);
if (file_exists($local->getRoot())) {
$stdout = '';
$stderr = '';
$cmd = 'rm -rf ' . $local->getRoot();
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
Console::exit();
}
}
$files = $local->getRoot() . '/files';
if (!file_exists($files) && !mkdir($files, 0755, true)) {
Backup::log('Creating Directory: ' . $files);
if (!mkdir($files, 0755, true)) {
Console::error('Error creating directory: ' . $files);
Console::exit();
}
$file = $local->getPath($filename);
$stdout = '';
$stderr = '';
$cmd = 'tar -xzf ' . $file . ' -C ' . $files;
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
Console::exit();
}
if (!file_exists($file)) {
Console::error('Restore file not found: ' . $file);
Console::exit();
}
$this->download($filename, $local);
$this->untar($local->getPath($filename), $files);
} else {
$local = new Local(Backup::BACKUPS . '/' . $database . '/full/' . $id);
$files = $local->getRoot() . '/files';
}
$this->decompress($files);
@@ -101,26 +104,9 @@ class Restore extends Action
Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds");
}
public function download(string $project, string $filename, Device $local)
public function download(string $filename, Device $local)
{
if (file_exists($local->getRoot())) {
$stdout = '';
$stderr = '';
$cmd = 'rm -rf ' . $local->getRoot();
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
Console::exit();
}
}
if (!file_exists($local->getRoot()) && !mkdir($local->getRoot(), 0755, true)) {
Console::error('Error creating directory: ' . $local->getRoot());
Console::exit();
}
$s3 = new DOSpaces($project . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
$s3 = new DOSpaces($this->database . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
try {
$path = $s3->getPath($filename);
@@ -143,6 +129,24 @@ class Restore extends Action
}
}
public function untar(string $file, string $directory)
{
$stdout = '';
$stderr = '';
$cmd = 'tar -xzf ' . $file . ' -C ' . $directory;
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
Console::error($stderr);
Console::exit();
}
if (!file_exists($file)) {
Console::error('Restore file not found: ' . $file);
Console::exit();
}
}
public function decompress(string $target)
{
if (!file_exists($target)) {
@@ -150,15 +154,15 @@ class Restore extends Action
Console::exit();
}
$logfile = $target . '/../log.txt';
$logfile = $target . '/../decompress.log';
$args = [
'--user=root',
'--password=' . App::getEnv('_APP_DB_ROOT_PASS'),
'--host=' . $this->host,
'--user=' . $this->dsn->getUser(),
'--password=' . $this->dsn->getPassword(),
'--host=' . $this->dsn->getHost(),
'--decompress',
'--strict',
'--remove-original', // Removes *.lz4
'--remove-original', // Removes *.lz4 compressed files
'--parallel=' . self::PROCESSORS,
'--compress-threads=' . self::PROCESSORS,
'--target-dir=' . $target,
@@ -185,12 +189,12 @@ class Restore extends Action
Console::exit();
}
$logfile = $target . '/../log.txt';
$logfile = $target . '/../prepare.log';
$args = [
'--user=root',
'--password=' . App::getEnv('_APP_DB_ROOT_PASS'),
'--host=' . $this->host,
'--user=' . $this->dsn->getUser(),
'--password=' . $this->dsn->getPassword(),
'--host=' . $this->dsn->getHost(),
'--prepare',
'--strict',
'--target-dir=' . $target,
@@ -217,12 +221,12 @@ class Restore extends Action
Console::exit();
}
$logfile = $target . '/../log.txt';
$logfile = $target . '/../restore.log';
$args = [
'--user=root',
'--password=' . App::getEnv('_APP_DB_ROOT_PASS'),
'--host=' . $this->host,
'--user=' . $this->dsn->getUser(),
'--password=' . $this->dsn->getPassword(),
'--host=' . $this->dsn->getHost(),
$cloud ? '--move-back' : '--copy-back',
'--strict',
'--target-dir=' . $target,
@@ -254,7 +258,6 @@ class Restore extends Action
'_DO_SPACES_ACCESS_KEY',
'_DO_SPACES_SECRET_KEY',
'_DO_SPACES_REGION',
'_APP_DB_ROOT_PASS'
] as $env
) {
if (empty(App::getEnv($env))) {
@@ -266,7 +269,8 @@ class Restore extends Action
public function checkContainerIsStopped(): void
{
$cmd = 'docker ps --filter "status=running" --filter "name=' . $this->host . '"';
$host = $this->dsn->getHost();
$cmd = 'docker ps --filter "status=running" --filter "name=' . $host . '"';
Backup::log($cmd);
$stderr = '';
$stdout = '';
@@ -282,9 +286,8 @@ class Restore extends Action
$info = array_shift($stdout);
if (!empty($info)) {
Console::error('Please stop container: ' . $this->host);
Console::error('Please stop container: ' . $host);
//Console::exit();
}
}
}