From bf1db0dced217b3f5fa9a95bab91d233bbc6dbb2 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 25 Oct 2023 01:10:00 +0400 Subject: [PATCH] chore: refactor code --- .env | 4 +- Dockerfile | 5 +- app/init.php | 6 -- bin/{db-backup => database-backup} | 0 bin/{db-restore => database-restore} | 0 docker-compose.yml | 6 +- src/Appwrite/Platform/Tasks/Backup.php | 96 ++++++++++++++++++-------- 7 files changed, 74 insertions(+), 43 deletions(-) rename bin/{db-backup => database-backup} (100%) rename bin/{db-restore => database-restore} (100%) diff --git a/.env b/.env index a66b5b5190..feeb74d849 100644 --- a/.env +++ b/.env @@ -98,6 +98,4 @@ _APP_VCS_GITHUB_CLIENT_SECRET= _APP_VCS_GITHUB_WEBHOOK_SECRET= _APP_MIGRATIONS_FIREBASE_CLIENT_ID= _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET= -_APP_ASSISTANT_OPENAI_API_KEY= -_APP_CONNECTIONS_DB_REPLICAS=db_fra1_03=mariadb://root:rootsecretpassword@mariadb:3306/appwrite -_APP_CONNECTIONS_BACKUPS_STORAGE=dospaces://ACCESS:SECRET@default/backups-v1?region=fra1 \ No newline at end of file +_APP_ASSISTANT_OPENAI_API_KEY= \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2024c83e68..edb8d1a5ce 100755 --- a/Dockerfile +++ b/Dockerfile @@ -74,9 +74,8 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/maintenance && \ - chmod +x /usr/local/bin/db-backup && \ - chmod +x /usr/local/bin/db-restore && \ - chmod +x /usr/local/bin/volume-sync && \ + chmod +x /usr/local/bin/database-backup && \ + chmod +x /usr/local/bin/database-restore && \ chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/upgrade && \ diff --git a/app/init.php b/app/init.php index 3b70533cd0..86e4790a30 100644 --- a/app/init.php +++ b/app/init.php @@ -614,12 +614,6 @@ $register->set('pools', function () { 'multiple' => true, 'schemes' => ['mariadb', 'mysql'], ], - 'replica' => [ - 'type' => 'database', - 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_REPLICAS', $fallbackForDB), - 'multiple' => true, - 'schemes' => ['mariadb', 'mysql'], - ], 'queue' => [ 'type' => 'queue', 'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), diff --git a/bin/db-backup b/bin/database-backup similarity index 100% rename from bin/db-backup rename to bin/database-backup diff --git a/bin/db-restore b/bin/database-restore similarity index 100% rename from bin/db-restore rename to bin/database-restore diff --git a/docker-compose.yml b/docker-compose.yml index ee5aa6681c..f93e357e7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -834,8 +834,8 @@ services: appwrite-backup: image: appwrite-dev <<: *x-logging - entrypoint: db-backup container_name: appwrite-backup + command: database-backup --database=db_fra1_03 networks: - appwrite volumes: @@ -846,6 +846,8 @@ services: depends_on: - redis environment: + - _APP_BACKUPS_INTERVAL=120 + - _APP_BACKUPS_START_TIME=21:08 - _APP_CONNECTIONS_DB_REPLICAS - _APP_CONNECTIONS_BACKUPS_STORAGE - _APP_LOGGING_PROVIDER @@ -853,7 +855,7 @@ services: appwrite-restore: command: sleep infinity - container_name: appwrite-backup-restore + container_name: appwrite-restore image: appwrite-dev networks: - appwrite diff --git a/src/Appwrite/Platform/Tasks/Backup.php b/src/Appwrite/Platform/Tasks/Backup.php index 32bdbd7c9d..ebee91fac6 100644 --- a/src/Appwrite/Platform/Tasks/Backup.php +++ b/src/Appwrite/Platform/Tasks/Backup.php @@ -3,11 +3,16 @@ namespace Appwrite\Platform\Tasks; use Exception; +use PDO; use Utopia\DSN\DSN; use Utopia\Platform\Action; use Utopia\App; +use Utopia\Cache\Adapter\None; +use Utopia\Cache\Cache; use Utopia\CLI\Console; -use Utopia\Pools\Group; +use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Adapter\MySQL; +use Utopia\Database\Database; use Utopia\Storage\Device; use Utopia\Storage\Device\DOSpaces; use Utopia\Storage\Device\Local; @@ -25,6 +30,7 @@ class Backup extends Action public const RETRY_TAR = 1; public const RETRY_UPLOAD = 2; public const VERSION = 'v1'; + protected ?DSN $dsn = null; protected ?string $database = null; protected ?DOSpaces $s3 = null; @@ -39,49 +45,78 @@ class Backup extends Action $this ->desc('Backup a database') ->param('database', null, new Text(20), 'Database name, for example: db_fra1_03') - ->inject('pools') ->inject('logError') - ->callback(fn(string $database, Group $pools, callable $logError) => $this->action($database, $pools, $logError)); - } - - public static function getName(): string - { - return 'backup'; + ->callback(fn (string $database, callable $logError) => $this->action($database, $logError)); } /** * @throws Exception */ - public function action(string $database, Group $pools, callable $logError): void + public function action(string $database, callable $logError): void { - $this->checkEnvVariables(); + Console::title('Backups V1'); + Console::success(APP_NAME . ' Database Backup Process has started..'); $this->database = $database; + + $this->checkEnvVariables(); $this->dsn = $this->getDsn($database); if (is_null($this->dsn)) { - throw new Exception('No DSN match'); + throw new Exception('No DSN found for database ' . $database . '. Check the value of _APP_CONNECTIONS_DB_REPLICAS'); } - - console::info('Trying to connect to ' . $this->dsn->getHost()); - - $dsn = new DSN(App::getEnv('_APP_CONNECTIONS_BACKUPS_STORAGE', '')); - $this->s3 = new DOSpaces('/' . $database . '/' . self::VERSION, $dsn->getUser(), $dsn->getPassword(), $dsn->getPath(), $dsn->getParam('region')); - + Console::info('Trying to connect to ' . $this->dsn->getHost()); try { - $this->retry(function () use ($pools, $database) { - $pools - ->get('replica_' . $database) - ->pop() - ->getResource(); + $this->retry(function () { + $dsnHost = $this->dsn->getHost(); + $dsnPort = $this->dsn->getPort(); + $dsnUser = $this->dsn->getUser(); + $dsnPass = $this->dsn->getPassword(); + $dsnScheme = $this->dsn->getScheme(); + $dsnDatabase = $this->dsn->getPath(); + $pdo = new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass); + + $adapter = match ($dsnScheme) { + 'mysql' => new MySQL($pdo), + 'mariadb' => new MariaDB($pdo) + }; + $database = new Database($adapter, new Cache(new None())); + $database->ping(); + Console::success('Connected to ' . $dsnHost); }, 10, 5); } catch (Exception $error) { throw new Exception('Failed to connect to database: ' . $error->getMessage()); } + $storageDSN = new DSN(App::getEnv('_APP_CONNECTIONS_BACKUPS_STORAGE', '')); + $this->s3 = new DOSpaces('/' . $database . '/' . self::VERSION, $storageDSN->getUser(), $storageDSN->getPassword(), $storageDSN->getPath(), $storageDSN->getParam('region')); + $this->setContainerId(); $this->setProcessors(); + $sleep = (int) App::getEnv('_APP_BACKUPS_INTERVAL', 0); // 120 seconds (by default) + $jobInitTime = App::getEnv('_APP_BACKUPS_START_TIME'); // (hour:minutes) + + if (!$sleep || !$jobInitTime) { + throw new Exception('Invalid backup interval or start time'); + } + + $now = new \DateTime(); + $now->setTimezone(new \DateTimeZone(date_default_timezone_get())); + $next = new \DateTime($now->format("Y-m-d $jobInitTime")); + $next->setTimezone(new \DateTimeZone(date_default_timezone_get())); + $delay = $next->getTimestamp() - $now->getTimestamp(); + + /** + * If time passed for the target day. + */ + if ($delay <= 0) { + $next->add(\DateInterval::createFromDateString('1 days')); + $delay = $next->getTimestamp() - $now->getTimestamp(); + } + + self::log('Setting loop start time to ' . $next->format("Y-m-d H:i:s.v") . '. Delaying for ' . $delay . ' seconds.'); + Console::loop(function () use ($logError) { try { $this->start(); @@ -93,7 +128,7 @@ class Backup extends Action $logError($error, 'backup', 'action'); } - }, self::BACKUP_INTERVAL_SECONDS); + }, $sleep, $delay); } /** @@ -248,12 +283,10 @@ class Backup extends Action */ public function checkEnvVariables(): void { - foreach ( - [ - '_APP_CONNECTIONS_BACKUPS_STORAGE', - '_APP_CONNECTIONS_DB_REPLICAS', - ] as $env - ) { + foreach ([ + '_APP_CONNECTIONS_BACKUPS_STORAGE', + '_APP_CONNECTIONS_DB_REPLICAS', + ] as $env) { if (empty(App::getEnv($env))) { throw new Exception('Can\'t read ' . $env); } @@ -413,4 +446,9 @@ class Backup extends Action return false; } + + public static function getName(): string + { + return 'backup'; + } }