Restore.php

.env

@@ -85,4 +85,5 @@ _DO_SPACES_REGION=fra1
 _DO_SPACES_ACCESS_KEY=
 _DO_SPACES_SECRET_KEY=
 _DO_SPACES_ACCESS_KEY=
-_DO_SPACES_SECRET_KEY=
+_DO_SPACES_SECRET_KEY=
+_APP_BACKUP_FOLDER=daily

docker-compose.yml

@@ -634,7 +634,6 @@ services:
       - ./src:/usr/src/code/src
       - appwrite-mariadb:/var/lib/mysql:rw
       - ./backups:/backups:rw
-      - ./extracts:/extracts:rw
       - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
       - redis
@@ -644,6 +643,7 @@ services:
       - _DO_SPACES_ACCESS_KEY
       - _DO_SPACES_SECRET_KEY
       - _DO_SPACES_REGION
+      - _APP_BACKUP_FOLDER
 
   appwrite-usage:
     entrypoint: usage

src/Appwrite/Platform/Tasks/Backup.php

@@ -7,11 +7,10 @@ use Utopia\App;
 use Utopia\CLI\Console;
 use Utopia\Storage\Device\DOSpaces;
 use Utopia\Storage\Storage;
-use Utopia\Validator\Hostname;
 
 class Backup extends Action
 {
-    protected string $directory = '/var/lib/mysql';
+    public static string $mysqlDirectory = '/var/lib/mysql';
     public static string $backups = '/backups'; // Mounted volume
     protected string $containerName = 'appwrite-mariadb';
 
@@ -22,53 +21,30 @@ class Backup extends Action
 
     public function __construct()
     {
-        //docker compose exec appwrite backup
         $this
             ->desc('Backup a DB')
-            ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain.', true)
-            ->callback(fn($domain) => $this->action($domain));
+            ->callback(fn() => $this->action());
     }
 
     /**
      * @throws \Exception
      */
-    public function action(string $domain): void
+    public function action(): void
     {
+        $this->checkEnvVariables();
+
+        $folder = App::getEnv('_APP_BACKUP_FOLDER');
+
         $start = microtime(true);
         $time = date('Y-m-d_H:i:s');
-        $this->log('--- Backup Start --- ');
-
-        if (empty(App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))) {
-            Console::error('Can\'t read _APP_CONNECTIONS_DB_PROJECT');
-            Console::exit();
-        }
-
-        if (empty(App::getEnv('_DO_SPACES_BUCKET_NAME'))) {
-            Console::error('Can\'t read _DO_SPACES_BUCKET_NAME');
-            Console::exit();
-        }
-
-        if (empty(App::getEnv('_DO_SPACES_ACCESS_KEY'))) {
-            Console::error('Can\'t read _DO_SPACES_ACCESS_KEY');
-            Console::exit();
-        }
-
-        if (empty(App::getEnv('_DO_SPACES_SECRET_KEY'))) {
-            Console::error('Can\'t read _DO_SPACES_SECRET_KEY');
-            Console::exit();
-        }
-
-        if (empty(App::getEnv('_DO_SPACES_REGION'))) {
-            Console::error('Can\'t read _DO_SPACES_REGION');
-            Console::exit();
-        }
+        self::log('--- Backup Start --- ');
 
         Storage::setDevice('files', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')));
         $device = Storage::getDevice('files');
 
         // Todo: do we want to have the backup ready on the mounted dir? or to abort?
         if (!$device->exists('/')) {
-            Console::error('Can\'t read from DO aborting');
+            Console::error('Can\'t read from DO ');
             Console::exit();
         }
 
@@ -79,7 +55,7 @@ class Backup extends Action
             Console::exit();
         }
 
-        $backups = self::$backups . '/' . $dsn;
+        $backups = self::$backups . '/' . $dsn . '/' . $folder;
 
         if (!file_exists($backups) && !mkdir($backups, 0755, true)) {
             Console::error('Error creating directory: ' . $backups);
@@ -89,10 +65,10 @@ class Backup extends Action
         $stdout = '';
         $stderr = '';
         $cmd = 'docker stop ' . $this->containerName;
-        $this->log($cmd);
+        self::log($cmd);
         Console::execute($cmd, '', $stdout, $stderr);
-        $this->log($stdout);
         if (!empty($stderr)) {
+            self::log($stdout);
             Console::error($stderr);
             Console::exit();
         }
@@ -110,10 +86,10 @@ class Backup extends Action
 
         $stdout = '';
         $stderr = '';
-        $cmd = 'cd ' . $this->directory . ' && tar zcf ' . $file . ' .';
-        $this->log($cmd);
+        $cmd = 'cd ' . self::$mysqlDirectory . ' && tar zcf ' . $file . ' .';
+        self::log($cmd);
         Console::execute($cmd, '', $stdout, $stderr);
-        $this->log($stdout);
+        self::log($stdout);
         if (!empty($stderr)) {
             Console::error($stderr);
             Console::exit();
@@ -122,50 +98,57 @@ class Backup extends Action
         $stdout = '';
         $stderr = '';
         $cmd = 'docker start ' . $this->containerName;
-        $this->log($cmd);
+        self::log($cmd);
         Console::execute($cmd, '', $stdout, $stderr);
-        $this->log($stdout);
         if (!empty($stderr)) {
+            self::log($stdout);
             Console::error($stderr);
             Console::exit();
         }
 
         try {
-            $folder = App::getEnv('_APP_BACKUP_FOLDER', 'hourly');
-
             $path = '/' . $dsn . '/' . $folder . '/' . $filename;
-            $this->log('Uploading ' . $path);
+            self::log('Uploading ' . $path);
             if (!$device->upload($file, $path)) {
                 Console::error('Error uploading to ' . $path);
                 Console::exit();
             }
-
-//            $isDaily = true;
-//
-//            if ($isDaily) {
-//                $dailyPath = '/' . $dsn . '/daily/' . $filename;
-//                $this->log('Moving ' . $dailyPath);
-//                if (!$device->move($hourlyPath, $dailyPath)) {
-//                    Console::error('Error moving to hourly to ' . $dailyPath);
-//                    Console::exit();
-//                }
-//            }
         } catch (\Exception $e) {
             Console::error($e->getMessage());
             Console::exit();
         }
 
-        $this->log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- '   . PHP_EOL . PHP_EOL);
+        self::log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- '   . PHP_EOL . PHP_EOL);
 
         Console::loop(function () {
-            Console::log('Hello');
+            self::log('loop');
         }, 100);
     }
 
-    public function log(string $message): void
+    public static function log(string $message): void
     {
         if (!empty($message)) {
             Console::log(date('Y-m-d H:i:s') . ' ' . $message);
         }
     }
+
+    public function checkEnvVariables(): void
+    {
+        foreach (
+            [
+                '_APP_BACKUP_FOLDER',
+                '_APP_CONNECTIONS_DB_PROJECT',
+                '_DO_SPACES_BUCKET_NAME',
+                '_DO_SPACES_ACCESS_KEY',
+                '_DO_SPACES_SECRET_KEY',
+                '_DO_SPACES_REGION'
+            ] as $env
+        ) {
+            if (empty(App::getEnv($env))) {
+                Console::error('Can\'t read ' . $env);
+                Console::exit();
+            }
+        }
+    }
+    
 }

src/Appwrite/Platform/Tasks/Restore.php

@@ -8,15 +8,15 @@ use Utopia\CLI\Console;
 use Utopia\Storage\Device\DOSpaces;
 use Utopia\Storage\Device\Local;
 use Utopia\Storage\Storage;
-use Utopia\Validator\Boolean;
 use Utopia\Validator\Text;
 use Utopia\Validator\WhiteList;
 
-//  docker compose exec appwrite-backup db-restore --file=2023-07-17_11:36:12.tar.gz
+//  docker compose exec appwrite-backup db-restore --cloud=false --filename=2023-07-18_12:39:59.tar.gz
 
 class Restore extends Action
 {
-    protected string $extract = '/extracts';
+    // todo: double check this is not a production value!!!!!!!!!!!!!!!
+    // todo: it will be erased!!!!
     protected string $containerName = 'appwrite-mariadb';
 
     public static function getName(): string
@@ -28,51 +28,70 @@ class Restore extends Action
     {
         $this
             ->desc('Restore a DB')
-            ->param('file', '', new Text(100), 'Backup file name')
-            ->param('cloud', 'false', new Boolean(true), 'Take file from cloud?')
-            ->callback(fn ($file, $cloud) => $this->action($file, $cloud));
+            ->param('filename', '', new Text(100), 'Backup file name')
+            ->param('cloud', null, new WhiteList(['true', 'false'], true), 'Take file from cloud?')
+            ->param('project', null, new WhiteList(['db_fra1_02'], true), 'From _APP_CONNECTIONS_DB_PROJECT')
+            ->param('folder', null, new WhiteList(['hourly', 'daily'], true), 'Sub folder')
+            ->callback(fn ($file, $cloud, $project, $folder) => $this->action($file, $cloud, $project, $folder));
     }
 
     /**
      * @throws \Exception
      */
-    public function action(string $file, bool $cloud): void
+    public function action(string $filename, string $cloud, string $project, string $folder): void
     {
+        $this->checkEnvVariables();
 
-        var_dump($cloud);
+        Backup::log('--- Restore Start ' . $filename . ' --- ');
+        $start = microtime(true);
+
+        $cloud = $cloud === 'true';
+        $file = Backup::$backups . '/' . $project . '/' . $folder . '/' . $filename;
+        $extract = Backup::$backups . '/extract';
+        $original = $extract . '/original-' . time();
+
+        Backup::log('Creating directory ' . $original);
+        if (!file_exists($original) && !mkdir($original, 0755, true)) {
+            Console::error('Error creating directory: ' . $original);
+            Console::exit();
+        }
+
+//        $tarDirectory = $extract . '/' . str_replace(['.', '-', '_', ':', 'tar', 'gz'], '', $filename);
+//        Backup::log('Creating directory ' . $tarDirectory);
+//        if (!file_exists($tarDirectory) && !mkdir($tarDirectory, 0755, true)) {
+//            Console::error('Error creating directory: ' . $tarDirectory);
+//            Console::exit();
+//        }
 
         if ($cloud) {
-            Storage::setDevice('files', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')));
+            Storage::setDevice('s3', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')));
             Storage::setDevice('mount', new Local(Backup::$backups));
 
-            $device = Storage::getDevice('files');
+            $device = Storage::getDevice('s3');
             $mount = Storage::getDevice('mount');
 
             try {
                 $folder = App::getEnv('_APP_BACKUP_FOLDER', 'hourly');
-                $dsn = explode('=', App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))[0];
-                $path = '/' . $dsn . '/' . $folder . '/' . $file;
+                $path = '/' . $project . '/' . $folder . '/' . $filename;
 
-                $this->log('Downloading from Space ' . $path);
+                if (!$device->exists($path)) {
+                    Console::error('File: ' . $path . ' does not exist on cloud');
+                    Console::exit();
+                }
 
-                $x = $device->read($path);
-
-                $mount->write('/bla-' . $file, $device->read($path));
+                while ($data = $device->read($path)) {
+                    $mount->write('file-' . $file, $data);
+                }
 
+                Backup::log('Downloading from s3 ' . $path);
+                $mount->write('file-' . $file, $device->read($path));
+               // $mount->get('file-' . $file, $device->read($path));
             } catch (\Exception $e) {
                 Console::error($e->getMessage());
                 Console::exit();
             }
         }
 
-
-        exit;
-
-        $file = Backup::$backups . '/' . $file;
-        $start = microtime(true);
-
-        $this->log('--- Restore Start ' . $file . ' --- ');
-
         if (!file_exists($file) || empty($file)) {
             Console::error('Restore file not found: ' . $file);
             Console::exit();
@@ -81,50 +100,79 @@ class Restore extends Action
         $stdout = '';
         $stderr = '';
         $cmd = 'docker stop ' . $this->containerName;
-        $this->log($cmd);
+        Backup::log($cmd);
         Console::execute($cmd, '', $stdout, $stderr);
-        $this->log($stdout);
         if (!empty($stderr)) {
+            Backup::log($stdout);
             Console::error($stderr);
             Console::exit();
         }
 
-        $extract = $this->extract . '/' . time();
-        $this->log('Creating extract directory ' . $extract);
-        if (!file_exists($extract) && !mkdir($extract, 0755, true)) {
-            Console::error('Error creating directory: ' . $extract);
-            Console::exit();
-        }
 
         $stdout = '';
         $stderr = '';
-        $cmd = 'tar -xzf ' . $file . ' -C ' . $this->extract;
-        $this->log($cmd);
+        $cmd = 'mv ' . Backup::$mysqlDirectory . '/* ' . ' ' . $original . '/';
+        Backup::log($cmd);
         Console::execute($cmd, '', $stdout, $stderr);
-        $this->log($stdout);
+        Backup::log($stdout);
         if (!empty($stderr)) {
             Console::error($stderr);
             Console::exit();
         }
 
+
+        $stdout = '';
+        $stderr = '';
+       // $cmd = 'tar -xzf ' . $file . ' -C ' . $tarDirectory;
+        $cmd = 'tar -xzf ' . $file . ' -C ' . Backup::$mysqlDirectory;
+
+        Backup::log($cmd);
+        Console::execute($cmd, '', $stdout, $stderr);
+        if (!empty($stderr)) {
+            Backup::log($stdout);
+            Console::error($stderr);
+            Console::exit();
+        }
+
+//        $stdout = '';
+//        $stderr = '';
+//        $cmd = 'mv ' . $tarDirectory . ' ' . Backup::$mysqlDirectory;
+//        Backup::log($cmd);
+//        Console::execute($cmd, '', $stdout, $stderr);
+//        Backup::log($stdout);
+//        if (!empty($stderr)) {
+//            Console::error($stderr);
+//            Console::exit();
+//        }
+
         $stdout = '';
         $stderr = '';
         $cmd = 'docker start ' . $this->containerName;
-        $this->log($cmd);
+        Backup::log($cmd);
         Console::execute($cmd, '', $stdout, $stderr);
-        $this->log($stdout);
+        Backup::log($stdout);
         if (!empty($stderr)) {
             Console::error($stderr);
             Console::exit();
         }
 
-        $this->log("Restore Finish in " . (microtime(true) - $start) . " seconds");
+        Backup::log("Restore Finish in " . (microtime(true) - $start) . " seconds");
     }
 
-    public function log(string $message): void
+    public function checkEnvVariables(): void
     {
-        if (!empty($message)) {
-            Console::log(date('Y-m-d H:i:s') . ' ' . $message);
+        foreach (
+            [
+                '_DO_SPACES_BUCKET_NAME',
+                '_DO_SPACES_ACCESS_KEY',
+                '_DO_SPACES_SECRET_KEY',
+                '_DO_SPACES_REGION'
+            ] as $env
+        ) {
+            if (empty(App::getEnv($env))) {
+                Console::error('Can\'t read ' . $env);
+                Console::exit();
+            }
         }
     }
 }