Merge remote-tracking branch 'origin/db-pools-backups' into db-pools-backups

Co-authored-by: Jake Barnby <jakeb994@gmail.com>

.env

@@ -27,6 +27,7 @@ _APP_DB_PASS=password
 _APP_DB_ROOT_PASS=rootsecretpassword
 _APP_CONNECTIONS_MAX=3100
 _APP_POOL_CLIENTS=14
+_APP_CONNECTIONS_DB_REPLICAS=db_fra1_02=mariadb://root:rootsecretpassword@mariadb:3306/appwrite
 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
 _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
 _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
@@ -79,4 +80,8 @@ _APP_REGION=default
 _APP_DOCKER_HUB_USERNAME=
 _APP_DOCKER_HUB_PASSWORD=
 _APP_CONSOLE_GITHUB_SECRET=
-_APP_CONSOLE_GITHUB_APP_ID=
+_APP_CONSOLE_GITHUB_APP_ID=
+_DO_SPACES_BUCKET_NAME=backups-v1
+_DO_SPACES_REGION=fra1
+_DO_SPACES_ACCESS_KEY=
+_DO_SPACES_SECRET_KEY=

.github/workflows/publish.yml

@@ -16,7 +16,7 @@ jobs:
       with:
         fetch-depth: 2
         submodules: recursive
-        ref: feat-db-pools-master
+        ref: db-pools-backups
 
     - name: Login to Docker Hub
       uses: docker/login-action@v2

.gitignore

@@ -11,3 +11,4 @@ debug/
 app/sdks
 dev/yasd_init.php
 .phpunit.result.cache
+/backup-data/

Dockerfile

@@ -123,6 +123,8 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/calc-tier-stats && \
     chmod +x /usr/local/bin/patch-delete-project-collections && \
     chmod +x /usr/local/bin/maintenance &&  \
+    chmod +x /usr/local/bin/db-backup &&  \
+    chmod +x /usr/local/bin/db-restore &&  \
     chmod +x /usr/local/bin/volume-sync && \
     chmod +x /usr/local/bin/usage && \
     chmod +x /usr/local/bin/install && \
@@ -159,6 +161,12 @@ RUN echo "default_socket_timeout=-1" >> /usr/local/etc/php/conf.d/appwrite.ini
 RUN echo "opcache.jit_buffer_size=100M" >> /usr/local/etc/php/conf.d/appwrite.ini
 RUN echo "opcache.jit=1235" >> /usr/local/etc/php/conf.d/appwrite.ini
 
+RUN \
+  apk update \
+  && apk add --no-cache \
+  docker \
+  && rm -rf /var/cache/apk/*
+
 EXPOSE 80
 
 CMD [ "php", "app/http.php", "-dopcache.preload=opcache.preload=/usr/src/code/app/preload.php" ]

app/init.php

@@ -532,6 +532,12 @@ $register->set('pools', function () {
             'multiple' => true,
             'schemes' => ['mariadb', 'mysql'],
         ],
+        'replica' => [
+            'type' => 'database',
+            'dsns' => App::getEnv('_APP_CONNECTIONS_DB_REPLICAS', $fallbackForDB),
+            'multiple' => true,
+            'schemes' => ['mysql'],
+        ],
         'queue' => [
             'type' => 'queue',
             'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis),

bin/db-backup

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php backup $@

bin/db-restore

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php restore $@

composer.json

@@ -63,7 +63,7 @@
         "utopia-php/preloader": "0.2.*",
         "utopia-php/queue": "0.5.*",
         "utopia-php/registry": "0.5.*",
-        "utopia-php/storage": "0.13.*",
+        "utopia-php/storage": "dev-feat-transfer as 0.13.3",
         "utopia-php/swoole": "0.5.*",
         "utopia-php/websocket": "0.1.0",
         "resque/php-resque": "1.3.6",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "329498d12fabaa764c4178881cb07788",
+    "content-hash": "ff72579b1906a3ee181ec1209198c13f",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -534,16 +534,16 @@
         },
         {
             "name": "guzzlehttp/promises",
-            "version": "2.0.0",
+            "version": "2.0.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/guzzle/promises.git",
-                "reference": "3a494dc7dc1d7d12e511890177ae2d0e6c107da6"
+                "reference": "111166291a0f8130081195ac4556a5587d7f1b5d"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/guzzle/promises/zipball/3a494dc7dc1d7d12e511890177ae2d0e6c107da6",
-                "reference": "3a494dc7dc1d7d12e511890177ae2d0e6c107da6",
+                "url": "https://api.github.com/repos/guzzle/promises/zipball/111166291a0f8130081195ac4556a5587d7f1b5d",
+                "reference": "111166291a0f8130081195ac4556a5587d7f1b5d",
                 "shasum": ""
             },
             "require": {
@@ -597,7 +597,7 @@
             ],
             "support": {
                 "issues": "https://github.com/guzzle/promises/issues",
-                "source": "https://github.com/guzzle/promises/tree/2.0.0"
+                "source": "https://github.com/guzzle/promises/tree/2.0.1"
             },
             "funding": [
                 {
@@ -613,20 +613,20 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-05-21T13:50:22+00:00"
+            "time": "2023-08-03T15:11:55+00:00"
         },
         {
             "name": "guzzlehttp/psr7",
-            "version": "2.5.0",
+            "version": "2.6.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/guzzle/psr7.git",
-                "reference": "b635f279edd83fc275f822a1188157ffea568ff6"
+                "reference": "8bd7c33a0734ae1c5d074360512beb716bef3f77"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/guzzle/psr7/zipball/b635f279edd83fc275f822a1188157ffea568ff6",
-                "reference": "b635f279edd83fc275f822a1188157ffea568ff6",
+                "url": "https://api.github.com/repos/guzzle/psr7/zipball/8bd7c33a0734ae1c5d074360512beb716bef3f77",
+                "reference": "8bd7c33a0734ae1c5d074360512beb716bef3f77",
                 "shasum": ""
             },
             "require": {
@@ -713,7 +713,7 @@
             ],
             "support": {
                 "issues": "https://github.com/guzzle/psr7/issues",
-                "source": "https://github.com/guzzle/psr7/tree/2.5.0"
+                "source": "https://github.com/guzzle/psr7/tree/2.6.0"
             },
             "funding": [
                 {
@@ -729,7 +729,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-04-17T16:11:26+00:00"
+            "time": "2023-08-03T15:06:02+00:00"
         },
         {
             "name": "influxdb/influxdb-php",
@@ -2622,26 +2622,32 @@
         },
         {
             "name": "utopia-php/storage",
-            "version": "0.13.2",
+            "version": "dev-feat-transfer",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/storage.git",
-                "reference": "ad1c00f24ca56e73888acc2af3deee4919b1194b"
+                "reference": "cef63e2d76156c2069c98ccd1430ebac5df3a35c"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/storage/zipball/ad1c00f24ca56e73888acc2af3deee4919b1194b",
-                "reference": "ad1c00f24ca56e73888acc2af3deee4919b1194b",
+                "url": "https://api.github.com/repos/utopia-php/storage/zipball/cef63e2d76156c2069c98ccd1430ebac5df3a35c",
+                "reference": "cef63e2d76156c2069c98ccd1430ebac5df3a35c",
                 "shasum": ""
             },
             "require": {
+                "ext-brotli": "*",
                 "ext-fileinfo": "*",
+                "ext-lz4": "*",
+                "ext-snappy": "*",
+                "ext-xz": "*",
                 "ext-zlib": "*",
                 "ext-zstd": "*",
                 "php": ">=8.0",
-                "utopia-php/framework": "0.*.*"
+                "utopia-php/framework": "0.*.*",
+                "utopia-php/system": "0.*.*"
             },
             "require-dev": {
+                "laravel/pint": "1.2.*",
                 "phpunit/phpunit": "^9.3",
                 "vimeo/psalm": "4.0.1"
             },
@@ -2655,12 +2661,6 @@
             "license": [
                 "MIT"
             ],
-            "authors": [
-                {
-                    "name": "Eldad Fux",
-                    "email": "eldad@appwrite.io"
-                }
-            ],
             "description": "A simple Storage library to manage application storage",
             "keywords": [
                 "framework",
@@ -2671,9 +2671,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/storage/issues",
-                "source": "https://github.com/utopia-php/storage/tree/0.13.2"
+                "source": "https://github.com/utopia-php/storage/tree/feat-transfer"
             },
-            "time": "2022-12-20T11:11:35+00:00"
+            "time": "2023-08-08T08:09:59+00:00"
         },
         {
             "name": "utopia-php/swoole",
@@ -3253,16 +3253,16 @@
         },
         {
             "name": "nikic/php-parser",
-            "version": "v4.15.5",
+            "version": "v4.16.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/nikic/PHP-Parser.git",
-                "reference": "11e2663a5bc9db5d714eedb4277ee300403b4a9e"
+                "reference": "19526a33fb561ef417e822e85f08a00db4059c17"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/11e2663a5bc9db5d714eedb4277ee300403b4a9e",
-                "reference": "11e2663a5bc9db5d714eedb4277ee300403b4a9e",
+                "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/19526a33fb561ef417e822e85f08a00db4059c17",
+                "reference": "19526a33fb561ef417e822e85f08a00db4059c17",
                 "shasum": ""
             },
             "require": {
@@ -3303,9 +3303,9 @@
             ],
             "support": {
                 "issues": "https://github.com/nikic/PHP-Parser/issues",
-                "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.5"
+                "source": "https://github.com/nikic/PHP-Parser/tree/v4.16.0"
             },
-            "time": "2023-05-19T20:20:00+00:00"
+            "time": "2023-06-25T14:52:30+00:00"
         },
         {
             "name": "phar-io/manifest",
@@ -3656,16 +3656,16 @@
         },
         {
             "name": "phpstan/phpdoc-parser",
-            "version": "1.22.0",
+            "version": "1.23.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/phpstan/phpdoc-parser.git",
-                "reference": "ec58baf7b3c7f1c81b3b00617c953249fb8cf30c"
+                "reference": "846ae76eef31c6d7790fac9bc399ecee45160b26"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/ec58baf7b3c7f1c81b3b00617c953249fb8cf30c",
-                "reference": "ec58baf7b3c7f1c81b3b00617c953249fb8cf30c",
+                "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/846ae76eef31c6d7790fac9bc399ecee45160b26",
+                "reference": "846ae76eef31c6d7790fac9bc399ecee45160b26",
                 "shasum": ""
             },
             "require": {
@@ -3697,22 +3697,22 @@
             "description": "PHPDoc parser with support for nullable, intersection and generic types",
             "support": {
                 "issues": "https://github.com/phpstan/phpdoc-parser/issues",
-                "source": "https://github.com/phpstan/phpdoc-parser/tree/1.22.0"
+                "source": "https://github.com/phpstan/phpdoc-parser/tree/1.23.1"
             },
-            "time": "2023-06-01T12:35:21+00:00"
+            "time": "2023-08-03T16:32:59+00:00"
         },
         {
             "name": "phpunit/php-code-coverage",
-            "version": "9.2.26",
+            "version": "9.2.27",
             "source": {
                 "type": "git",
                 "url": "https://github.com/sebastianbergmann/php-code-coverage.git",
-                "reference": "443bc6912c9bd5b409254a40f4b0f4ced7c80ea1"
+                "reference": "b0a88255cb70d52653d80c890bd7f38740ea50d1"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/443bc6912c9bd5b409254a40f4b0f4ced7c80ea1",
-                "reference": "443bc6912c9bd5b409254a40f4b0f4ced7c80ea1",
+                "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/b0a88255cb70d52653d80c890bd7f38740ea50d1",
+                "reference": "b0a88255cb70d52653d80c890bd7f38740ea50d1",
                 "shasum": ""
             },
             "require": {
@@ -3768,7 +3768,8 @@
             ],
             "support": {
                 "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues",
-                "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.26"
+                "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy",
+                "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.27"
             },
             "funding": [
                 {
@@ -3776,7 +3777,7 @@
                     "type": "github"
                 }
             ],
-            "time": "2023-03-06T12:58:08+00:00"
+            "time": "2023-07-26T13:44:30+00:00"
         },
         {
             "name": "phpunit/php-file-iterator",
@@ -4628,16 +4629,16 @@
         },
         {
             "name": "sebastian/global-state",
-            "version": "5.0.5",
+            "version": "5.0.6",
             "source": {
                 "type": "git",
                 "url": "https://github.com/sebastianbergmann/global-state.git",
-                "reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2"
+                "reference": "bde739e7565280bda77be70044ac1047bc007e34"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/0ca8db5a5fc9c8646244e629625ac486fa286bf2",
-                "reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2",
+                "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/bde739e7565280bda77be70044ac1047bc007e34",
+                "reference": "bde739e7565280bda77be70044ac1047bc007e34",
                 "shasum": ""
             },
             "require": {
@@ -4680,7 +4681,7 @@
             ],
             "support": {
                 "issues": "https://github.com/sebastianbergmann/global-state/issues",
-                "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.5"
+                "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.6"
             },
             "funding": [
                 {
@@ -4688,7 +4689,7 @@
                     "type": "github"
                 }
             ],
-            "time": "2022-02-14T08:28:10+00:00"
+            "time": "2023-08-02T09:26:13+00:00"
         },
         {
             "name": "sebastian/lines-of-code",
@@ -5451,16 +5452,16 @@
         },
         {
             "name": "twig/twig",
-            "version": "v3.6.1",
+            "version": "v3.7.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/twigphp/Twig.git",
-                "reference": "7e7d5839d4bec168dfeef0ac66d5c5a2edbabffd"
+                "reference": "5cf942bbab3df42afa918caeba947f1b690af64b"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/twigphp/Twig/zipball/7e7d5839d4bec168dfeef0ac66d5c5a2edbabffd",
-                "reference": "7e7d5839d4bec168dfeef0ac66d5c5a2edbabffd",
+                "url": "https://api.github.com/repos/twigphp/Twig/zipball/5cf942bbab3df42afa918caeba947f1b690af64b",
+                "reference": "5cf942bbab3df42afa918caeba947f1b690af64b",
                 "shasum": ""
             },
             "require": {
@@ -5506,7 +5507,7 @@
             ],
             "support": {
                 "issues": "https://github.com/twigphp/Twig/issues",
-                "source": "https://github.com/twigphp/Twig/tree/v3.6.1"
+                "source": "https://github.com/twigphp/Twig/tree/v3.7.0"
             },
             "funding": [
                 {
@@ -5518,12 +5519,21 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-06-08T12:52:13+00:00"
+            "time": "2023-07-26T07:16:09+00:00"
+        }
+    ],
+    "aliases": [
+        {
+            "package": "utopia-php/storage",
+            "version": "dev-feat-transfer",
+            "alias": "0.13.3",
+            "alias_normalized": "0.13.3.0"
         }
     ],
-    "aliases": [],
     "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {
+        "utopia-php/storage": 20
+    },
     "prefer-stable": false,
     "prefer-lowest": false,
     "platform": {
@@ -5547,5 +5557,5 @@
     "platform-overrides": {
         "php": "8.0"
     },
-    "plugin-api-version": "2.3.0"
+    "plugin-api-version": "2.2.0"
 }

docker-compose.yml

@@ -721,7 +721,7 @@ services:
       - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG
 
   mariadb:
-    image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p
+    image: mysql:8.0.33 # fix issues when upgrading using: mysql_upgrade -u root -p
     container_name: appwrite-mariadb
     <<: *x-logging
     networks:
@@ -748,6 +748,64 @@ services:
   #     - SMARTHOST_HOST=smtp
   #     - SMARTHOST_PORT=587
 
+  xtrabackup:
+    image: percona/percona-xtrabackup:latest
+    container_name: appwrite-xtrabackup
+    command: sleep infinity
+    volumes:
+      - backup-data:/backups:rw
+      - appwrite-mariadb:/var/lib/mysql:r
+    networks:
+      - appwrite
+    user: 'root:root'
+
+  appwrite-backup:
+    entrypoint: db-backup
+    command:
+      - --database=db_fra1_02
+    <<: *x-logging
+    container_name: appwrite-backup
+    image: appwrite-dev
+    networks:
+      - appwrite
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+      - backup-data:/backups:rw
+      - appwrite-mariadb:/var/lib/mysql:r
+    depends_on:
+      - redis
+    environment:
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_DB_REPLICAS
+      - _DO_SPACES_BUCKET_NAME
+      - _DO_SPACES_ACCESS_KEY
+      - _DO_SPACES_SECRET_KEY
+      - _DO_SPACES_REGION
+
+  appwrite-restore:
+    command: sleep infinity
+    container_name: appwrite-backup-restore
+    image: appwrite-dev
+    networks:
+      - appwrite
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+      - backup-data:/backups:rw
+      - appwrite-mariadb:/var/lib/mysql:rw
+    depends_on:
+      - redis
+    environment:
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_DB_REPLICAS
+      - _DO_SPACES_BUCKET_NAME
+      - _DO_SPACES_ACCESS_KEY
+      - _DO_SPACES_SECRET_KEY
+      - _DO_SPACES_REGION
+
   redis:
     image: redis:7.0.4-alpine
     <<: *x-logging
@@ -910,4 +968,5 @@ volumes:
   appwrite-config:
   appwrite-functions:
   appwrite-builds:
+  backup-data:
   # appwrite-chronograf:

src/Appwrite/Platform/Services/Tasks.php

@@ -2,6 +2,8 @@
 
 namespace Appwrite\Platform\Services;
 
+use Appwrite\Platform\Tasks\Backup;
+use Appwrite\Platform\Tasks\Restore;
 use Utopia\Platform\Service;
 use Appwrite\Platform\Tasks\Doctor;
 use Appwrite\Platform\Tasks\Install;
@@ -48,6 +50,8 @@ class Tasks extends Service
             ->addAction(CalcUsersStats::getName(), new CalcUsersStats())
             ->addAction(CalcTierStats::getName(), new CalcTierStats())
             ->addAction(PatchDeleteProjectCollections::getName(), new PatchDeleteProjectCollections())
+            ->addAction(Backup::getName(), new Backup())
+            ->addAction(Restore::getName(), new Restore())
         ;
     }
 }

src/Appwrite/Platform/Tasks/Backup.php

@@ -0,0 +1,256 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Exception;
+use Utopia\DSN\DSN;
+use Utopia\Platform\Action;
+use Utopia\App;
+use Utopia\CLI\Console;
+use Utopia\Pools\Group;
+use Utopia\Storage\Device;
+use Utopia\Storage\Device\DOSpaces;
+use Utopia\Storage\Device\Local;
+use Utopia\Validator\Text;
+
+class Backup extends Action
+{
+    public const BACKUPS_PATH = '/backups';
+    public const BACKUP_INTERVAL_SECONDS = 60 * 60 * 4; // 4 hours;
+    public const COMPRESS_ALGORITHM = 'lz4';
+    public const CONFIG_PATH = '/etc/my.cnf';
+    public const PROCESSORS = 4;
+    protected ?DSN $dsn = null;
+    protected ?string $database = null;
+    protected ?DOSpaces $s3 = null;
+
+    /**
+     * @throws Exception
+     */
+    public function __construct()
+    {
+        $this->checkEnvVariables();
+
+        $this
+            ->desc('Backup a database')
+            ->param('database', null, new Text(20), 'Database name, for example db_fra1_01')
+            ->inject('pools')
+            ->callback(fn(string $database, Group $pools) => $this->action($database, $pools));
+    }
+
+    public static function getName(): string
+    {
+        return 'backup';
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function action(string $database, Group $pools): void
+    {
+        $this->database = $database;
+        $this->dsn = $this->getDsn($database);
+        $this->s3 = new DOSpaces('/' . $database . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
+
+        if (is_null($this->dsn)) {
+            Console::error('No DSN match');
+            Console::exit();
+        }
+
+        $attempts = 0;
+        $max = 10;
+        $sleep = 5;
+
+        do {
+            try {
+                $attempts++;
+                $resource = $pools
+                    ->get('replica_' . $database)
+                    ->pop()
+                    ->getResource();
+                var_dump($resource);
+                break; // leave the do-while if successful
+            } catch (Exception $e) {
+                Console::warning("Database not ready. Retrying connection ({$attempts})...");
+                if ($attempts >= $max) {
+                    throw new Exception('Failed to connect to database: ' . $e->getMessage());
+                }
+
+                sleep($sleep);
+            }
+        } while ($attempts < $max);
+
+        Console::loop(function () {
+            $this->start();
+        }, self::BACKUP_INTERVAL_SECONDS);
+    }
+
+    public function start(): void
+    {
+        $start = microtime(true);
+        $time = date('Y_m_d_H_i_s');
+
+        self::log('--- Backup Start --- ');
+        self::log('--- Creating backup ' . $time . '  --- ');
+
+        $filename = $time . '.tar.gz';
+        $local = new Local(self::BACKUPS_PATH . '/' . $this->database . '/full/' . $time);
+        $local->setTransferChunkSize(5 * 1024 * 1024); // 5MB
+
+        $backups = $local->getRoot() . '/files';
+        $tarFile = $local->getPath($filename);
+
+        $this->fullBackup($backups);
+        $this->tar($backups, $tarFile);
+        $this->upload($tarFile, $local);
+
+        self::log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- '   . PHP_EOL . PHP_EOL);
+    }
+
+    public function fullBackup(string $target)
+    {
+        if (!file_exists(self::BACKUPS_PATH)) {
+            Console::error('Mount directory does not exist');
+            Console::exit();
+        }
+
+        if (!file_exists($target) && !mkdir($target, 0755, true)) {
+            Console::error('Error creating directory: ' . $target);
+            Console::exit();
+        }
+
+        $logfile = $target . '/../backup.log';
+
+        $args = [
+            '--user=' . $this->dsn->getUser(),
+            '--password=' . $this->dsn->getPassword(),
+            '--host=' . $this->dsn->getHost(),
+            '--backup',
+            '--strict',
+            '--history=' . $this->database, // logs PERCONA_SCHEMA.xtrabackup_history name attribute
+            '--slave-info',
+            '--safe-slave-backup',
+            '--safe-slave-backup-timeout=300',
+            '--check-privileges', // checks if Percona XtraBackup has all the required privileges.
+            '--target-dir=' . $target,
+            '--parallel=' . self::PROCESSORS,
+            '--compress=' . self::COMPRESS_ALGORITHM,
+            '--compress-threads=' . self::PROCESSORS,
+            '--rsync', // https://docs.percona.com/percona-xtrabackup/8.0/accelerate-backup-process.html
+            '2> ' . $logfile,
+        ];
+
+        $cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
+        self::log($cmd);
+        shell_exec($cmd);
+
+        $stderr = shell_exec('tail -1 ' . $logfile);
+        self::log($stderr);
+
+        if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
+            Console::error('Backup failed');
+            Console::exit();
+        }
+
+        if (!unlink($logfile)) {
+            Console::error('Error deleting: ' . $logfile);
+            Console::exit();
+        }
+    }
+
+    public function tar(string $directory, string $file)
+    {
+        $stdout = '';
+        $stderr = '';
+
+        $cmd = 'cd ' . $directory . ' && tar zcf ' . $file . ' . && cd ' . getcwd();
+        self::log($cmd);
+        Console::execute($cmd, '', $stdout, $stderr);
+        self::log($stdout);
+        if (!empty($stderr)) {
+            Console::error($stderr);
+            Console::exit();
+        }
+
+        if (!file_exists($file)) {
+            Console::error('Can\'t find tar file: ' . $file);
+            Console::exit();
+        }
+
+        $filesize = \filesize($file);
+        self::log('Tar file size is: ' . ceil($filesize / 1024 / 1024) . 'MB');
+        if ($filesize < (2 * 1024 * 1024)) {
+            Console::error('File size is very small: ' . $file);
+            Console::exit();
+        }
+    }
+
+    public function upload(string $file, Device $local)
+    {
+        $filename = basename($file);
+
+        if (!$this->s3->exists('/')) {
+            Console::error('Can\'t read s3 root directory');
+            Console::exit();
+        }
+
+        try {
+            self::log('Uploading: ' . $file);
+            $destination = $this->s3->getRoot() . '/' . $filename;
+
+            if (!$local->transfer($file, $destination, $this->s3)) {
+                Console::error('Error uploading to ' . $destination);
+                Console::exit();
+            }
+
+            if (!$this->s3->exists($destination)) {
+                Console::error('File not found in destination: ' . $destination);
+                Console::exit();
+            }
+        } catch (Exception $e) {
+            Console::error($e->getMessage());
+            Console::exit();
+        }
+
+        if (!unlink($file)) {
+            Console::error('Error deleting: ' . $file);
+            Console::exit();
+        }
+    }
+
+    public static function log(string $message): void
+    {
+        if (!empty($message)) {
+            Console::log(date('Y-m-d H:i:s') . ' ' . $message);
+        }
+    }
+
+    public function checkEnvVariables(): void
+    {
+        foreach (
+            [
+                '_APP_CONNECTIONS_DB_REPLICAS',
+                '_DO_SPACES_BUCKET_NAME',
+                '_DO_SPACES_ACCESS_KEY',
+                '_DO_SPACES_SECRET_KEY',
+                '_DO_SPACES_REGION',
+            ] as $env
+        ) {
+            if (empty(App::getEnv($env))) {
+                Console::error('Can\'t read ' . $env);
+                Console::exit();
+            }
+        }
+    }
+
+    public function getDsn(string $database): ?DSN
+    {
+        foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_REPLICAS')) as $project) {
+            [$db, $dsn] = explode('=', $project);
+            if ($db === $database) {
+                return new DSN($dsn);
+            }
+        }
+        return null;
+    }
+}

src/Appwrite/Platform/Tasks/Restore.php

@@ -0,0 +1,276 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Exception;
+use Utopia\App;
+use Utopia\DSN\DSN;
+use Utopia\Platform\Action;
+use Utopia\CLI\Console;
+use Utopia\Storage\Device;
+use Utopia\Storage\Device\DOSpaces;
+use Utopia\Storage\Device\Local;
+use Utopia\Validator\Text;
+use Utopia\Validator\WhiteList;
+
+class Restore extends Action
+{
+    public const BACKUPS_PATH = '/backups';
+    public const DATADIR = '/var/lib/mysql';
+    public const PROCESSORS = 4;
+    protected ?DSN $dsn = null;
+    protected string $database;
+    protected ?DOSpaces $s3 = null;
+
+    public function __construct()
+    {
+        $this->checkEnvVariables();
+
+        $this
+            ->desc('Restore a DB')
+            ->param('id', '', new Text(20), 'The backup identification')
+            ->param('cloud', null, new WhiteList(['true', 'false'], true), 'Download backup from cloud or use local directory')
+            ->param('database', null, new Text(10), 'The Database name for example db_fra1_01')
+            ->callback(fn ($id, $cloud, $project) => $this->action($id, $cloud, $project));
+    }
+
+    public static function getName(): string
+    {
+        return 'restore';
+    }
+
+    public function action(string $id, string $cloud, string $database): void
+    {
+        $this->database = $database;
+        $this->dsn = $this->getDsn($database);
+        $this->s3 = new DOSpaces('/' . $database . '/full', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
+        $datadir = self::DATADIR;
+
+        if (is_null($this->dsn)) {
+            Console::error('No DSN match');
+            Console::exit();
+        }
+
+        if (!file_exists($datadir)) {
+            Console::error('Datadir not found: ' . $datadir);
+            Console::exit();
+        }
+
+        if (file_exists($datadir . '/sys') || file_exists($datadir . '/appwrite')) {
+            Console::error('Datadir ' . $datadir . ' must be empty!');
+            Console::exit();
+        }
+
+        $this->log('--- Restore Start ' . $id . ' --- ');
+
+        $filename = $id . '.tar.gz';
+        $start = microtime(true);
+        $cloud = $cloud === 'true';
+
+        if ($cloud) {
+            $local = new Local(self::BACKUPS_PATH . '/downloads/' . $id);
+
+            $files = $local->getRoot() . '/files';
+
+            if (!file_exists($files) && !mkdir($files, 0755, true)) {
+                Console::error('Error creating directory: ' . $files);
+                Console::exit();
+            }
+
+            $file = $local->getPath($filename);
+
+            if (!file_exists($file)) {
+                $this->download($file, $local);
+            }
+
+            $this->untar($file, $files);
+        } else {
+            $local = new Local(self::BACKUPS_PATH . '/' . $database . '/full/' . $id);
+            $files = $local->getRoot() . '/files';
+        }
+
+        if (!file_exists($files)) {
+            Console::error('Directory not found: ' . $files);
+            Console::exit();
+        }
+
+        $this->decompress($files);
+        $this->prepare($files);
+        $this->restore($files, $cloud, $datadir);
+
+        $this->log('Restore Finish in ' . (microtime(true) - $start) . ' seconds');
+    }
+
+    public function download(string $file, Device $local)
+    {
+        $filename = basename($file);
+        try {
+            $path = $this->s3->getPath($filename);
+
+            if (!$this->s3->exists($path)) {
+                Console::error('File: ' . $path . ' does not exist on cloud');
+                Console::exit();
+            }
+
+            $this->log('Downloading: ' . $file);
+
+            if (!$this->s3->transfer($path, $file, $local)) {
+                Console::error('Error Downloading ' . $file);
+                Console::exit();
+            }
+        } catch (Exception $e) {
+            Console::error($e->getMessage());
+            Console::exit();
+        }
+    }
+
+    public function untar(string $file, string $directory)
+    {
+        $stdout = '';
+        $stderr = '';
+        $cmd = 'tar -xzf ' . $file . ' -C ' . $directory;
+        $this->log($cmd);
+        Console::execute($cmd, '', $stdout, $stderr);
+        if (!empty($stderr)) {
+            Console::error($stderr);
+            Console::exit();
+        }
+
+        if (!file_exists($file)) {
+            Console::error('Restore file not found: ' . $file);
+            Console::exit();
+        }
+    }
+
+    public function decompress(string $target)
+    {
+        $logfile = $target . '/../log.txt';
+
+        $args = [
+            '--user=' . $this->dsn->getUser(),
+            '--password=' . $this->dsn->getPassword(),
+            '--host=' . $this->dsn->getHost(),
+            '--decompress',
+            '--strict',
+            '--remove-original', // Removes *.lz4 compressed files
+            '--parallel=' . self::PROCESSORS,
+            '--compress-threads=' . self::PROCESSORS,
+            '--target-dir=' . $target,
+            '2> ' . $logfile,
+        ];
+
+        $cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
+        $this->log($cmd);
+        shell_exec($cmd);
+
+        $stderr = shell_exec('tail -1 ' . $logfile);
+        $this->log($stderr);
+
+        if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
+            Console::error('Decompress failed');
+            Console::exit();
+        }
+    }
+
+    public function prepare(string $target)
+    {
+        if (!file_exists($target)) {
+            Console::error('prepare error directory not found: ' . $target);
+            Console::exit();
+        }
+
+        $logfile = $target . '/../log.txt';
+
+        $args = [
+            '--user=' . $this->dsn->getUser(),
+            '--password=' . $this->dsn->getPassword(),
+            '--host=' . $this->dsn->getHost(),
+            '--prepare',
+            '--strict',
+            '--target-dir=' . $target,
+            '2> ' . $logfile,
+        ];
+
+        $cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
+        $this->log($cmd);
+        shell_exec($cmd);
+
+        $stderr = shell_exec('tail -1 ' . $logfile);
+        $this->log($stderr);
+
+        if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
+            Console::error('Prepare failed');
+            Console::exit();
+        }
+    }
+
+    public function restore(string $target, bool $cloud, string $datadir)
+    {
+        if (!file_exists($target)) {
+            Console::error('restore error directory not found: ' . $target);
+            Console::exit();
+        }
+
+        $logfile = $target . '/../log.txt';
+
+        $args = [
+            '--user=' . $this->dsn->getUser(),
+            '--password=' . $this->dsn->getPassword(),
+            '--host=' . $this->dsn->getHost(),
+            $cloud ? '--move-back' : '--copy-back',
+            '--strict',
+            '--target-dir=' . $target,
+            '--datadir=' . $datadir,
+            '--parallel=' . self::PROCESSORS,
+            '2> ' . $logfile,
+        ];
+
+        $cmd = 'docker exec appwrite-xtrabackup xtrabackup ' . implode(' ', $args);
+        $this->log($cmd);
+        shell_exec($cmd);
+
+        $stderr = shell_exec('tail -1 ' . $logfile);
+        $this->log($stderr);
+
+        if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
+            Console::error('Restore failed');
+            Console::exit();
+        }
+    }
+
+    public function checkEnvVariables(): void
+    {
+        foreach (
+            [
+                '_APP_CONNECTIONS_DB_REPLICAS',
+                '_DO_SPACES_BUCKET_NAME',
+                '_DO_SPACES_ACCESS_KEY',
+                '_DO_SPACES_SECRET_KEY',
+                '_DO_SPACES_REGION',
+            ] as $env
+        ) {
+            if (empty(App::getEnv($env))) {
+                Console::error('Can\'t read ' . $env);
+                Console::exit();
+            }
+        }
+    }
+
+    public function log(string $message): void
+    {
+        if (!empty($message)) {
+            Console::log(date('Y-m-d H:i:s') . ' ' . $message);
+        }
+    }
+
+    public function getDsn(string $database): ?DSN
+    {
+        foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_REPLICAS')) as $project) {
+            [$db, $dsn] = explode('=', $project);
+            if ($db === $database) {
+                return new DSN($dsn);
+            }
+        }
+        return null;
+    }
+}