Space

.env

@@ -27,14 +27,14 @@ _APP_DB_PASS=password
 _APP_DB_ROOT_PASS=rootsecretpassword
 _APP_CONNECTIONS_MAX=3100
 _APP_POOL_CLIENTS=14
-_APP_CONNECTIONS_DB_REPLICAS=db_fra1_02=mariadb://root:rootsecretpassword@mariadb:3306/appwrite
+_APP_CONNECTIONS_DB_REPLICAS=db_fra1_03=mariadb://root:rootsecretpassword@mariadb:3306/appwrite
 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
 _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
 _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
 _APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379
 _APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379
 _APP_CONNECTIONS_STORAGE=local://localhost
-_APP_CONNECTIONS_BACKUPS_STORAGE=dospaces://ACCESS:SECRET@default/backups-v1?region=fra1
+_APP_CONNECTIONS_BACKUPS_STORAGE=dospaces://DO006P96DUQVNMLTB8TP:QwxTmU1JfjWsuHecNGiDDvp1UNrMhgCnk5rvJ2FIpm0@default/backups-v1?region=fra1
 _APP_STORAGE_ANTIVIRUS=disabled
 _APP_STORAGE_ANTIVIRUS_HOST=clamav
 _APP_STORAGE_ANTIVIRUS_PORT=3310

composer.json

@@ -50,7 +50,7 @@
         "utopia-php/cli": "0.15.*",
         "utopia-php/config": "0.2.*",
         "utopia-php/database": "0.28.*",
-        "utopia-php/domains": "1.1.*",
+        "utopia-php/domains": "0.3.*",
         "utopia-php/dsn": "0.1.*",
         "utopia-php/framework": "0.26.*",
         "utopia-php/image": "0.5.*",
@@ -63,7 +63,7 @@
         "utopia-php/preloader": "0.2.*",
         "utopia-php/queue": "0.5.*",
         "utopia-php/registry": "0.5.*",
-        "utopia-php/storage": "dev-feat-transfer as 0.13.3",
+        "utopia-php/storage": "dev-s3-getfiles as 0.16.0",
         "utopia-php/swoole": "0.5.*",
         "utopia-php/websocket": "0.1.0",
         "resque/php-resque": "1.3.6",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "ff72579b1906a3ee181ec1209198c13f",
+    "content-hash": "2972c8c647bca05756f87f2bb39b035c",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -408,22 +408,22 @@
         },
         {
             "name": "guzzlehttp/guzzle",
-            "version": "7.7.0",
+            "version": "7.8.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/guzzle/guzzle.git",
-                "reference": "fb7566caccf22d74d1ab270de3551f72a58399f5"
+                "reference": "1110f66a6530a40fe7aea0378fe608ee2b2248f9"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/guzzle/guzzle/zipball/fb7566caccf22d74d1ab270de3551f72a58399f5",
-                "reference": "fb7566caccf22d74d1ab270de3551f72a58399f5",
+                "url": "https://api.github.com/repos/guzzle/guzzle/zipball/1110f66a6530a40fe7aea0378fe608ee2b2248f9",
+                "reference": "1110f66a6530a40fe7aea0378fe608ee2b2248f9",
                 "shasum": ""
             },
             "require": {
                 "ext-json": "*",
-                "guzzlehttp/promises": "^1.5.3 || ^2.0",
-                "guzzlehttp/psr7": "^1.9.1 || ^2.4.5",
+                "guzzlehttp/promises": "^1.5.3 || ^2.0.1",
+                "guzzlehttp/psr7": "^1.9.1 || ^2.5.1",
                 "php": "^7.2.5 || ^8.0",
                 "psr/http-client": "^1.0",
                 "symfony/deprecation-contracts": "^2.2 || ^3.0"
@@ -514,7 +514,7 @@
             ],
             "support": {
                 "issues": "https://github.com/guzzle/guzzle/issues",
-                "source": "https://github.com/guzzle/guzzle/tree/7.7.0"
+                "source": "https://github.com/guzzle/guzzle/tree/7.8.0"
             },
             "funding": [
                 {
@@ -530,7 +530,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-05-21T14:04:53+00:00"
+            "time": "2023-08-27T10:20:53+00:00"
         },
         {
             "name": "guzzlehttp/promises",
@@ -617,16 +617,16 @@
         },
         {
             "name": "guzzlehttp/psr7",
-            "version": "2.6.0",
+            "version": "2.6.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/guzzle/psr7.git",
-                "reference": "8bd7c33a0734ae1c5d074360512beb716bef3f77"
+                "reference": "be45764272e8873c72dbe3d2edcfdfcc3bc9f727"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/guzzle/psr7/zipball/8bd7c33a0734ae1c5d074360512beb716bef3f77",
-                "reference": "8bd7c33a0734ae1c5d074360512beb716bef3f77",
+                "url": "https://api.github.com/repos/guzzle/psr7/zipball/be45764272e8873c72dbe3d2edcfdfcc3bc9f727",
+                "reference": "be45764272e8873c72dbe3d2edcfdfcc3bc9f727",
                 "shasum": ""
             },
             "require": {
@@ -713,7 +713,7 @@
             ],
             "support": {
                 "issues": "https://github.com/guzzle/psr7/issues",
-                "source": "https://github.com/guzzle/psr7/tree/2.6.0"
+                "source": "https://github.com/guzzle/psr7/tree/2.6.1"
             },
             "funding": [
                 {
@@ -729,7 +729,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-08-03T15:06:02+00:00"
+            "time": "2023-08-27T10:13:57+00:00"
         },
         {
             "name": "influxdb/influxdb-php",
@@ -1903,16 +1903,16 @@
         },
         {
             "name": "utopia-php/database",
-            "version": "0.28.0",
+            "version": "0.28.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/database.git",
-                "reference": "ef6506af1c09c22f5dc1e7859159d323f7fafa94"
+                "reference": "b9f62a9496e33aa8182c49540a649c1f36806a42"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/database/zipball/ef6506af1c09c22f5dc1e7859159d323f7fafa94",
-                "reference": "ef6506af1c09c22f5dc1e7859159d323f7fafa94",
+                "url": "https://api.github.com/repos/utopia-php/database/zipball/b9f62a9496e33aa8182c49540a649c1f36806a42",
+                "reference": "b9f62a9496e33aa8182c49540a649c1f36806a42",
                 "shasum": ""
             },
             "require": {
@@ -1951,29 +1951,31 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/database/issues",
-                "source": "https://github.com/utopia-php/database/tree/0.28.0"
+                "source": "https://github.com/utopia-php/database/tree/0.28.2"
             },
-            "time": "2022-10-31T09:58:46+00:00"
+            "time": "2023-08-20T16:22:59+00:00"
         },
         {
             "name": "utopia-php/domains",
-            "version": "v1.1.0",
+            "version": "0.3.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/domains.git",
-                "reference": "1665e1d9932afa3be63b5c1e0dcfe01fe77d8e73"
+                "reference": "aaa8c9a96c69ccb397997b1f4f2299c66f77eefb"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/domains/zipball/1665e1d9932afa3be63b5c1e0dcfe01fe77d8e73",
-                "reference": "1665e1d9932afa3be63b5c1e0dcfe01fe77d8e73",
+                "url": "https://api.github.com/repos/utopia-php/domains/zipball/aaa8c9a96c69ccb397997b1f4f2299c66f77eefb",
+                "reference": "aaa8c9a96c69ccb397997b1f4f2299c66f77eefb",
                 "shasum": ""
             },
             "require": {
-                "php": ">=7.1"
+                "php": ">=8.0",
+                "utopia-php/framework": "0.*.*"
             },
             "require-dev": {
-                "phpunit/phpunit": "^7.0"
+                "laravel/pint": "1.2.*",
+                "phpunit/phpunit": "^9.3"
             },
             "type": "library",
             "autoload": {
@@ -1989,6 +1991,10 @@
                 {
                     "name": "Eldad Fux",
                     "email": "eldad@appwrite.io"
+                },
+                {
+                    "name": "Wess Cope",
+                    "email": "wess@appwrite.io"
                 }
             ],
             "description": "Utopia Domains library is simple and lite library for parsing web domains. This library is aiming to be as simple and easy to learn and use.",
@@ -2005,9 +2011,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/domains/issues",
-                "source": "https://github.com/utopia-php/domains/tree/master"
+                "source": "https://github.com/utopia-php/domains/tree/0.3.2"
             },
-            "time": "2020-02-23T07:40:02+00:00"
+            "time": "2023-07-19T16:39:24+00:00"
         },
         {
             "name": "utopia-php/dsn",
@@ -2622,16 +2628,16 @@
         },
         {
             "name": "utopia-php/storage",
-            "version": "dev-feat-transfer",
+            "version": "dev-s3-getfiles",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/storage.git",
-                "reference": "cef63e2d76156c2069c98ccd1430ebac5df3a35c"
+                "reference": "76e440f1374bf23bb82f28701786af1a5a3b2d05"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/storage/zipball/cef63e2d76156c2069c98ccd1430ebac5df3a35c",
-                "reference": "cef63e2d76156c2069c98ccd1430ebac5df3a35c",
+                "url": "https://api.github.com/repos/utopia-php/storage/zipball/76e440f1374bf23bb82f28701786af1a5a3b2d05",
+                "reference": "76e440f1374bf23bb82f28701786af1a5a3b2d05",
                 "shasum": ""
             },
             "require": {
@@ -2671,9 +2677,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/storage/issues",
-                "source": "https://github.com/utopia-php/storage/tree/feat-transfer"
+                "source": "https://github.com/utopia-php/storage/tree/s3-getfiles"
             },
-            "time": "2023-08-08T08:09:59+00:00"
+            "time": "2023-09-05T08:27:10+00:00"
         },
         {
             "name": "utopia-php/swoole",
@@ -3253,16 +3259,16 @@
         },
         {
             "name": "nikic/php-parser",
-            "version": "v4.16.0",
+            "version": "v4.17.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/nikic/PHP-Parser.git",
-                "reference": "19526a33fb561ef417e822e85f08a00db4059c17"
+                "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/19526a33fb561ef417e822e85f08a00db4059c17",
-                "reference": "19526a33fb561ef417e822e85f08a00db4059c17",
+                "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d",
+                "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d",
                 "shasum": ""
             },
             "require": {
@@ -3303,9 +3309,9 @@
             ],
             "support": {
                 "issues": "https://github.com/nikic/PHP-Parser/issues",
-                "source": "https://github.com/nikic/PHP-Parser/tree/v4.16.0"
+                "source": "https://github.com/nikic/PHP-Parser/tree/v4.17.1"
             },
-            "time": "2023-06-25T14:52:30+00:00"
+            "time": "2023-08-13T19:53:39+00:00"
         },
         {
             "name": "phar-io/manifest",
@@ -3530,16 +3536,16 @@
         },
         {
             "name": "phpdocumentor/type-resolver",
-            "version": "1.7.2",
+            "version": "1.7.3",
             "source": {
                 "type": "git",
                 "url": "https://github.com/phpDocumentor/TypeResolver.git",
-                "reference": "b2fe4d22a5426f38e014855322200b97b5362c0d"
+                "reference": "3219c6ee25c9ea71e3d9bbaf39c67c9ebd499419"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/b2fe4d22a5426f38e014855322200b97b5362c0d",
-                "reference": "b2fe4d22a5426f38e014855322200b97b5362c0d",
+                "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/3219c6ee25c9ea71e3d9bbaf39c67c9ebd499419",
+                "reference": "3219c6ee25c9ea71e3d9bbaf39c67c9ebd499419",
                 "shasum": ""
             },
             "require": {
@@ -3582,9 +3588,9 @@
             "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names",
             "support": {
                 "issues": "https://github.com/phpDocumentor/TypeResolver/issues",
-                "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.7.2"
+                "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.7.3"
             },
-            "time": "2023-05-30T18:13:47+00:00"
+            "time": "2023-08-12T11:01:26+00:00"
         },
         {
             "name": "phpspec/prophecy",
@@ -3656,16 +3662,16 @@
         },
         {
             "name": "phpstan/phpdoc-parser",
-            "version": "1.23.1",
+            "version": "1.24.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/phpstan/phpdoc-parser.git",
-                "reference": "846ae76eef31c6d7790fac9bc399ecee45160b26"
+                "reference": "3510b0a6274cc42f7219367cb3abfc123ffa09d6"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/846ae76eef31c6d7790fac9bc399ecee45160b26",
-                "reference": "846ae76eef31c6d7790fac9bc399ecee45160b26",
+                "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/3510b0a6274cc42f7219367cb3abfc123ffa09d6",
+                "reference": "3510b0a6274cc42f7219367cb3abfc123ffa09d6",
                 "shasum": ""
             },
             "require": {
@@ -3697,9 +3703,9 @@
             "description": "PHPDoc parser with support for nullable, intersection and generic types",
             "support": {
                 "issues": "https://github.com/phpstan/phpdoc-parser/issues",
-                "source": "https://github.com/phpstan/phpdoc-parser/tree/1.23.1"
+                "source": "https://github.com/phpstan/phpdoc-parser/tree/1.24.0"
             },
-            "time": "2023-08-03T16:32:59+00:00"
+            "time": "2023-09-07T20:46:32+00:00"
         },
         {
             "name": "phpunit/php-code-coverage",
@@ -5188,16 +5194,16 @@
         },
         {
             "name": "symfony/polyfill-ctype",
-            "version": "v1.27.0",
+            "version": "v1.28.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/polyfill-ctype.git",
-                "reference": "5bbc823adecdae860bb64756d639ecfec17b050a"
+                "reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/5bbc823adecdae860bb64756d639ecfec17b050a",
-                "reference": "5bbc823adecdae860bb64756d639ecfec17b050a",
+                "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb",
+                "reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb",
                 "shasum": ""
             },
             "require": {
@@ -5212,7 +5218,7 @@
             "type": "library",
             "extra": {
                 "branch-alias": {
-                    "dev-main": "1.27-dev"
+                    "dev-main": "1.28-dev"
                 },
                 "thanks": {
                     "name": "symfony/polyfill",
@@ -5250,7 +5256,7 @@
                 "portable"
             ],
             "support": {
-                "source": "https://github.com/symfony/polyfill-ctype/tree/v1.27.0"
+                "source": "https://github.com/symfony/polyfill-ctype/tree/v1.28.0"
             },
             "funding": [
                 {
@@ -5266,20 +5272,20 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2022-11-03T14:55:06+00:00"
+            "time": "2023-01-26T09:26:14+00:00"
         },
         {
             "name": "symfony/polyfill-mbstring",
-            "version": "v1.27.0",
+            "version": "v1.28.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/polyfill-mbstring.git",
-                "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534"
+                "reference": "42292d99c55abe617799667f454222c54c60e229"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/8ad114f6b39e2c98a8b0e3bd907732c207c2b534",
-                "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534",
+                "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/42292d99c55abe617799667f454222c54c60e229",
+                "reference": "42292d99c55abe617799667f454222c54c60e229",
                 "shasum": ""
             },
             "require": {
@@ -5294,7 +5300,7 @@
             "type": "library",
             "extra": {
                 "branch-alias": {
-                    "dev-main": "1.27-dev"
+                    "dev-main": "1.28-dev"
                 },
                 "thanks": {
                     "name": "symfony/polyfill",
@@ -5333,7 +5339,7 @@
                 "shim"
             ],
             "support": {
-                "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.27.0"
+                "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.28.0"
             },
             "funding": [
                 {
@@ -5349,7 +5355,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2022-11-03T14:55:06+00:00"
+            "time": "2023-07-28T09:04:16+00:00"
         },
         {
             "name": "textalk/websocket",
@@ -5452,16 +5458,16 @@
         },
         {
             "name": "twig/twig",
-            "version": "v3.7.0",
+            "version": "v3.7.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/twigphp/Twig.git",
-                "reference": "5cf942bbab3df42afa918caeba947f1b690af64b"
+                "reference": "a0ce373a0ca3bf6c64b9e3e2124aca502ba39554"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/twigphp/Twig/zipball/5cf942bbab3df42afa918caeba947f1b690af64b",
-                "reference": "5cf942bbab3df42afa918caeba947f1b690af64b",
+                "url": "https://api.github.com/repos/twigphp/Twig/zipball/a0ce373a0ca3bf6c64b9e3e2124aca502ba39554",
+                "reference": "a0ce373a0ca3bf6c64b9e3e2124aca502ba39554",
                 "shasum": ""
             },
             "require": {
@@ -5471,7 +5477,7 @@
             },
             "require-dev": {
                 "psr/container": "^1.0|^2.0",
-                "symfony/phpunit-bridge": "^4.4.9|^5.0.9|^6.0"
+                "symfony/phpunit-bridge": "^5.4.9|^6.3"
             },
             "type": "library",
             "autoload": {
@@ -5507,7 +5513,7 @@
             ],
             "support": {
                 "issues": "https://github.com/twigphp/Twig/issues",
-                "source": "https://github.com/twigphp/Twig/tree/v3.7.0"
+                "source": "https://github.com/twigphp/Twig/tree/v3.7.1"
             },
             "funding": [
                 {
@@ -5519,15 +5525,15 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-07-26T07:16:09+00:00"
+            "time": "2023-08-28T11:09:02+00:00"
         }
     ],
     "aliases": [
         {
             "package": "utopia-php/storage",
-            "version": "dev-feat-transfer",
-            "alias": "0.13.3",
-            "alias_normalized": "0.13.3.0"
+            "version": "dev-s3-getfiles",
+            "alias": "0.16.0",
+            "alias_normalized": "0.16.0.0"
         }
     ],
     "minimum-stability": "stable",

docker-compose.yml

@@ -750,11 +750,11 @@ services:
   #     - SMARTHOST_PORT=587
 
   xtrabackup:
-    image: percona/percona-xtrabackup:latest
+    image: percona/percona-xtrabackup:8.0.33
     container_name: xtrabackup
     command: sleep infinity
     volumes:
-      - backup-data:/backups:rw
+      - ./backup-data:/backups:rw
       - appwrite-mariadb:/var/lib/mysql:r
     networks:
       - appwrite
@@ -763,7 +763,7 @@ services:
   appwrite-backup:
     entrypoint: db-backup
     command:
-      - --database=db_fra1_02
+      - --database=db_fra1_03
     <<: *x-logging
     container_name: appwrite-backup
     image: appwrite-dev
@@ -773,13 +773,13 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./app:/usr/src/code/app
       - ./src:/usr/src/code/src
-      - backup-data:/backups:rw
-      - appwrite-mariadb:/var/lib/mysql:r
+      - ./backup-data:/backups:rw
+
     depends_on:
       - redis
     environment:
       - _APP_CONNECTIONS_DB_REPLICAS
-      - _APP_CONNECTIONS_BACKUPS_STORAGE=dospaces://DO00ZZLVBLWLP6JJJVM2:aaEZ+eu8k80raNoDPlcwX8fwzSr3CzL5P+4UiwMnugw@default/cloud-fra1-stg-backups-1?region=fra1
+      - _APP_CONNECTIONS_BACKUPS_STORAGE
 
   appwrite-restore:
     command: sleep infinity
@@ -791,7 +791,7 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./app:/usr/src/code/app
       - ./src:/usr/src/code/src
-      - backup-data:/backups:rw
+      - ./backup-data:/backups:rw
       - appwrite-mariadb:/var/lib/mysql:rw
     depends_on:
       - redis
@@ -961,5 +961,5 @@ volumes:
   appwrite-config:
   appwrite-functions:
   appwrite-builds:
-  backup-data:
+  #backup-data:
   # appwrite-chronograf:

src/Appwrite/Platform/Tasks/Backup.php

@@ -17,21 +17,22 @@ class Backup extends Action
 {
     public const BACKUPS_PATH = '/backups';
     public const BACKUP_INTERVAL_SECONDS = 60 * 60 * 4; // 4 hours;
-    public const COMPRESS_ALGORITHM = 'lz4';
-    public const CONFIG_PATH = '/etc/my.cnf';
-    public const PROCESSORS = 4;
+    public const COMPRESS_ALGORITHM = 'zstd'; // https://www.percona.com/blog/get-your-backup-to-half-of-its-size-introducing-zstd-support-in-percona-xtrabackup/
+    public const CLEANUP_LOCAL_FILES_SECONDS = 60 * 60 * 24 * 7; // 2 days?
+    public const CLEANUP_CLOUD_FILES_SECONDS = 60 * 60 * 24 * 14; // 14 days?;
+    public const UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024; // Must be greater than 5MB;
+    protected string $filename;
     protected ?DSN $dsn = null;
     protected ?string $database = null;
     protected ?DOSpaces $s3 = null;
     protected string $xtrabackupContainerId;
+    protected int $processors = 1;
 
     /**
      * @throws Exception
      */
     public function __construct()
     {
-        $this->checkEnvVariables();
-
         $this
             ->desc('Backup a database')
             ->param('database', null, new Text(20), 'Database name, for example db_fra1_01')
@@ -49,21 +50,16 @@ class Backup extends Action
      */
     public function action(string $database, Group $pools): void
     {
+        $this->checkEnvVariables();
+
         $this->database = $database;
         $this->dsn = $this->getDsn($database);
-
         if (is_null($this->dsn)) {
-            Console::error('No DSN match');
-            Console::exit();
+            throw new Exception('No DSN match');
         }
 
-        try {
-            $dsn = new DSN(App::getEnv('_APP_CONNECTIONS_BACKUPS_STORAGE'));
-            $this->s3 = new DOSpaces('/' . $database . '/full', $dsn->getUser(), $dsn->getPassword(), $dsn->getPath(), $dsn->getParam('region'));
-        } catch (\Exception $e) {
-            Console::error($e->getMessage() . 'Invalid DSN.');
-            Console::exit();
-        }
+        $dsn = new DSN(App::getEnv('_APP_CONNECTIONS_BACKUPS_STORAGE', ''));
+        $this->s3 = new DOSpaces('/' . $database . '/full', $dsn->getUser(), $dsn->getPassword(), $dsn->getPath(), $dsn->getParam('region'));
 
         $attempts = 0;
         $max = 10;
@@ -87,154 +83,230 @@ class Backup extends Action
                 sleep($sleep);
             }
         } while ($attempts < $max);
-        $containerId = shell_exec('docker ps -f "name=xtrabackup" --format "{{.ID}}"');
-        $containerId = str_replace(PHP_EOL, '', $containerId);
 
-        if (empty($containerId)) {
-            Console::error('Xtrabackup Container ID not found');
-            Console::exit();
-        }
-        $this->xtrabackupContainerId = $containerId;
+        $this->setContainerId();
+        $this->setProcessors();
 
         Console::loop(function () {
-            $this->start();
+            try {
+                $this->start();
+            } catch (Exception $e) {
+                //todo: send alerts to admin!
+                // todo: Do we want to terminate the script? or wait for next backup iteration?
+                Console::error($e->getMessage());
+            }
         }, self::BACKUP_INTERVAL_SECONDS);
     }
 
+    /**
+     * @throws Exception
+     */
     public function start(): void
     {
         $start = microtime(true);
-        $time = date('Y_m_d_H_i_s');
+        $this->filename = date('Y_m_d_H_i_s') . '.xbstream';
+        //$this->filename = '2023_09_06_08_19_24.xbstream';
 
-        self::log('--- Backup Start --- ');
-        self::log('--- Creating backup ' . $time . '  --- ');
+        self::log('--- Backup Start ' . $this->filename . ' --- ');
 
-        $filename = $time . '.tar.gz';
-        $local = new Local(self::BACKUPS_PATH . '/' . $this->database . '/full/' . $time);
-        $local->setTransferChunkSize(5 * 1024 * 1024); // 5MB
+        $local = new Local(self::BACKUPS_PATH . '/' . $this->database . '/full');
+        $local->setTransferChunkSize(self::UPLOAD_CHUNK_SIZE);
 
-        $backups = $local->getRoot() . '/files';
-        $tarFile = $local->getPath($filename);
+        $this->fullBackup($local);
 
-        $this->fullBackup($backups);
-        $this->tar($backups, $tarFile);
-        $this->upload($tarFile, $local);
+        $max = 1;
+        $i = 0;
+        while (true) {
+            $i++;
+            try {
+                $this->upload($local);
+                break;
+            } catch (Exception $e) {
+                if ($i <= $max) {
+                    Console::warning($e->getMessage() . ' (' . $i . ' attempt)');
+                } else {
+                    throw new Exception($e->getMessage());
+                }
+            }
+        }
 
-        self::log('--- Backup End ' . (microtime(true) - $start) . ' seconds --- '   . PHP_EOL . PHP_EOL);
+        $this->cleanLocalFiles($local);
+        $this->cleanCloudFiles();
+
+        self::log('--- Backup Finish ' . (microtime(true) - $start) . ' seconds --- '   . PHP_EOL . PHP_EOL);
     }
 
-    public function fullBackup(string $target)
+    /**
+     * @throws Exception
+     */
+    public function fullBackup(Device $device)
     {
+        self::log('Backup start');
+
+        $target = $device->getRoot();
+        $log = $device->getPath($this->filename . '.log');
+        $file = $device->getPath($this->filename);
+
         if (!file_exists(self::BACKUPS_PATH)) {
-            Console::error('Mount directory does not exist');
-            Console::exit();
+            throw new Exception('Mount directory does not exist');
         }
 
         if (!file_exists($target) && !mkdir($target, 0755, true)) {
-            Console::error('Error creating directory: ' . $target);
-            Console::exit();
+            throw new Exception('Error creating directory: ' . $target);
         }
 
-        $logfile = $target . '/../backup.log';
-
         $args = [
             'xtrabackup',
             '--user=' . $this->dsn->getUser(),
             '--password=' . $this->dsn->getPassword(),
             '--host=' . $this->dsn->getHost(),
             '--port=' . $this->dsn->getPort(),
-            '--no-server-version-check',
             '--backup',
+            '--stream=xbstream',
             '--strict',
-            '--history=' . $this->database, // logs PERCONA_SCHEMA.xtrabackup_history name attribute
+            '--history="' . $this->database . '|' . pathinfo($this->filename, PATHINFO_FILENAME) . '"', // PERCONA_SCHEMA.xtrabackup_history
             '--slave-info',
-            '--safe-slave-backup',
+            '--safe-slave-backup', // https://docs.percona.com/percona-xtrabackup/8.0/make-backup-in-replication-env.html
             '--safe-slave-backup-timeout=300',
             '--check-privileges', // checks if Percona XtraBackup has all the required privileges.
-            '--target-dir=' . $target,
-            '--parallel=' . self::PROCESSORS,
-            '--compress=' . self::COMPRESS_ALGORITHM,
-            '--compress-threads=' . self::PROCESSORS,
-            '--rsync', // https://docs.percona.com/percona-xtrabackup/8.0/accelerate-backup-process.html
-            '2> ' . $logfile,
+            '--target-dir=./',
+            //'--compress=' . self::COMPRESS_ALGORITHM,
+            '--compress=lz4',
+            '--compress-threads=' . $this->processors,
+            '--parallel=' . $this->processors,
+            '> ' . $file,
+            '2> ' . $log,
         ];
 
         $cmd = 'docker exec ' . $this->xtrabackupContainerId . ' ' . implode(' ', $args);
-        self::log($cmd);
+        Console::success($cmd);
         shell_exec($cmd);
 
-        $stderr = shell_exec('tail -1 ' . $logfile);
-        self::log($stderr);
+        $stderr = shell_exec('tail -1 ' . $log);
 
-        if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
-            Console::error('Backup failed');
-            Console::exit();
+        if (!str_contains($stderr, 'completed OK!')) {
+            unlink($file);
+            throw new Exception('Backup failed: ' . $stderr);
         }
 
-        if (!unlink($logfile)) {
-            Console::error('Error deleting: ' . $logfile);
-            Console::exit();
+        if (!unlink($log)) {
+            throw new Exception('Error deleting: ' . $log);
         }
+
+        // tar if using lz4...
+//        $tar = basename($this->filename, '.xbstream') . '.tar.gz';
+//        $cmd = 'cd ' . $target . ' && tar zcf ' . $tar . ' ' . $this->filename . ' && cd ' . getcwd();
+//        console::success($cmd);
+//        shell_exec($cmd);
+
     }
 
-    public function tar(string $directory, string $file)
+    /**
+     * @throws Exception
+     */
+    public function upload(Device $local)
     {
-        $stdout = '';
-        $stderr = '';
+        self::log('Upload start');
 
-        $cmd = 'cd ' . $directory . ' && tar zcf ' . $file . ' . && cd ' . getcwd();
-        self::log($cmd);
-        Console::execute($cmd, '', $stdout, $stderr);
-        self::log($stdout);
-        if (!empty($stderr)) {
-            Console::error($stderr);
-            Console::exit();
-        }
-
-        if (!file_exists($file)) {
-            Console::error('Can\'t find tar file: ' . $file);
-            Console::exit();
-        }
-
-        $filesize = \filesize($file);
-        self::log('Tar file size is: ' . ceil($filesize / 1024 / 1024) . 'MB');
-        if ($filesize < (2 * 1024 * 1024)) {
-            Console::error('File size is very small: ' . $file);
-            Console::exit();
-        }
-    }
-
-    public function upload(string $file, Device $local)
-    {
-        $filename = basename($file);
+        $file = $local->getPath($this->filename);
 
         if (!$this->s3->exists('/')) {
-            Console::error('Can\'t read s3 root directory');
-            Console::exit();
+            throw new Exception('Can\'t read s3 root directory');
         }
 
+        $destination = $this->s3->getRoot() . '/' . $this->filename;
+
+        if (!$local->transfer($file, $destination, $this->s3)) {
+            throw new Exception('Error uploading to ' . $destination);
+        }
+
+        if (!$this->s3->exists($destination)) {
+            throw new Exception('File not found in destination: ' . $destination);
+        }
+    }
+
+    public function cleanLocalFiles(Device $local)
+    {
+        self::log('cleanLocalFiles start');
+
+        $folder = scandir($local->getRoot());
+        if ($folder !== false) {
+            foreach ($folder as $item) {
+                if ($this->isDelete($item, self::CLEANUP_LOCAL_FILES_SECONDS)) {
+                    if (str_ends_with($item, '.xbstream.log')) {
+                        $delete = true;
+                    } else {
+                        // Check if file exist on cloud before delete
+                        if ($this->s3->exists($this->s3->getRoot() . '/' . $item)) {
+                            $delete = true;
+                        } else {
+                            if ($this->isDelete($item, self::CLEANUP_CLOUD_FILES_SECONDS)) {
+                                $delete = true;
+                            } else {
+                                Console::warning('Skipping delete not found on cloud: ' . $local->getPath($item) . ' ');
+                                $delete = false;
+                            }
+                        }
+                    }
+
+                    if ($delete === true) {
+                        if (unlink($local->getPath($item))) {
+                            Console::success($local->getPath($item) . ' Deleted!');
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function cleanCloudFiles(): void
+    {
+        self::log('cleanCloudFiles start');
+
+        $files = $this->s3->getFiles($this->s3->getRoot());
+
+        if ($files['KeyCount'] > 0) {
+            foreach ($files['Contents'] as $file) {
+                if ($this->isDelete(basename($file['Key']), self::CLEANUP_CLOUD_FILES_SECONDS)) {
+                    if ($this->s3->delete($file['Key'])) {
+                        Console::success($file['Key'] . ' Deleted!');
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * @param string $item
+     * @param int $seconds
+     * @return bool
+     */
+    public function isDelete(string $item, int $seconds): bool
+    {
+        if (str_ends_with($item, '.xbstream')) {
+            $item = substr($item, 0, -9);
+        } elseif (str_ends_with($item, '.xbstream.log')) {
+            $item = substr($item, 0, -13);
+        } else {
+            return false;
+        }
+
+        [$year, $month, $day, $hour, $minute, $second] = explode('_', $item);
+        $date = $year . '-' . $month . '-' . $day . ' ' . $hour . ':' . $minute . ':' . $second;
+
         try {
-            self::log('Uploading: ' . $file);
-            $destination = $this->s3->getRoot() . '/' . $filename;
-
-            if (!$local->transfer($file, $destination, $this->s3)) {
-                Console::error('Error uploading to ' . $destination);
-                Console::exit();
-            }
-
-            if (!$this->s3->exists($destination)) {
-                Console::error('File not found in destination: ' . $destination);
-                Console::exit();
+            $now = new \DateTime();
+            $backupDate = new \DateTime($date);
+            if (($now->getTimestamp() - $backupDate->getTimestamp()) > $seconds) {
+                return true;
             }
         } catch (Exception $e) {
-            Console::error($e->getMessage());
-            Console::exit();
         }
 
-        if (!unlink($file)) {
-            Console::error('Error deleting: ' . $file);
-            Console::exit();
-        }
+        return false;
     }
 
     public static function log(string $message): void
@@ -244,6 +316,9 @@ class Backup extends Action
         }
     }
 
+    /**
+     * @throws Exception
+     */
     public function checkEnvVariables(): void
     {
         foreach (
@@ -253,15 +328,14 @@ class Backup extends Action
             ] as $env
         ) {
             if (empty(App::getEnv($env))) {
-                Console::error('Can\'t read ' . $env);
-                Console::exit();
+                throw new Exception('Can\'t read ' . $env);
             }
         }
     }
 
     public function getDsn(string $database): ?DSN
     {
-        foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_REPLICAS')) as $project) {
+        foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_REPLICAS', '')) as $project) {
             [$db, $dsn] = explode('=', $project);
             if ($db === $database) {
                 return new DSN($dsn);
@@ -269,4 +343,46 @@ class Backup extends Action
         }
         return null;
     }
+
+    /**
+     * @throws Exception
+     */
+    public function setContainerId()
+    {
+        $stdout = '';
+        $stderr = '';
+        Console::execute('docker ps -f "name=xtrabackup" --format "{{.ID}}"', '', $stdout, $stderr);
+        if (!empty($stderr)) {
+            throw new Exception('Error setting container Id: ' . $stderr);
+        }
+
+        $containerId = str_replace(PHP_EOL, '', $stdout);
+        if (empty($containerId)) {
+            throw new Exception('Xtrabackup Container ID not found');
+        }
+
+        $this->xtrabackupContainerId = $containerId;
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function setProcessors()
+    {
+        $stdout = '';
+        $stderr = '';
+        Console::execute('docker exec ' . $this->xtrabackupContainerId . ' nproc', '', $stdout, $stderr);
+        if (!empty($stderr)) {
+            throw new Exception('Error setting processors: ' . $stderr);
+        }
+
+        $processors = str_replace(PHP_EOL, '', $stdout);
+        $processors = intval($processors);
+
+        if ($processors === 0) {
+            throw new Exception('Set Processors Error');
+        }
+
+        $this->processors = $processors;
+    }
 }

src/Appwrite/Platform/Tasks/Restore.php

@@ -10,36 +10,23 @@ use Utopia\CLI\Console;
 use Utopia\Storage\Device;
 use Utopia\Storage\Device\DOSpaces;
 use Utopia\Storage\Device\Local;
+use Utopia\Validator\Boolean;
 use Utopia\Validator\Text;
-use Utopia\Validator\WhiteList;
 
 class Restore extends Action
 {
     public const BACKUPS_PATH = '/backups';
     public const DATADIR = '/var/lib/mysql';
-    public const PROCESSORS = 4;
-    protected ?DSN $dsn = null;
-    protected string $database;
     protected ?DOSpaces $s3 = null;
     protected string $xtrabackupContainerId;
+    protected int $processors = 1;
 
     public function __construct()
     {
-        $this->checkEnvVariables();
-
-        $containerId = shell_exec('docker ps -aqf "name=xtrabackup"');
-        $containerId = str_replace(PHP_EOL, '', $containerId);
-        if (empty($containerId)) {
-            Console::error('Xtrabackup Container ID not found');
-            Console::exit();
-        }
-
-        $this->xtrabackupContainerId = $containerId;
-
         $this
             ->desc('Restore a DB')
-            ->param('id', '', new Text(20), 'The backup identification')
-            ->param('cloud', null, new WhiteList(['true', 'false'], true), 'Download backup from cloud or use local directory')
+            ->param('id', '', new Text(19), 'The backup identification')
+            ->param('cloud', null, new Boolean(true), 'Download backup from cloud or use local directory')
             ->param('database', null, new Text(10), 'The Database name for example db_fra1_01')
             ->callback(fn ($id, $cloud, $project) => $this->action($id, $cloud, $project));
     }
@@ -51,17 +38,16 @@ class Restore extends Action
 
     public function action(string $id, string $cloud, string $database): void
     {
-        $this->database = $database;
+        $this->checkEnvVariables();
+        $this->setContainerId();
+        $this->setProcessors();
+
         $datadir = self::DATADIR;
-        $this->dsn = $this->getDsn($database);
-        if (is_null($this->dsn)) {
-            Console::error('No DSN match');
-            Console::exit();
-        }
 
         try {
-            $dsn = new DSN(App::getEnv('_APP_CONNECTIONS_BACKUPS_STORAGE'));
+            $dsn = new DSN(App::getEnv('_APP_CONNECTIONS_BACKUPS_STORAGE', ''));
             $this->s3 = new DOSpaces('/' . $database . '/full', $dsn->getUser(), $dsn->getPassword(), $dsn->getPath(), $dsn->getParam('region'));
+            $this->s3->setTransferChunkSize(20 * 1024 * 1024); // 5MB is the minimum
         } catch (\Exception $e) {
             Console::error($e->getMessage() . 'Invalid DSN.');
             Console::exit();
@@ -79,40 +65,45 @@ class Restore extends Action
 
         $this->log('--- Restore Start ' . $id . ' --- ');
 
-        $filename = $id . '.tar.gz';
         $start = microtime(true);
-        $cloud = $cloud === 'true';
+        $filename = $id . '.xbstream';
+        $cloud = $cloud === 'true' || $cloud === '1';
+        $destination = new Local(self::BACKUPS_PATH . '/restore/' . $id);
 
-        if ($cloud) {
-            $local = new Local(self::BACKUPS_PATH . '/downloads/' . $id);
-
-            $files = $local->getRoot() . '/files';
-
-            if (!file_exists($files) && !mkdir($files, 0755, true)) {
-                Console::error('Error creating directory: ' . $files);
-                Console::exit();
-            }
-
-            $file = $local->getPath($filename);
-
-            if (!file_exists($file)) {
-                $this->download($file, $local);
-            }
-
-            $this->untar($file, $files);
-        } else {
-            $local = new Local(self::BACKUPS_PATH . '/' . $database . '/full/' . $id);
-            $files = $local->getRoot() . '/files';
-        }
-
-        if (!file_exists($files)) {
-            Console::error('Directory not found: ' . $files);
+        if (!file_exists($destination->getRoot()) && !mkdir($destination->getRoot(), 0755, true)) {
+            Console::error('Error creating directory: ' . $destination->getRoot());
             Console::exit();
         }
 
-        $this->decompress($files);
+        if ($cloud) {
+            $stream = $destination->getPath($filename);
+            if (!file_exists($stream)) {
+                $this->download($stream, $destination);
+            }
+        } else {
+            $stream = self::BACKUPS_PATH . '/' . $database . '/full/' . $filename;
+        }
+
+        if (!file_exists($stream)) {
+            Console::error('File not found: ' . $stream);
+            Console::exit();
+        }
+
+        $files = $destination->getRoot() . '/files';
+
+        if (file_exists($files)) {
+            Console::error('Directory exist: ' . $files);
+            Console::exit();
+        }
+
+        if (!file_exists($files) && !mkdir($files, 0755, true)) {
+            Console::error('Error creating directory: ' . $files);
+            Console::exit();
+        }
+
+        $this->decompress($stream, $files);
         $this->prepare($files);
-        $this->restore($files, $cloud, $datadir);
+        $this->restore($files, $datadir);
 
         $this->log('Restore Finish in ' . (microtime(true) - $start) . ' seconds');
     }
@@ -120,19 +111,24 @@ class Restore extends Action
     public function download(string $file, Device $local)
     {
         $filename = basename($file);
+
+        $tmp = $local->getRoot() . '/tmp_' . $filename;
+        if (file_exists($tmp)) { // todo: this should be taken care of in the lib, cleanup previous iteration
+            Console::error('Deleting: ' . $tmp);
+            $local->delete($tmp, true);
+        }
+
+        $this->log('Download start');
+
+        $path = $this->s3->getPath($filename);
+        if (!$this->s3->exists($path)) {
+            Console::error('File: ' . $path . ' does not exist on cloud');
+            Console::exit();
+        }
+
         try {
-            $path = $this->s3->getPath($filename);
-
-            if (!$this->s3->exists($path)) {
-                Console::error('File: ' . $path . ' does not exist on cloud');
-                Console::exit();
-            }
-
-            $this->log('Downloading: ' . $file);
-
             if (!$this->s3->transfer($path, $file, $local)) {
-                Console::error('Error Downloading ' . $file);
-                Console::exit();
+                throw new Exception('Transfer failed ' . $file);
             }
         } catch (Exception $e) {
             Console::error($e->getMessage());
@@ -140,119 +136,87 @@ class Restore extends Action
         }
     }
 
-    public function untar(string $file, string $directory)
+    public function decompress(string $file, string $target)
     {
-        $stdout = '';
-        $stderr = '';
-        $cmd = 'tar -xzf ' . $file . ' -C ' . $directory;
-        $this->log($cmd);
-        Console::execute($cmd, '', $stdout, $stderr);
-        if (!empty($stderr)) {
-            Console::error($stderr);
-            Console::exit();
-        }
-
-        if (!file_exists($file)) {
-            Console::error('Restore file not found: ' . $file);
-            Console::exit();
-        }
-    }
-
-    public function decompress(string $target)
-    {
-        $logfile = $target . '/../log.txt';
+        $this->log('Xbstream decompress start');
 
         $args = [
-            'xtrabackup',
-            '--user=' . $this->dsn->getUser(),
-            '--password=' . $this->dsn->getPassword(),
-            '--host=' . $this->dsn->getHost(),
+            'xbstream -x < ' . $file,
             '--decompress',
-            '--strict',
-            '--remove-original', // Removes *.lz4 compressed files
-            '--parallel=' . self::PROCESSORS,
-            '--compress-threads=' . self::PROCESSORS,
-            '--target-dir=' . $target,
-            '2> ' . $logfile,
+            '--decompress-threads=' . intval($this->processors / 2),
+            '--parallel=' . $this->processors,
+            '-C ' . $target,
         ];
 
-        $cmd = 'docker exec ' . $this->xtrabackupContainerId . ' ' . implode(' ', $args);
-        $this->log($cmd);
-        shell_exec($cmd);
-
-        $stderr = shell_exec('tail -1 ' . $logfile);
-        $this->log($stderr);
-
-        if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
-            Console::error('Decompress failed');
+        $stdout = '';
+        $stderr = '';
+        $cmd = 'docker exec -i ' . $this->xtrabackupContainerId . ' ' . implode(' ', $args);
+        Console::execute($cmd, '', $stdout, $stderr);
+        if (!empty($stderr)) {
+            Console::error('Xbstream Error: ' . $stderr);
             Console::exit();
         }
     }
 
     public function prepare(string $target)
     {
+        $this->log('Prepare start');
+
         if (!file_exists($target)) {
             Console::error('prepare error directory not found: ' . $target);
             Console::exit();
         }
 
-        $logfile = $target . '/../log.txt';
+        $logfile = $target . '/../prepare.txt';
 
         $args = [
             'xtrabackup',
-            '--user=' . $this->dsn->getUser(),
-            '--password=' . $this->dsn->getPassword(),
-            '--host=' . $this->dsn->getHost(),
             '--prepare',
+            '--parallel=' . $this->processors,
             '--strict',
             '--target-dir=' . $target,
             '2> ' . $logfile,
         ];
 
         $cmd = 'docker exec ' . $this->xtrabackupContainerId . ' ' . implode(' ', $args);
-        $this->log($cmd);
         shell_exec($cmd);
 
         $stderr = shell_exec('tail -1 ' . $logfile);
-        $this->log($stderr);
 
-        if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
-            Console::error('Prepare failed');
+        if (!str_contains($stderr, 'completed OK!')) {
+            Console::error('Prepare error: ' . $stderr);
             Console::exit();
         }
     }
 
-    public function restore(string $target, bool $cloud, string $datadir)
+    public function restore(string $target, string $datadir)
     {
+        $this->log('Restore start');
+
         if (!file_exists($target)) {
             Console::error('restore error directory not found: ' . $target);
             Console::exit();
         }
 
-        $logfile = $target . '/../log.txt';
+        $logfile = $target . '/../restore.txt';
 
         $args = [
             'xtrabackup',
-            '--user=' . $this->dsn->getUser(),
-            '--password=' . $this->dsn->getPassword(),
-            '--host=' . $this->dsn->getHost(),
-            $cloud ? '--move-back' : '--copy-back',
+            '--move-back',
             '--strict',
+            '--parallel=' . $this->processors,
             '--target-dir=' . $target,
             '--datadir=' . $datadir,
-            '--parallel=' . self::PROCESSORS,
             '2> ' . $logfile,
         ];
 
         $cmd = 'docker exec ' . $this->xtrabackupContainerId . ' ' . implode(' ', $args);
-        $this->log($cmd);
         shell_exec($cmd);
 
         $stderr = shell_exec('tail -1 ' . $logfile);
-        $this->log($stderr);
 
-        if (!str_contains($stderr, 'completed OK!') || !file_exists($target . '/xtrabackup_checkpoints')) {
-            Console::error('Restore failed');
+        if (!str_contains($stderr, 'completed OK!')) {
+            Console::error(date('Y-m-d H:i:s') . ' Restore failed: ' . $stderr);
             Console::exit();
         }
     }
@@ -279,14 +243,43 @@ class Restore extends Action
         }
     }
 
-    public function getDsn(string $database): ?DSN
+    public function setContainerId()
     {
-        foreach (explode(',', App::getEnv('_APP_CONNECTIONS_DB_REPLICAS')) as $project) {
-            [$db, $dsn] = explode('=', $project);
-            if ($db === $database) {
-                return new DSN($dsn);
-            }
+        $stdout = '';
+        $stderr = '';
+        Console::execute('docker ps -f "name=xtrabackup" --format "{{.ID}}"', '', $stdout, $stderr);
+        if (!empty($stderr)) {
+            Console::error('Error setting container Id: ' . $stderr);
+            Console::exit();
         }
-        return null;
+
+        $containerId = str_replace(PHP_EOL, '', $stdout);
+        if (empty($containerId)) {
+            Console::error('Xtrabackup Container ID not found');
+            Console::exit();
+        }
+
+        $this->xtrabackupContainerId = $containerId;
+    }
+
+    public function setProcessors()
+    {
+        $stdout = '';
+        $stderr = '';
+        Console::execute('docker exec ' . $this->xtrabackupContainerId . ' nproc', '', $stdout, $stderr);
+        if (!empty($stderr)) {
+            Console::error('Error setting processors: ' . $stderr);
+            Console::exit();
+        }
+
+        $processors = str_replace(PHP_EOL, '', $stdout);
+        $processors = intval($processors);
+
+        if ($processors === 0) {
+            Console::error('Set Processors Error');
+            Console::exit();
+        }
+
+        $this->processors = $processors;
     }
 }

tests/e2e/Services/Databases/DatabasesBase.php

@@ -128,8 +128,7 @@ trait DatabasesBase
         $this->assertEquals(200, $response['headers']['status-code']);
         $this->assertTrue($response['body']['enabled']);
     }
-
-
+    
     /**
      * @depends testCreateCollection
      */