folder backup

Co-authored-by: Eldad A. Fux <eldad.fux@gmail.com>

.env

@@ -32,7 +32,7 @@ _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appw
 _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
 _APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379
 _APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379
-_APP_CONNECTIONS_STORAGE=local://localhost
+_APP_CONNECTIONS_STORAGE=
 _APP_STORAGE_ANTIVIRUS=disabled
 _APP_STORAGE_ANTIVIRUS_HOST=clamav
 _APP_STORAGE_ANTIVIRUS_PORT=3310
@@ -66,10 +66,8 @@ _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
 _APP_USAGE_STATS=enabled
 _APP_USAGE_AGGREGATION_INTERVAL=30
 _APP_LOGGING_PROVIDER=
-_APP_LOGGING_CONFIG=
 _APP_GRAPHQL_MAX_BATCH_SIZE=10
 _APP_GRAPHQL_MAX_COMPLEXITY=250
 _APP_GRAPHQL_MAX_DEPTH=3
 _APP_REGION=default
-DOCKERHUB_PULL_USERNAME
-DOCKERHUB_PULL_PASSWORD
+_APP_BACKUP_HEARTBEAT=

CHANGES.md

@@ -33,7 +33,7 @@
 - Fix max mimetype size [#4814](https://github.com/appwrite/appwrite/pull/4814)
 - New usage metrics collection flow [#4770](https://github.com/appwrite/appwrite/pull/4770)
   - Deprecated influxdb, telegraf containers and removed all of their occurrences from the code.
-  - Removed _APP_INFLUXDB_HOST, _APP_INFLUXDB_PORT, _APP_STATSD_HOST, _APP_STATSD_PORT env variables.
+  - Removed _APP_INFLUXDB_HOST, _APP_INFLUXDB_PORT, _APP_STATSD_HOST, _APP_STATSD_PORT, _APP_USAGE_AGGREGATION_INTERVAL env variables.
   - Removed usage labels dependency.
   - Dropped type attribute from stats collection.
   - Usage metrics are processed via new usage worker.

Dockerfile

@@ -132,7 +132,9 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/worker-mails && \
     chmod +x /usr/local/bin/worker-messaging && \
     chmod +x /usr/local/bin/worker-webhooks && \
-    chmod +x /usr/local/bin/worker-usage
+    chmod +x /usr/local/bin/worker-usage && \
+    chmod +x /usr/local/bin/backup && \
+    chmod +x /usr/local/bin/restore
 
 # Letsencrypt Permissions
 RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

app/cli.php

@@ -7,6 +7,7 @@ use Appwrite\Event\Func;
 use Appwrite\Platform\Appwrite;
 use Utopia\CLI\CLI;
 use Utopia\Database\Validator\Authorization;
+use Utopia\DSN\DSN;
 use Utopia\Platform\Service;
 use Utopia\App;
 use Utopia\CLI\Console;
@@ -18,6 +19,14 @@ use Utopia\Database\Document;
 use Utopia\Logger\Log;
 use Utopia\Pools\Group;
 use Utopia\Registry\Registry;
+use Utopia\Storage\Device;
+use Utopia\Storage\Device\Backblaze;
+use Utopia\Storage\Device\DOSpaces;
+use Utopia\Storage\Device\Linode;
+use Utopia\Storage\Device\Local;
+use Utopia\Storage\Device\S3;
+use Utopia\Storage\Device\Wasabi;
+use Utopia\Storage\Storage;
 
 Authorization::disable();
 
@@ -121,6 +130,22 @@ CLI::setResource('queueForFunctions', function (Group $pools) {
     return new Func($pools->get('queue')->pop()->getResource());
 }, ['pools']);
 
+CLI::setResource('deviceLocal', function () {
+    return new Local();
+});
+
+CLI::setResource('deviceFiles', function ($project) {
+    return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId());
+}, ['project']);
+
+CLI::setResource('deviceFunctions', function ($project) {
+    return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId());
+}, ['project']);
+
+CLI::setResource('deviceBuilds', function ($project) {
+    return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId());
+}, ['project']);
+
 CLI::setResource('logError', function (Registry $register) {
     return function (Throwable $error, string $namespace, string $action) use ($register) {
         $logger = $register->get('logger');

bin/backup

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php backup $@

bin/restore

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php restore $@

composer.json

@@ -47,7 +47,7 @@
         "utopia-php/analytics": "0.2.*",
         "utopia-php/audit": "0.20.*",
         "utopia-php/cache": "0.8.*",
-        "utopia-php/cli": "0.14.*",
+        "utopia-php/cli": "0.15.*",
         "utopia-php/config": "0.2.*",
         "utopia-php/database": "0.30.*",
         "utopia-php/queue": "0.5.*",
@@ -63,7 +63,7 @@
         "utopia-php/logger": "0.3.*",
         "utopia-php/messaging": "0.1.*",
         "utopia-php/registry": "0.5.*",
-        "utopia-php/storage": "0.13.*",
+        "utopia-php/storage": "dev-feat-transfer as 0.13.2",
         "utopia-php/swoole": "0.5.*",
         "utopia-php/websocket": "0.1.*",
         "resque/php-resque": "1.3.6",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "60f1cefcf6d210ed231acb4a99f2f20c",
+    "content-hash": "d82e08efceeebb37f8ed6b75972af70a",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -2168,16 +2168,16 @@
         },
         {
             "name": "utopia-php/storage",
-            "version": "0.13.2",
+            "version": "dev-feat-transfer",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/storage.git",
-                "reference": "ad1c00f24ca56e73888acc2af3deee4919b1194b"
+                "reference": "00b26ef3d04b04f2df64a64636f6b81f8ae1a166"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/storage/zipball/ad1c00f24ca56e73888acc2af3deee4919b1194b",
-                "reference": "ad1c00f24ca56e73888acc2af3deee4919b1194b",
+                "url": "https://api.github.com/repos/utopia-php/storage/zipball/00b26ef3d04b04f2df64a64636f6b81f8ae1a166",
+                "reference": "00b26ef3d04b04f2df64a64636f6b81f8ae1a166",
                 "shasum": ""
             },
             "require": {
@@ -2217,9 +2217,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/storage/issues",
-                "source": "https://github.com/utopia-php/storage/tree/0.13.2"
+                "source": "https://github.com/utopia-php/storage/tree/feat-transfer"
             },
-            "time": "2022-12-20T11:11:35+00:00"
+            "time": "2022-11-02T07:42:34+00:00"
         },
         {
             "name": "utopia-php/swoole",
@@ -4604,16 +4604,16 @@
         },
         {
             "name": "squizlabs/php_codesniffer",
-            "version": "3.7.1",
+            "version": "3.7.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/squizlabs/PHP_CodeSniffer.git",
-                "reference": "1359e176e9307e906dc3d890bcc9603ff6d90619"
+                "reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/1359e176e9307e906dc3d890bcc9603ff6d90619",
-                "reference": "1359e176e9307e906dc3d890bcc9603ff6d90619",
+                "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/ed8e00df0a83aa96acf703f8c2979ff33341f879",
+                "reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879",
                 "shasum": ""
             },
             "require": {
@@ -4649,14 +4649,15 @@
             "homepage": "https://github.com/squizlabs/PHP_CodeSniffer",
             "keywords": [
                 "phpcs",
-                "standards"
+                "standards",
+                "static analysis"
             ],
             "support": {
                 "issues": "https://github.com/squizlabs/PHP_CodeSniffer/issues",
                 "source": "https://github.com/squizlabs/PHP_CodeSniffer",
                 "wiki": "https://github.com/squizlabs/PHP_CodeSniffer/wiki"
             },
-            "time": "2022-06-18T07:21:10+00:00"
+            "time": "2023-02-22T23:07:41+00:00"
         },
         {
             "name": "swoole/ide-helper",
@@ -5041,9 +5042,18 @@
             "time": "2023-02-08T07:49:20+00:00"
         }
     ],
-    "aliases": [],
+    "aliases": [
+        {
+            "package": "utopia-php/storage",
+            "version": "dev-feat-transfer",
+            "alias": "0.13.2",
+            "alias_normalized": "0.13.2.0"
+        }
+    ],
     "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {
+        "utopia-php/storage": 20
+    },
     "prefer-stable": false,
     "prefer-lowest": false,
     "platform": {
@@ -5067,5 +5077,5 @@
     "platform-overrides": {
         "php": "8.0"
     },
-    "plugin-api-version": "2.3.0"
+    "plugin-api-version": "2.2.0"
 }

docker-compose.yml

@@ -85,6 +85,7 @@ services:
       - ./public:/usr/src/code/public
       - ./src:/usr/src/code/src
       - ./dev:/usr/local/dev
+      - ./vendor/utopia-php/storage:/usr/src/code/vendor/utopia-php/storage
     depends_on:
       - mariadb
       - redis
@@ -581,6 +582,23 @@ services:
       - _APP_MAINTENANCE_RETENTION_AUDIT
       - _APP_MAINTENANCE_RETENTION_SCHEDULES
 
+  appwrite-backup:
+    entrypoint: backup
+    <<: *x-logging
+    container_name: appwrite-backup
+    image: appwrite-dev
+    networks:
+      - appwrite
+    volumes:
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+      - appwrite-certificates:/storage/certificates:rw
+      - ./vendor/utopia-php/cli:/usr/src/code/vendor/utopia-php/cli
+    environment:
+      - _APP_ENV
+      - _APP_CONNECTIONS_STORAGE
+      - _APP_BACKUP_HEARTBEAT
+
   appwrite-worker-usage:
     entrypoint: worker-usage
     <<: *x-logging

src/Appwrite/Platform/Services/Tasks.php

@@ -15,6 +15,8 @@ use Appwrite\Platform\Tasks\SSL;
 use Appwrite\Platform\Tasks\Vars;
 use Appwrite\Platform\Tasks\Version;
 use Appwrite\Platform\Tasks\VolumeSync;
+use Appwrite\Platform\Tasks\Backup;
+use Appwrite\Platform\Tasks\Restore;
 
 class Tasks extends Service
 {
@@ -33,6 +35,8 @@ class Tasks extends Service
             ->addAction(Migrate::getName(), new Migrate())
             ->addAction(SDKs::getName(), new SDKs())
             ->addAction(VolumeSync::getName(), new VolumeSync())
-            ->addAction(Specs::getName(), new Specs());
+            ->addAction(Specs::getName(), new Specs())
+            ->addAction(Backup::getName(), new Backup())
+            ->addAction(Restore::getName(), new Restore());
     }
 }

src/Appwrite/Platform/Tasks/Backup.php

@@ -0,0 +1,112 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Utopia\App;
+use Utopia\CLI\Console;
+use Utopia\Platform\Action;
+use Utopia\Storage\Device\Local;
+
+class Backup extends Action
+{
+    public static function getName(): string
+    {
+        return 'backup';
+    }
+
+    public function __construct()
+    {
+        $this
+            ->desc('Backup process')
+            ->callback(fn() => $this->action());
+    }
+
+    /**
+     * @throws \Exception
+     */
+    public function action(): void
+    {
+        Console::title('Backup V1');
+        Console::success(APP_NAME . ' backup process v1 has started');
+
+        $jobInitTime = '13:10'; // (hour:minutes)
+        $now = new \DateTime();
+        $now->setTimezone(new \DateTimeZone(date_default_timezone_get()));
+        $next = new \DateTime($now->format("Y-m-d $jobInitTime"));
+        $next->setTimezone(new \DateTimeZone(date_default_timezone_get()));
+        $delay = $next->getTimestamp() - $now->getTimestamp();
+
+        /**
+         * If time passed for the target day.
+         */
+        if ($delay <= 0) {
+            $next->add(\DateInterval::createFromDateString('1 days'));
+            $delay = $next->getTimestamp() - $now->getTimestamp();
+        }
+
+        $folders = [
+            'cert' => APP_STORAGE_CERTIFICATES,
+            'config' => APP_STORAGE_CONFIG,
+        ];
+
+        $sleep = 86400; //24 hours
+        $remote = getDevice('/');
+
+        Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']');
+        Console::loop(function () use ($delay, $sleep, $folders, $remote) {
+            $success = 0;
+            foreach ($folders as $key => $folder) {
+                $local = new Local($folder);
+                $filename = $key . '-' . date("Y-m-d") . '.tar.gz';
+                $source = $local->getRoot() . '/' . $filename;
+                $destination = '/' . $key . '/' . $filename;
+
+                $stdout = '';
+                $stderr = '';
+
+                Console::execute(
+                    'cd ' . $folder . ' && tar --exclude ' . $filename . ' -zcf ' . $source . ' *',
+                    '',
+                    $stdout,
+                    $stderr
+                );
+
+                try {
+                    if (!$local->exists($source)) {
+                        continue;
+                    }
+
+                    $local->transfer($source, $destination, $remote);
+
+                    if ($remote->exists($destination)) {
+                        $success++;
+                    }
+
+                    Console::info("Backing up local $source to {$remote->getName()} $destination");
+                    /**
+                     * Backup folder, long tail cleanup.
+                     */
+                    $now = new \DateTime();
+                    $now->setTimezone(new \DateTimeZone(date_default_timezone_get()));
+                    for ($i = 0; $i < 20; $i++) {
+                        $now->sub(\DateInterval::createFromDateString('1 days'));
+                        if ($i >= 10) {
+                            $destination = '/' . $key . '/' . $key . '-' . $now->format("Y-m-d") . '.tar.gz';
+                            $remote->delete($destination);
+                        }
+                    }
+                } catch (\Exception $e) {
+                    Console::error($e->getMessage());
+                }
+            }
+
+            /**
+             * heartbeat url
+             */
+            $url = App::getEnv('_APP_BACKUP_HEARTBEAT', '');
+            if ($success === count($folders) && $url !== '') {
+                file_get_contents($url);
+            }
+        }, $sleep, $delay);
+    }
+}

src/Appwrite/Platform/Tasks/Restore.php

@@ -0,0 +1,79 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Utopia\CLI\Console;
+use Utopia\Platform\Action;
+use Utopia\Storage\Device\Local;
+
+class Restore extends Action
+{
+    public static function getName(): string
+    {
+        return 'restore';
+    }
+
+    public function __construct()
+    {
+        $this
+            ->desc('Restore process')
+            ->callback(fn() => $this->action());
+    }
+
+    public function action(): void
+    {
+
+        $folders = [
+            'cert' => APP_STORAGE_CERTIFICATES,
+            'config' => APP_STORAGE_CONFIG,
+        ];
+
+        $folder = \strtolower(Console::confirm('Please enter the folder you wish to restore'));
+        $date   = Console::confirm('Please enter the date of the backup file in datetime format(Y-m-d)');
+
+        if (!array_key_exists($folder, $folders)) {
+            console::Error('Unknown folder given');
+            exit;
+        }
+
+        if (!is_a(\DateTime::createFromFormat("Y-m-d", $date), 'DateTime')) {
+            console::Error('Unknown date format');
+            exit;
+        }
+
+        $remote = getDevice('/');
+        $local  = new Local();
+        $filename = $folder . '-' . $date . '.tar.gz';
+        $destination = $folders[$folder] . '/' . $filename;
+        $source = '/' . $folder . '/' . $filename;
+
+        try {
+            if (!$remote->exists($source)) {
+                Console::error("Backup file not found for folder :$folder ,date: $date");
+            }
+
+            $result = $remote->transfer($source, $destination, $local);
+
+            if (!$result) {
+                Console::error("Error while trying to download back file");
+            }
+
+//            if (!$local->exists($destination)) {
+//                Console::error("Backup file not found for folder :$folder ,date $date");
+//            }
+
+            $stdout = '';
+            $stderr = '';
+            Console::execute(
+                'cd ' . $folders[$folder] . ' &&  tar -xf ' . $filename,
+                '',
+                $stdout,
+                $stderr
+            );
+
+            console::info("Restoring from {$remote->getName()}  $source to  $destination");
+        } catch (\Exception $e) {
+            Console::error($e->getMessage());
+        }
+    }
+}