diff --git a/.env b/.env index ea666a2acc..fea44f3dde 100644 --- a/.env +++ b/.env @@ -32,14 +32,10 @@ _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appw _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379 -_APP_CONNECTIONS_STORAGE=local://localhost +_APP_CONNECTIONS_STORAGE=DOSpaces://DO006K9PPED99XTQ7FJ9:DkQQ5zBU1FvrjeOXQtT9bANUGjdYHS0BFHegI5Nphcc@null:0/utopia-storage-tests?region=nyc3 _APP_STORAGE_ANTIVIRUS=disabled _APP_STORAGE_ANTIVIRUS_HOST=clamav _APP_STORAGE_ANTIVIRUS_PORT=3310 -_APP_INFLUXDB_HOST=influxdb -_APP_INFLUXDB_PORT=8086 -_APP_STATSD_HOST=telegraf -_APP_STATSD_PORT=8125 _APP_SMTP_HOST=maildev _APP_SMTP_PORT=1025 _APP_SMTP_SECURE= diff --git a/Dockerfile b/Dockerfile index bfc7ac600b..7bcd5f9726 100755 --- a/Dockerfile +++ b/Dockerfile @@ -133,8 +133,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-messaging && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-usage && \ - chmod +x /usr/local/bin/folder-backup && \ - chmod +x /usr/local/bin/folder-restore + chmod +x /usr/local/bin/backup && \ + chmod +x /usr/local/bin/restore # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/bin/backup b/bin/backup new file mode 100644 index 0000000000..97d6dfb156 --- /dev/null +++ b/bin/backup @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php backup $@ \ No newline at end of file diff --git a/bin/folder-backup b/bin/folder-backup deleted file mode 100644 index 372b6a1139..0000000000 --- a/bin/folder-backup +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php folder-backup $@ \ No newline at end of file diff --git a/bin/folder-restore b/bin/folder-restore deleted file mode 100644 index d66367b77a..0000000000 --- a/bin/folder-restore +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php folder-restore $@ \ No newline at end of file diff --git a/bin/restore b/bin/restore new file mode 100644 index 0000000000..6935f47778 --- /dev/null +++ b/bin/restore @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php restore $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 7462ea5b56..1ee79cfd06 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -581,10 +581,10 @@ services: - _APP_MAINTENANCE_RETENTION_AUDIT - _APP_MAINTENANCE_RETENTION_SCHEDULES - appwrite-folder-backup: - entrypoint: folder-backup + appwrite-backup: + entrypoint: backup <<: *x-logging - container_name: appwrite-folder-backup + container_name: appwrite-backup image: appwrite-dev networks: - appwrite @@ -595,7 +595,6 @@ services: - ./vendor/utopia-php/cli:/usr/src/code/vendor/utopia-php/cli environment: - _APP_ENV - - _APP_MAINTENANCE_INTERVAL - _APP_CONNECTIONS_STORAGE appwrite-worker-usage: diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index a65b97f646..8b16edc76e 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -15,8 +15,8 @@ use Appwrite\Platform\Tasks\SSL; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; use Appwrite\Platform\Tasks\VolumeSync; -use Appwrite\Platform\Tasks\FolderBackup; -use Appwrite\Platform\Tasks\FolderRestore; +use Appwrite\Platform\Tasks\Backup; +use Appwrite\Platform\Tasks\Restore; class Tasks extends Service { @@ -36,7 +36,7 @@ class Tasks extends Service ->addAction(SDKs::getName(), new SDKs()) ->addAction(VolumeSync::getName(), new VolumeSync()) ->addAction(Specs::getName(), new Specs()) - ->addAction(FolderBackup::getName(), new FolderBackup()) - ->addAction(FolderRestore::getName(), new FolderRestore()); + ->addAction(Backup::getName(), new Backup()) + ->addAction(Restore::getName(), new Restore()); } } diff --git a/src/Appwrite/Platform/Tasks/FolderBackup.php b/src/Appwrite/Platform/Tasks/Backup.php similarity index 74% rename from src/Appwrite/Platform/Tasks/FolderBackup.php rename to src/Appwrite/Platform/Tasks/Backup.php index bbce580d28..be805d218a 100644 --- a/src/Appwrite/Platform/Tasks/FolderBackup.php +++ b/src/Appwrite/Platform/Tasks/Backup.php @@ -3,57 +3,58 @@ namespace Appwrite\Platform\Tasks; use Utopia\CLI\Console; -use Utopia\Database\DateTime; use Utopia\Platform\Action; use Utopia\Storage\Device\Local; -class FolderBackup extends Action +class Backup extends Action { public static function getName(): string { - return 'folder-backup'; + return 'backup'; } public function __construct() { $this - ->desc('Folder backup process') + ->desc('Backup process') ->callback(fn() => $this->action()); } public function action(): void { - Console::title('Folder backup V1'); - Console::success(APP_NAME . ' folder backup process v1 has started'); + Console::title('Backup V1'); + Console::success(APP_NAME . ' backup process v1 has started'); gc_enable(); $time = 0; while (!connection_aborted() || PHP_SAPI == "cli") { $now = new \DateTime(); $now->setTimezone(new \DateTimeZone(date_default_timezone_get())); - $next = new \DateTime($now->format("Y-m-d 16:35.0")); + $next = new \DateTime($now->format("Y-m-d 11:20.0")); $next->setTimezone(new \DateTimeZone(date_default_timezone_get())); - $sleep = $next->getTimestamp() - $now->getTimestamp(); + $sleep = $next->getTimestamp() - $now->getTimestamp(); /** * If time passed the target time */ if ($sleep <= 0) { $next->add(\DateInterval::createFromDateString('1 days')); - $sleep = $next->getTimestamp() - $now->getTimestamp(); + $sleep = $next->getTimestamp() - $now->getTimestamp(); } console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Sleeping for ' . $sleep . ' seconds next run will be at [' . $next->format("Y-m-d H:i:s.v") . ']'); sleep($sleep); + $total = 0; + $start = \microtime(true); $folders = [ 'cert' => APP_STORAGE_CERTIFICATES, 'config' => APP_STORAGE_CONFIG, ]; - $remote = getDevice('/'); + $remote = getDevice('/'); foreach ($folders as $key => $folder) { $local = new Local($folder); @@ -61,12 +62,13 @@ class FolderBackup extends Action $source = $local->getRoot() . '/' . $filename; $destination = '/' . $key . '/' . $filename; -// for ($i = 0; $i < 1000; $i++) { -// file_put_contents($local->getRoot() . '/' . $i . '.txt', ''); -// } + for ($i = 0; $i < 1000; $i++) { + file_put_contents($local->getRoot() . '/' . $i . '.txt', ''); + } $stdout = ''; $stderr = ''; + Console::execute( 'cd ' . $folder . ' && tar --exclude ' . $filename . ' -zcf ' . $source . '*', '', @@ -75,11 +77,15 @@ class FolderBackup extends Action ); try { + if (!$local->exists($source)) { + continue; + } + $local->transfer($source, $destination, $remote); - console::info("backing up local $source to {$remote->getName()} $destination"); + Console::info("Backing up local $source to {$remote->getName()} $destination"); /** - * Clean up + * Remote storage old backup files clean-up */ $now = new \DateTime(); $now->setTimezone(new \DateTimeZone(date_default_timezone_get())); @@ -87,7 +93,7 @@ class FolderBackup extends Action $now->sub(\DateInterval::createFromDateString('1 days')); if ($i >= 10) { $destination = '/' . $key . '/' . $key . '-' . $now->format("Y-m-d") . '.tar.gz'; - console::info("Trying to delete from {$remote->getName()} $destination"); + Console::info("Trying to delete from {$remote->getName()} $destination"); $remote->delete($destination); } } @@ -102,6 +108,7 @@ class FolderBackup extends Action } } } + $total = (microtime(true) - $start); } } } diff --git a/src/Appwrite/Platform/Tasks/FolderRestore.php b/src/Appwrite/Platform/Tasks/Restore.php similarity index 93% rename from src/Appwrite/Platform/Tasks/FolderRestore.php rename to src/Appwrite/Platform/Tasks/Restore.php index a022b5da37..73c8a36839 100644 --- a/src/Appwrite/Platform/Tasks/FolderRestore.php +++ b/src/Appwrite/Platform/Tasks/Restore.php @@ -6,17 +6,17 @@ use Utopia\CLI\Console; use Utopia\Platform\Action; use Utopia\Storage\Device\Local; -class FolderRestore extends Action +class Restore extends Action { public static function getName(): string { - return 'folder-restore'; + return 'restore'; } public function __construct() { $this - ->desc('Folder restore process') + ->desc('Restore process') ->callback(fn() => $this->action()); }