Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Transfer big data

Browse Source
This commit is contained in:
fogelito
2023-07-23 19:05:35 +03:00
parent 38eb24d0ad
commit 970c35b31a
5 changed files with 96 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
composer.json
+1 -1
View File
@@ -63,7 +63,7 @@
"utopia-php/preloader": "0.2.*",
"utopia-php/queue": "0.5.*",
"utopia-php/registry": "0.5.*",
"utopia-php/storage": "0.13.*",
"utopia-php/storage": "dev-feat-transfer as 0.13.3",
"utopia-php/swoole": "0.5.*",
"utopia-php/websocket": "0.1.0",
"resque/php-resque": "1.3.6",
composer.lock
Generated
+23 -15
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "329498d12fabaa764c4178881cb07788",
"content-hash": "ff72579b1906a3ee181ec1209198c13f",
"packages": [
{
"name": "adhocore/jwt",
@@ -2622,26 +2622,31 @@
},
{
"name": "utopia-php/storage",
"version": "0.13.2",
"version": "dev-feat-transfer",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/storage.git",
"reference": "ad1c00f24ca56e73888acc2af3deee4919b1194b"
"reference": "2ee3666511b3da604e7e4d66263f1dbea9f6fb04"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/ad1c00f24ca56e73888acc2af3deee4919b1194b",
"reference": "ad1c00f24ca56e73888acc2af3deee4919b1194b",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/2ee3666511b3da604e7e4d66263f1dbea9f6fb04",
"reference": "2ee3666511b3da604e7e4d66263f1dbea9f6fb04",
"shasum": ""
},
"require": {
"ext-brotli": "*",
"ext-fileinfo": "*",
"ext-lz4": "*",
"ext-snappy": "*",
"ext-xz": "*",
"ext-zlib": "*",
"ext-zstd": "*",
"php": ">=8.0",
"utopia-php/framework": "0.*.*"
},
"require-dev": {
"laravel/pint": "1.2.*",
"phpunit/phpunit": "^9.3",
"vimeo/psalm": "4.0.1"
},
@@ -2655,12 +2660,6 @@
"license": [
"MIT"
],
"authors": [
{
"name": "Eldad Fux",
"email": "eldad@appwrite.io"
}
],
"description": "A simple Storage library to manage application storage",
"keywords": [
"framework",
@@ -2671,9 +2670,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/storage/issues",
"source": "https://github.com/utopia-php/storage/tree/0.13.2"
"source": "https://github.com/utopia-php/storage/tree/feat-transfer"
},
"time": "2022-12-20T11:11:35+00:00"
"time": "2023-07-19T07:14:15+00:00"
},
{
"name": "utopia-php/swoole",
@@ -5521,9 +5520,18 @@
"time": "2023-06-08T12:52:13+00:00"
}
],
"aliases": [],
"aliases": [
{
"package": "utopia-php/storage",
"version": "dev-feat-transfer",
"alias": "0.13.3",
"alias_normalized": "0.13.3.0"
}
],
"minimum-stability": "stable",
"stability-flags": [],
"stability-flags": {
"utopia-php/storage": 20
},
"prefer-stable": false,
"prefer-lowest": false,
"platform": {
docker-compose.yml
+3 -1
View File
@@ -635,6 +635,8 @@ services:
- appwrite-mariadb:/var/lib/mysql:rw
- ./backups:/backups:rw
- /var/run/docker.sock:/var/run/docker.sock
- ./dev:/usr/local/dev
- ./vendor/utopia-php/storage:/usr/src/code/vendor/utopia-php/storage
depends_on:
- redis
environment:
@@ -744,7 +746,7 @@ services:
- OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG
mariadb:
image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p
image: mysql:8.0.30 # fix issues when upgrading using: mysql_upgrade -u root -p
container_name: appwrite-mariadb
<<: *x-logging
networks:
src/Appwrite/Platform/Tasks/Backup.php
+47 -23
View File
@@ -6,6 +6,7 @@ use Utopia\Platform\Action;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Storage\Device\DOSpaces;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
class Backup extends Action
@@ -31,34 +32,41 @@ class Backup extends Action
*/
public function action(): void
{
self::log('--- Backup Start --- ');
$this->checkEnvVariables();
$folder = App::getEnv('_APP_BACKUP_FOLDER');
$start = microtime(true);
$time = date('Y-m-d_H:i:s');
self::log('--- Backup Start --- ');
$filename = date('Ymd_His') . '.tar.gz';
$folder = App::getEnv('_APP_BACKUP_FOLDER');
$project = explode('=', App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))[0];
$s3 = new DOSpaces('/v1/' . $project . '/' . $folder, App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
$local = new Local(self::$backups . '/' . $project . '/' . $folder);
$source = $local->getRoot() . '/' . $filename;
Storage::setDevice('files', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')));
$device = Storage::getDevice('files');
$local->setTransferChunkSize(10 * 1024 * 1024); // > 5MB
// Todo: do we want to have the backup ready on the mounted dir? or to abort?
if (!$device->exists('/')) {
// $source = '/backups/shmuel.tar.gz'; // 1452521974
// $destination = '/shmuel/' . $time . '.tar.gz';
//
// $local->transfer($source, $destination, $s3);
//
// if ($s3->exists($destination)) {
// Console::success('Uploaded successfully !!! ' . $destination);
// Console::exit();
// }
if (!$s3->exists('/')) {
Console::error('Can\'t read from DO ');
Console::exit();
}
$dsn = explode('=', App::getEnv('_APP_CONNECTIONS_DB_PROJECT'))[0];
if (!file_exists(self::$backups)) {
Console::error('Mount directory does not exist');
Console::exit();
}
$backups = self::$backups . '/' . $dsn . '/' . $folder;
if (!file_exists($backups) && !mkdir($backups, 0755, true)) {
Console::error('Error creating directory: ' . $backups);
if (!file_exists($local->getRoot()) && !mkdir($local->getRoot(), 0755, true)) {
Console::error('Error creating directory: ' . $local->getRoot());
Console::exit();
}
@@ -73,9 +81,6 @@ class Backup extends Action
Console::exit();
}
$filename = $time . '.tar.gz';
$file = $backups . '/' . $filename;
// $cmd = 'tar czf ' . $file . ' --absolute-names ' . $this->directory;
//
// $cmd = '
@@ -86,7 +91,7 @@ class Backup extends Action
$stdout = '';
$stderr = '';
$cmd = 'cd ' . self::$mysqlDirectory . ' && tar zcf ' . $file . ' .';
$cmd = 'cd ' . self::$mysqlDirectory . ' && tar zcf ' . $source . ' .';
self::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
self::log($stdout);
@@ -95,6 +100,19 @@ class Backup extends Action
Console::exit();
}
if (!file_exists($source)) {
Console::error("Can't find tar file: " . $source);
Console::exit();
}
$filesize = \filesize($source);
self::log("Tar file size is :" . ceil($filesize / 1024 / 1024) . 'MB');
if ($filesize < (2 * 1024)) {
Console::error("File size is very small: " . $source);
Console::exit();
}
$stdout = '';
$stderr = '';
$cmd = 'docker start ' . $this->containerName;
@@ -107,10 +125,17 @@ class Backup extends Action
}
try {
$path = '/' . $dsn . '/' . $folder . '/' . $filename;
self::log('Uploading ' . $path);
if (!$device->upload($file, $path)) {
Console::error('Error uploading to ' . $path);
self::log('Start transferring ' . $source);
$destination = $s3->getRoot() . '/' . $filename;
if (!$local->transfer($source, $destination, $s3)) {
Console::error('Error transferring to ' . $destination);
Console::exit();
}
if (!$s3->exists($destination)) {
Console::error('File not found on s3 ' . $destination);
Console::exit();
}
} catch (\Exception $e) {
@@ -150,5 +175,4 @@ class Backup extends Action
}
}
}
}
src/Appwrite/Platform/Tasks/Restore.php
+22 -41
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Tasks;
use Exception;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
@@ -11,11 +12,10 @@ use Utopia\Storage\Storage;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
// docker compose exec appwrite-backup db-restore --cloud=false --filename=2023-07-18_12:39:59.tar.gz
class Restore extends Action
{
// todo: double check this is not a production value!!!!!!!!!!!!!!!
// docker compose exec appwrite-backup db-restore --cloud=false --filename=2023-07-19_09:25:11.tar.gz --project=db_fra1_02 --folder=daily
// todo: Carefully double check this is not a production value!!!!!!!!!!!!!!!
// todo: it will be erased!!!!
protected string $containerName = 'appwrite-mariadb';
@@ -36,7 +36,7 @@ class Restore extends Action
}
/**
* @throws \Exception
* @throws Exception
*/
public function action(string $filename, string $cloud, string $project, string $folder): void
{
@@ -49,6 +49,8 @@ class Restore extends Action
$file = Backup::$backups . '/' . $project . '/' . $folder . '/' . $filename;
$extract = Backup::$backups . '/extract';
$original = $extract . '/original-' . time();
$s3 = new DOSpaces('/v1/' . $project . '/' . $folder, App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION'));
$download = new Local(Backup::$backups . '/downloads');
Backup::log('Creating directory ' . $original);
if (!file_exists($original) && !mkdir($original, 0755, true)) {
@@ -56,37 +58,28 @@ class Restore extends Action
Console::exit();
}
// $tarDirectory = $extract . '/' . str_replace(['.', '-', '_', ':', 'tar', 'gz'], '', $filename);
// Backup::log('Creating directory ' . $tarDirectory);
// if (!file_exists($tarDirectory) && !mkdir($tarDirectory, 0755, true)) {
// Console::error('Error creating directory: ' . $tarDirectory);
// Console::exit();
// }
if (!file_exists($download->getRoot()) && !mkdir($download->getRoot(), 0755, true)) {
Console::error('Error creating directory: ' . $download->getRoot());
Console::exit();
}
if ($cloud) {
Storage::setDevice('s3', new DOSpaces('backups', App::getEnv('_DO_SPACES_ACCESS_KEY'), App::getEnv('_DO_SPACES_SECRET_KEY'), App::getEnv('_DO_SPACES_BUCKET_NAME'), App::getEnv('_DO_SPACES_REGION')));
Storage::setDevice('mount', new Local(Backup::$backups));
$device = Storage::getDevice('s3');
$mount = Storage::getDevice('mount');
try {
$folder = App::getEnv('_APP_BACKUP_FOLDER', 'hourly');
$path = '/' . $project . '/' . $folder . '/' . $filename;
$path = $s3->getPath($filename);
if (!$device->exists($path)) {
if (!$s3->exists($path)) {
Console::error('File: ' . $path . ' does not exist on cloud');
Console::exit();
}
while ($data = $device->read($path)) {
$mount->write('file-' . $file, $data);
}
$file = $download->getPath($filename);
Backup::log('Transferring ' . $path . ' => ' . $file);
Backup::log('Downloading from s3 ' . $path);
$mount->write('file-' . $file, $device->read($path));
// $mount->get('file-' . $file, $device->read($path));
} catch (\Exception $e) {
if (!$s3->transfer($path, $file, $download)) {
Console::error('Error transferring ' . $file);
Console::exit();
}
} catch (Exception $e) {
Console::error($e->getMessage());
Console::exit();
}
@@ -112,6 +105,8 @@ class Restore extends Action
$stdout = '';
$stderr = '';
$cmd = 'mv ' . Backup::$mysqlDirectory . '/* ' . ' ' . $original . '/';
// todo: do we care about original?
$cmd = 'rm -r ' . Backup::$mysqlDirectory . '/*';
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
Backup::log($stdout);
@@ -120,12 +115,9 @@ class Restore extends Action
Console::exit();
}
$stdout = '';
$stderr = '';
// $cmd = 'tar -xzf ' . $file . ' -C ' . $tarDirectory;
$cmd = 'tar -xzf ' . $file . ' -C ' . Backup::$mysqlDirectory;
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
if (!empty($stderr)) {
@@ -134,24 +126,13 @@ class Restore extends Action
Console::exit();
}
// $stdout = '';
// $stderr = '';
// $cmd = 'mv ' . $tarDirectory . ' ' . Backup::$mysqlDirectory;
// Backup::log($cmd);
// Console::execute($cmd, '', $stdout, $stderr);
// Backup::log($stdout);
// if (!empty($stderr)) {
// Console::error($stderr);
// Console::exit();
// }
$stdout = '';
$stderr = '';
$cmd = 'docker start ' . $this->containerName;
Backup::log($cmd);
Console::execute($cmd, '', $stdout, $stderr);
Backup::log($stdout);
if (!empty($stderr)) {
Backup::log($stdout);
Console::error($stderr);
Console::exit();
}