diff --git a/app/config/collections.php b/app/config/collections.php index 973d1d3571..da89c85f89 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -3610,7 +3610,7 @@ $collections = [ 'filters' => [], ], [ - '$id' => ID::custom('progress'), + '$id' => ID::custom('totalProgress'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 3000, @@ -3621,7 +3621,7 @@ $collections = [ 'filters' => [], ], [ - '$id' => ID::custom('latestUpdate'), + '$id' => ID::custom('latestProgress'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => Database::LENGTH_KEY, @@ -3635,12 +3635,12 @@ $collections = [ '$id' => ID::custom('errorData'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 8046, + 'size' => 65535, 'signed' => true, 'required' => true, 'default' => null, 'array' => false, - 'filters' => [], + 'filters' => ['json'], ], [ '$id' => ID::custom('search'), @@ -3684,9 +3684,9 @@ $collections = [ 'orders' => [Database::ORDER_ASC], ], [ - '$id' => '_key_latestUpdate', + '$id' => '_key_latestProgress', 'type' => Database::INDEX_KEY, - 'attributes' => ['latestUpdate'], + 'attributes' => ['latestProgress'], 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], diff --git a/app/config/errors.php b/app/config/errors.php index 266e017e93..0c3a7ac602 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -559,4 +559,40 @@ return [ 'description' => 'Too many queries.', 'code' => 400, ], + // Transfers + Exception::TRANSFER_NOT_FOUND => [ + 'name' => Exception::TRANSFER_NOT_FOUND, + 'description' => 'Transfer with the requested ID could not be found.', + 'code' => 404, + ], + Exception::TRANSFER_ALREADY_EXISTS => [ + 'name' => Exception::TRANSFER_ALREADY_EXISTS, + 'description' => 'Transfer with the requested ID already exists.', + 'code' => 409, + ], + Exception::TRANSFER_SOURCE_NOT_FOUND => [ + 'name' => Exception::TRANSFER_SOURCE_NOT_FOUND, + 'description' => 'Transfer source with the requested ID could not be found.', + 'code' => 404, + ], + Exception::TRANSFER_DESTINATION_NOT_FOUND => [ + 'name' => Exception::TRANSFER_DESTINATION_NOT_FOUND, + 'description' => 'Transfer destination with the requested ID could not be found.', + 'code' => 404, + ], + Exception::TRANSFER_SOURCE_FAILED => [ + 'name' => Exception::TRANSFER_SOURCE_FAILED, + 'description' => 'Transfer source failed to complete the transfer.', + 'code' => 500, + ], + Exception::TRANSFER_DESTINATION_FAILED => [ + 'name' => Exception::TRANSFER_DESTINATION_FAILED, + 'description' => 'Transfer destination failed to complete the transfer.', + 'code' => 500, + ], + Exception::TRANSFER_IN_PROGRESS => [ + 'name' => Exception::TRANSFER_IN_PROGRESS, + 'description' => 'Transfer is already in progress.', + 'code' => 409, + ], ]; diff --git a/app/controllers/api/transfers.php b/app/controllers/api/transfers.php index b589f34a39..9d9894c789 100644 --- a/app/controllers/api/transfers.php +++ b/app/controllers/api/transfers.php @@ -67,11 +67,11 @@ App::post('/v1/transfers') 'source' => $source, 'destination' => $destination, 'resources' => $resources, - 'progress' => json_encode([ + 'totalProgress' => json_encode([ 'source' => [], 'destination' => [], ]), - 'latestUpdate' => "{}", + 'latestProgress' => "{}", 'errorData' => "" ])); diff --git a/app/workers/transfers.php b/app/workers/transfers.php index 36f1fa3d49..3ab86d1f56 100644 --- a/app/workers/transfers.php +++ b/app/workers/transfers.php @@ -1,11 +1,12 @@ dbForProject = $this->getProjectDB($this->args['project']['$id']); // Process $this->processTransfer(); @@ -51,35 +52,37 @@ class TransfersV1 extends Worker */ function processSource(): Source { - $source = $this->args['source']; + $source = $this->dbForProject->getDocument('sources', $this->args['transfer']['source']); + + $authObject = json_decode($source['data'], true) ?? []; switch ($source['type']) { case 'firebase': return new Firebase( - $source['authObject'] ?? '', + $authObject['serviceAccount'] ?? '', Firebase::AUTH_SERVICEACCOUNT ); break; case 'supabase': return new Supabase( - $source['host'] ?? '', - $source['databaseName'] ?? '', - $source['username'] ?? '', - $source['password'] ?? '', - $source['port'] ?? 5432, + $authObject['url'] ?? '', + $authObject['database'] ?? '', + $authObject['username'] ?? '', + $authObject['password'] ?? '', + $authObject['port'] ?? 5432, ); break; case 'nhost': return new NHost( - $source['host'] ?? '', - $source['databaseName'] ?? '', - $source['username'] ?? '', - $source['password'] ?? '', - $source['port'] ?? 5432, + $authObject['url'] ?? '', + $authObject['database'] ?? '', + $authObject['username'] ?? '', + $authObject['password'] ?? '', + $authObject['port'] ?? 5432, ); break; case 'appwrite': - return new Appwrite($source['projectId'], $source['endpoint'], $source['key']); + return new Appwrite($authObject['projectId'], $authObject['endpoint'], $authObject['key']); break; default: throw new \Exception('Invalid source type'); @@ -87,10 +90,37 @@ class TransfersV1 extends Worker } } + /** + * Process Destination + * + * @return Destination + * @throws \Exception + */ + function processDestination(): Destination + { + $destination = $this->dbForProject->getDocument('destinations', $this->args['transfer']['destination']); + + $authObject = json_decode($destination['data'], true) ?? []; + + + switch ($destination['type']) { + case 'appwrite': + if ($authObject['endpoint'] === 'http://localhost/v1') { // Rewrite into Internal Network. + return new DestinationsAppwrite($authObject['projectId'], 'http://appwrite/v1', $authObject['key']); + } else { + return new DestinationsAppwrite($authObject['projectId'], $authObject['endpoint'], $authObject['key']); + } + break; + default: + throw new \Exception('Invalid destination type'); + break; + } + } + protected function updateAttribute(string $attribute, mixed $value, Document $document): void { $document->setAttribute($attribute, $value); - $this->dbForConsole->updateDocument($document->getCollection(), $document->getId(), $document); + $this->dbForProject->updateDocument($document->getCollection(), $document->getId(), $document); } /** @@ -101,13 +131,14 @@ class TransfersV1 extends Worker protected function processTransfer(): void { $transferDocument = null; + $transfer = null; try { - $transferDocument = $this->dbForConsole->getDocument('transfers', $this->args['transferId']); + $transferDocument = $this->dbForProject->getDocument('transfers', $this->args['transfer']['$id']); $this->updateAttribute('status', 'processing', $transferDocument); $source = $this->processSource(); - $destination = new DestinationsAppwrite($this->args['projectId'], $this->args['endpoint'], $this->args['key']); + $destination = $this->processDestination(); $transfer = new \Utopia\Transfer\Transfer( $source, @@ -117,38 +148,64 @@ class TransfersV1 extends Worker $this->updateAttribute('stage', 'source-check', $transferDocument); if (!$source->check()) { $transferDocument->setAttribute('status', 'failed'); - $transferDocument->setAttribute('errorData', $transfer->getLogs('error')); + $transferDocument->setAttribute('errorData', json_encode($transfer->getLogs('error'))); } $this->updateAttribute('stage', 'destination-check', $transferDocument); if (!$destination->check()) { $transferDocument->setAttribute('status', 'failed'); $transferDocument->setAttribute('stage', 'destination-check'); - $transferDocument->setAttribute('errorData', $transfer->getLogs('error')); + $transferDocument->setAttribute('errorData', json_encode($transfer->getLogs('error'))); } /** Start Transfer */ - $transfer->run($this->args['resoruces'], function (Update $update) use ($transferDocument, $transfer, $source, $destination) { + $transfer->run($transferDocument->getAttribute('resources'), function (Progress $progress) use ($transferDocument, $transfer, $source, $destination) { + var_dump($progress->getProgress()); $transferDocument->setAttribute('stage', 'transfer'); - $transferDocument->setAttribute('latestUpdate', json_encode($update)); - $transferDocument->setAttribute('progress', json_encode([ - 'source' => $source->getCounter(''), - 'destination' => $destination->getCounter(''), + $transferDocument->setAttribute('latestProgress', json_encode($progress)); + $transferDocument->setAttribute('totalProgress', json_encode([ + 'source' => $source->getCounter(), + 'destination' => $destination->getCounter(), ])); - $this->dbForConsole->updateDocument($transferDocument->getCollection(), $transferDocument->getId(), $transferDocument); + $this->dbForProject->updateDocument($transferDocument->getCollection(), $transferDocument->getId(), $transferDocument); }); if (!empty($transfer->getLogs('error'))) { $transferDocument->setAttribute('status', 'failed'); - $transferDocument->setAttribute('errorData', $transfer->getLogs('error')); - } - $transferDocument->setAttribute('status', 'completed'); + $logs = []; + + foreach ($transfer->getLogs('error') as $log) { + $logs[] = $log->asArray(); + } + + var_dump($logs); + + $transferDocument->setAttribute('errorData', json_encode($logs)); + } else { + $transferDocument->setAttribute('status', 'completed'); + } } catch (\Throwable $th) { Console::error($th->getMessage()); + + // Improve Error Handler. + if ($transferDocument) { $transferDocument->setAttribute('status', 'failed'); - $transferDocument->setAttribute('errorData', $transfer->getLogs('error')); + + foreach ($transfer->getLogs('error') as $log) { + $logs[] = $log->asArray(); + } + + var_dump($logs); + + $transferDocument->setAttribute('errorData', json_encode($logs)); + } + + throw $th; + } finally { + if ($transferDocument) { + $this->dbForProject->updateDocument($transferDocument->getCollection(), $transferDocument->getId(), $transferDocument); } } } @@ -161,4 +218,8 @@ class TransfersV1 extends Worker protected function processVerification(): void { } + + public function shutdown(): void + { + } } diff --git a/docker-compose.yml b/docker-compose.yml index c20366bc31..5d12048d49 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -587,16 +587,24 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests + - ./vendor:/usr/src/code/vendor depends_on: - redis environment: - _APP_ENV + - _APP_OPENSSL_KEY_V1 + - _APP_DOMAIN + - _APP_DOMAIN_TARGET + - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS - - _APP_SMS_PROVIDER - - _APP_SMS_FROM + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -808,18 +816,18 @@ services: # ports: # - "8081:8081" - resque: - image: appwrite/resque-web:1.1.0 - restart: unless-stopped - networks: - - appwrite - ports: - - "5678:5678" - environment: - - RESQUE_WEB_HOST=redis - - RESQUE_WEB_PORT=6379 - - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user - - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password + # resque: + # image: appwrite/resque-web:1.1.0 + # restart: unless-stopped + # networks: + # - appwrite + # ports: + # - "5678:5678" + # environment: + # - RESQUE_WEB_HOST=redis + # - RESQUE_WEB_PORT=6379 + # - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user + # - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password # chronograf: # image: chronograf:1.6 diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 719fbd5543..0015b7234a 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -35,7 +35,7 @@ class Event public const MESSAGING_QUEUE_NAME = 'v1-messaging'; public const MESSAGING_CLASS_NAME = 'MessagingV1'; - public const TRANSFER_QUEUE_NAME = 'v1-transfer'; + public const TRANSFER_QUEUE_NAME = 'v1-transfers'; public const TRANSFER_CLASS_NAME = 'TransfersV1'; protected string $queue = ''; diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Transfers.php b/src/Appwrite/Utopia/Database/Validator/Queries/Transfers.php index 9a1462a77f..54a94339fd 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Transfers.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Transfers.php @@ -10,8 +10,8 @@ class Transfers extends Base 'source', 'destination', 'resources', - 'progress', - 'latestUpdate', + 'totalProgress', + 'latestProgress', 'errorData' ]; diff --git a/src/Appwrite/Utopia/Response/Model/Transfer.php b/src/Appwrite/Utopia/Response/Model/Transfer.php index e518ebfd25..b0699783d9 100644 --- a/src/Appwrite/Utopia/Response/Model/Transfer.php +++ b/src/Appwrite/Utopia/Response/Model/Transfer.php @@ -59,15 +59,15 @@ class Transfer extends Model 'example' => ['users'], 'array' => true ]) - ->addRule('progress', [ + ->addRule('totalProgress', [ 'type' => self::TYPE_JSON, - 'description' => 'Transfer progress.', + 'description' => 'A group of counters that represent the total progress of the transfer.', 'default' => [], 'example' => '{"source":[], "destination": []}', ]) - ->addRule('latestUpdate', [ + ->addRule('latestProgress', [ 'type' => self::TYPE_JSON, - 'description' => 'Latest update.', + 'description' => 'The latest progress of the transfer.', 'default' => [], 'example' => '{"source":[], "destination": []}', ])