Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement API Flow

Browse Source
This commit is contained in:
Bradley Schofield
2023-03-24 13:26:27 +09:00
parent fea2dd905c
commit acb3c250a4
8 changed files with 165 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/config/collections.php
+6 -6
View File
@@ -3610,7 +3610,7 @@ $collections = [
'filters' => [],
],
[
'$id' => ID::custom('progress'),
'$id' => ID::custom('totalProgress'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 3000,
@@ -3621,7 +3621,7 @@ $collections = [
'filters' => [],
],
[
'$id' => ID::custom('latestUpdate'),
'$id' => ID::custom('latestProgress'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
@@ -3635,12 +3635,12 @@ $collections = [
'$id' => ID::custom('errorData'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 8046,
'size' => 65535,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
'filters' => ['json'],
],
[
'$id' => ID::custom('search'),
@@ -3684,9 +3684,9 @@ $collections = [
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_latestUpdate',
'$id' => '_key_latestProgress',
'type' => Database::INDEX_KEY,
'attributes' => ['latestUpdate'],
'attributes' => ['latestProgress'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
app/config/errors.php
+36
View File
@@ -559,4 +559,40 @@ return [
'description' => 'Too many queries.',
'code' => 400,
],
// Transfers
Exception::TRANSFER_NOT_FOUND => [
'name' => Exception::TRANSFER_NOT_FOUND,
'description' => 'Transfer with the requested ID could not be found.',
'code' => 404,
],
Exception::TRANSFER_ALREADY_EXISTS => [
'name' => Exception::TRANSFER_ALREADY_EXISTS,
'description' => 'Transfer with the requested ID already exists.',
'code' => 409,
],
Exception::TRANSFER_SOURCE_NOT_FOUND => [
'name' => Exception::TRANSFER_SOURCE_NOT_FOUND,
'description' => 'Transfer source with the requested ID could not be found.',
'code' => 404,
],
Exception::TRANSFER_DESTINATION_NOT_FOUND => [
'name' => Exception::TRANSFER_DESTINATION_NOT_FOUND,
'description' => 'Transfer destination with the requested ID could not be found.',
'code' => 404,
],
Exception::TRANSFER_SOURCE_FAILED => [
'name' => Exception::TRANSFER_SOURCE_FAILED,
'description' => 'Transfer source failed to complete the transfer.',
'code' => 500,
],
Exception::TRANSFER_DESTINATION_FAILED => [
'name' => Exception::TRANSFER_DESTINATION_FAILED,
'description' => 'Transfer destination failed to complete the transfer.',
'code' => 500,
],
Exception::TRANSFER_IN_PROGRESS => [
'name' => Exception::TRANSFER_IN_PROGRESS,
'description' => 'Transfer is already in progress.',
'code' => 409,
],
];
app/controllers/api/transfers.php
+2 -2
View File
@@ -67,11 +67,11 @@ App::post('/v1/transfers')
'source' => $source,
'destination' => $destination,
'resources' => $resources,
'progress' => json_encode([
'totalProgress' => json_encode([
'source' => [],
'destination' => [],
]),
'latestUpdate' => "{}",
'latestProgress' => "{}",
'errorData' => ""
]));
app/workers/transfers.php
+92 -31
View File
@@ -1,11 +1,12 @@
<?php
use Appwrite\Resque\Worker;
use MongoDB\Operation\Update;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Transfer\Destination;
use Utopia\Transfer\Destinations\Appwrite as DestinationsAppwrite;
use Utopia\Transfer\Progress;
use Utopia\Transfer\Source;
use Utopia\Transfer\Sources\Appwrite;
use Utopia\Transfer\Sources\Firebase;
@@ -24,7 +25,7 @@ class TransfersV1 extends Worker
*
* @var Database
*/
private Database $dbForConsole;
private Database $dbForProject;
public function getName(): string
{
@@ -37,7 +38,7 @@ class TransfersV1 extends Worker
public function run(): void
{
var_dump("Test");
$this->dbForProject = $this->getProjectDB($this->args['project']['$id']);
// Process
$this->processTransfer();
@@ -51,35 +52,37 @@ class TransfersV1 extends Worker
*/
function processSource(): Source
{
$source = $this->args['source'];
$source = $this->dbForProject->getDocument('sources', $this->args['transfer']['source']);
$authObject = json_decode($source['data'], true) ?? [];
switch ($source['type']) {
case 'firebase':
return new Firebase(
$source['authObject'] ?? '',
$authObject['serviceAccount'] ?? '',
Firebase::AUTH_SERVICEACCOUNT
);
break;
case 'supabase':
return new Supabase(
$source['host'] ?? '',
$source['databaseName'] ?? '',
$source['username'] ?? '',
$source['password'] ?? '',
$source['port'] ?? 5432,
$authObject['url'] ?? '',
$authObject['database'] ?? '',
$authObject['username'] ?? '',
$authObject['password'] ?? '',
$authObject['port'] ?? 5432,
);
break;
case 'nhost':
return new NHost(
$source['host'] ?? '',
$source['databaseName'] ?? '',
$source['username'] ?? '',
$source['password'] ?? '',
$source['port'] ?? 5432,
$authObject['url'] ?? '',
$authObject['database'] ?? '',
$authObject['username'] ?? '',
$authObject['password'] ?? '',
$authObject['port'] ?? 5432,
);
break;
case 'appwrite':
return new Appwrite($source['projectId'], $source['endpoint'], $source['key']);
return new Appwrite($authObject['projectId'], $authObject['endpoint'], $authObject['key']);
break;
default:
throw new \Exception('Invalid source type');
@@ -87,10 +90,37 @@ class TransfersV1 extends Worker
}
}
/**
* Process Destination
*
* @return Destination
* @throws \Exception
*/
function processDestination(): Destination
{
$destination = $this->dbForProject->getDocument('destinations', $this->args['transfer']['destination']);
$authObject = json_decode($destination['data'], true) ?? [];
switch ($destination['type']) {
case 'appwrite':
if ($authObject['endpoint'] === 'http://localhost/v1') { // Rewrite into Internal Network.
return new DestinationsAppwrite($authObject['projectId'], 'http://appwrite/v1', $authObject['key']);
} else {
return new DestinationsAppwrite($authObject['projectId'], $authObject['endpoint'], $authObject['key']);
}
break;
default:
throw new \Exception('Invalid destination type');
break;
}
}
protected function updateAttribute(string $attribute, mixed $value, Document $document): void
{
$document->setAttribute($attribute, $value);
$this->dbForConsole->updateDocument($document->getCollection(), $document->getId(), $document);
$this->dbForProject->updateDocument($document->getCollection(), $document->getId(), $document);
}
/**
@@ -101,13 +131,14 @@ class TransfersV1 extends Worker
protected function processTransfer(): void
{
$transferDocument = null;
$transfer = null;
try {
$transferDocument = $this->dbForConsole->getDocument('transfers', $this->args['transferId']);
$transferDocument = $this->dbForProject->getDocument('transfers', $this->args['transfer']['$id']);
$this->updateAttribute('status', 'processing', $transferDocument);
$source = $this->processSource();
$destination = new DestinationsAppwrite($this->args['projectId'], $this->args['endpoint'], $this->args['key']);
$destination = $this->processDestination();
$transfer = new \Utopia\Transfer\Transfer(
$source,
@@ -117,38 +148,64 @@ class TransfersV1 extends Worker
$this->updateAttribute('stage', 'source-check', $transferDocument);
if (!$source->check()) {
$transferDocument->setAttribute('status', 'failed');
$transferDocument->setAttribute('errorData', $transfer->getLogs('error'));
$transferDocument->setAttribute('errorData', json_encode($transfer->getLogs('error')));
}
$this->updateAttribute('stage', 'destination-check', $transferDocument);
if (!$destination->check()) {
$transferDocument->setAttribute('status', 'failed');
$transferDocument->setAttribute('stage', 'destination-check');
$transferDocument->setAttribute('errorData', $transfer->getLogs('error'));
$transferDocument->setAttribute('errorData', json_encode($transfer->getLogs('error')));
}
/** Start Transfer */
$transfer->run($this->args['resoruces'], function (Update $update) use ($transferDocument, $transfer, $source, $destination) {
$transfer->run($transferDocument->getAttribute('resources'), function (Progress $progress) use ($transferDocument, $transfer, $source, $destination) {
var_dump($progress->getProgress());
$transferDocument->setAttribute('stage', 'transfer');
$transferDocument->setAttribute('latestUpdate', json_encode($update));
$transferDocument->setAttribute('progress', json_encode([
'source' => $source->getCounter(''),
'destination' => $destination->getCounter(''),
$transferDocument->setAttribute('latestProgress', json_encode($progress));
$transferDocument->setAttribute('totalProgress', json_encode([
'source' => $source->getCounter(),
'destination' => $destination->getCounter(),
]));
$this->dbForConsole->updateDocument($transferDocument->getCollection(), $transferDocument->getId(), $transferDocument);
$this->dbForProject->updateDocument($transferDocument->getCollection(), $transferDocument->getId(), $transferDocument);
});
if (!empty($transfer->getLogs('error'))) {
$transferDocument->setAttribute('status', 'failed');
$transferDocument->setAttribute('errorData', $transfer->getLogs('error'));
}
$transferDocument->setAttribute('status', 'completed');
$logs = [];
foreach ($transfer->getLogs('error') as $log) {
$logs[] = $log->asArray();
}
var_dump($logs);
$transferDocument->setAttribute('errorData', json_encode($logs));
} else {
$transferDocument->setAttribute('status', 'completed');
}
} catch (\Throwable $th) {
Console::error($th->getMessage());
// Improve Error Handler.
if ($transferDocument) {
$transferDocument->setAttribute('status', 'failed');
$transferDocument->setAttribute('errorData', $transfer->getLogs('error'));
foreach ($transfer->getLogs('error') as $log) {
$logs[] = $log->asArray();
}
var_dump($logs);
$transferDocument->setAttribute('errorData', json_encode($logs));
}
throw $th;
} finally {
if ($transferDocument) {
$this->dbForProject->updateDocument($transferDocument->getCollection(), $transferDocument->getId(), $transferDocument);
}
}
}
@@ -161,4 +218,8 @@ class TransfersV1 extends Worker
protected function processVerification(): void
{
}
public function shutdown(): void
{
}
}
docker-compose.yml
+22 -14
View File
@@ -587,16 +587,24 @@ services:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./tests:/usr/src/code/tests
- ./vendor:/usr/src/code/vendor
depends_on:
- redis
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_SMS_PROVIDER
- _APP_SMS_FROM
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@@ -808,18 +816,18 @@ services:
# ports:
# - "8081:8081"
resque:
image: appwrite/resque-web:1.1.0
restart: unless-stopped
networks:
- appwrite
ports:
- "5678:5678"
environment:
- RESQUE_WEB_HOST=redis
- RESQUE_WEB_PORT=6379
- RESQUE_WEB_HTTP_BASIC_AUTH_USER=user
- RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password
# resque:
# image: appwrite/resque-web:1.1.0
# restart: unless-stopped
# networks:
# - appwrite
# ports:
# - "5678:5678"
# environment:
# - RESQUE_WEB_HOST=redis
# - RESQUE_WEB_PORT=6379
# - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user
# - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password
# chronograf:
# image: chronograf:1.6
src/Appwrite/Event/Event.php
+1 -1
View File
@@ -35,7 +35,7 @@ class Event
public const MESSAGING_QUEUE_NAME = 'v1-messaging';
public const MESSAGING_CLASS_NAME = 'MessagingV1';
public const TRANSFER_QUEUE_NAME = 'v1-transfer';
public const TRANSFER_QUEUE_NAME = 'v1-transfers';
public const TRANSFER_CLASS_NAME = 'TransfersV1';
protected string $queue = '';
src/Appwrite/Utopia/Database/Validator/Queries/Transfers.php
+2 -2
View File
@@ -10,8 +10,8 @@ class Transfers extends Base
'source',
'destination',
'resources',
'progress',
'latestUpdate',
'totalProgress',
'latestProgress',
'errorData'
];
src/Appwrite/Utopia/Response/Model/Transfer.php
+4 -4
View File
@@ -59,15 +59,15 @@ class Transfer extends Model
'example' => ['users'],
'array' => true
])
->addRule('progress', [
->addRule('totalProgress', [
'type' => self::TYPE_JSON,
'description' => 'Transfer progress.',
'description' => 'A group of counters that represent the total progress of the transfer.',
'default' => [],
'example' => '{"source":[], "destination": []}',
])
->addRule('latestUpdate', [
->addRule('latestProgress', [
'type' => self::TYPE_JSON,
'description' => 'Latest update.',
'description' => 'The latest progress of the transfer.',
'default' => [],
'example' => '{"source":[], "destination": []}',
])