From 0d4b70633918eaecaf9f4ea9b4a1eeadf4840917 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 2 Aug 2021 11:56:08 +0545 Subject: [PATCH 01/45] save usage data directly to statsd without redis queue --- app/controllers/shared/api.php | 54 ++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index cdbf561e20..901178a94e 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -169,7 +169,7 @@ App::init(function ($utopia, $request, $response, $project, $user) { }, ['utopia', 'request', 'response', 'project', 'user'], 'auth'); -App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $usage, $deletes, $database, $mode) { +App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $statsd, $usage, $deletes, $database, $mode) { /** @var Utopia\App $utopia */ /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ @@ -221,9 +221,53 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits $usage ->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage')) - ->setParam('networkResponseSize', $response->getSize()) - ->trigger() - ; + ->setParam('networkResponseSize', $response->getSize()); + + statsdUpdate($statsd, $usage); + + } -}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api'); +}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'statsd', 'usage', 'deletes', 'database', 'mode'], 'api'); + +function statsdUpdate($statsd, $usage): void +{ + /** @var Appwrite\Event\Event $usage */ + + $projectId = $usage->getParam('projectId') ?? ''; + + $storage = $usage->getParam('storage') ?? 0; + + $networkRequestSize = $usage->getParam('networkRequestSize') ?? 0; + $networkResponseSize = $usage->getParam('networkResponseSize') ?? 0; + + $httpMethod = $usage->getParam('httpMethod') ?? ''; + $httpRequest = $usage->getParam('httpRequest') ?? 0; + + $functionId = $usage->getParam('functionId') ?? ''; + $functionExecution = $usage->getParam('functionExecution') ?? 0; + $functionExecutionTime = $usage->getParam('functionExecutionTime') ?? 0; + $functionStatus = $usage->getParam('functionStatus') ?? ''; + + $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); + + // the global namespace is prepended to every key (optional) + $statsd->setNamespace('appwrite.usage'); + + if($httpRequest >= 1) { + $statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod)); + } + + if($functionExecution >= 1) { + $statsd->increment('executions.all'.$tags.',functionId='.$functionId.',functionStatus='.$functionStatus); + $statsd->count('executions.time'.$tags.',functionId='.$functionId, $functionExecutionTime); + } + + $statsd->count('network.inbound'.$tags, $networkRequestSize); + $statsd->count('network.outbound'.$tags, $networkResponseSize); + $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize); + + if($storage >= 1) { + $statsd->count('storage.all'.$tags, $storage); + } +} \ No newline at end of file From ab841925ff351c6eb29a4953c1a469334e415016 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 2 Aug 2021 11:58:42 +0545 Subject: [PATCH 02/45] drop usage worker --- Dockerfile | 1 - app/workers/usage.php | 71 ------------------------------------------- bin/worker-usage | 10 ------ docker-compose.yml | 22 -------------- 4 files changed, 104 deletions(-) delete mode 100644 app/workers/usage.php delete mode 100644 bin/worker-usage diff --git a/Dockerfile b/Dockerfile index f3a87058af..8cba654921 100755 --- a/Dockerfile +++ b/Dockerfile @@ -239,7 +239,6 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-functions && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-tasks && \ - chmod +x /usr/local/bin/worker-usage && \ chmod +x /usr/local/bin/worker-webhooks # Letsencrypt Permissions diff --git a/app/workers/usage.php b/app/workers/usage.php deleted file mode 100644 index b5a3f885af..0000000000 --- a/app/workers/usage.php +++ /dev/null @@ -1,71 +0,0 @@ -get('statsd', true); - - $projectId = $this->args['projectId'] ?? ''; - - $storage = $this->args['storage'] ?? 0; - - $networkRequestSize = $this->args['networkRequestSize'] ?? 0; - $networkResponseSize = $this->args['networkResponseSize'] ?? 0; - - $httpMethod = $this->args['httpMethod'] ?? ''; - $httpRequest = $this->args['httpRequest'] ?? 0; - - $functionId = $this->args['functionId'] ?? ''; - $functionExecution = $this->args['functionExecution'] ?? 0; - $functionExecutionTime = $this->args['functionExecutionTime'] ?? 0; - $functionStatus = $this->args['functionStatus'] ?? ''; - - $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); - - // the global namespace is prepended to every key (optional) - $statsd->setNamespace('appwrite.usage'); - - if($httpRequest >= 1) { - $statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod)); - } - - if($functionExecution >= 1) { - $statsd->increment('executions.all'.$tags.',functionId='.$functionId.',functionStatus='.$functionStatus); - $statsd->count('executions.time'.$tags.',functionId='.$functionId, $functionExecutionTime); - } - - $statsd->count('network.inbound'.$tags, $networkRequestSize); - $statsd->count('network.outbound'.$tags, $networkResponseSize); - $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize); - - if($storage >= 1) { - $statsd->count('storage.all'.$tags, $storage); - } - } - - public function shutdown(): void - { - } -} \ No newline at end of file diff --git a/bin/worker-usage b/bin/worker-usage deleted file mode 100644 index 4174acce23..0000000000 --- a/bin/worker-usage +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=1 QUEUE='v1-usage' APP_INCLUDE='/usr/src/code/app/workers/usage.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index ed50016ed5..5f93426b4e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -122,28 +122,6 @@ services: - _APP_FUNCTIONS_MEMORY_SWAP - _APP_FUNCTIONS_RUNTIMES - appwrite-worker-usage: - entrypoint: worker-usage - container_name: appwrite-worker-usage - build: - context: . - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - telegraf - environment: - - _APP_ENV - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_STATSD_HOST - - _APP_STATSD_PORT - appwrite-worker-audits: entrypoint: worker-audits container_name: appwrite-worker-audits From 7d38b83abf5907983dc9b5b025fc2c713aa109f0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 2 Aug 2021 12:20:27 +0545 Subject: [PATCH 03/45] statsd env to appwrite image --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 5f93426b4e..89c88c68d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -121,6 +121,8 @@ services: - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP - _APP_FUNCTIONS_RUNTIMES + - _APP_STATSD_HOST + - _APP_STATSD_PORT appwrite-worker-audits: entrypoint: worker-audits From bc4fede216cd0a14ecd654005b0b3afa3c825913 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 2 Aug 2021 12:41:12 +0545 Subject: [PATCH 04/45] refactor usage update and fix functions worker --- app/controllers/shared/api.php | 69 +++++++++++----------------------- app/workers/functions.php | 35 +++++++++-------- docker-compose.yml | 2 + 3 files changed, 43 insertions(+), 63 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 901178a94e..29333f6488 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -218,56 +218,31 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits && $project->getId() && $mode !== APP_MODE_ADMIN //TODO: add check to make sure user is admin && !empty($route->getLabel('sdk.namespace', null))) { // Don't calculate console usage on admin mode + + $storage = $usage->getParam('storage') ?? 0; + + $networkRequestSize = $request->getSize() + $usage->getParam('storage'); + $networkResponseSize = $response->getSize(); - $usage - ->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage')) - ->setParam('networkResponseSize', $response->getSize()); + $httpMethod = $usage->getParam('httpMethod') ?? ''; + $httpRequest = $usage->getParam('httpRequest') ?? 0; - statsdUpdate($statsd, $usage); + $tags = ",project={$project->getId()},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); + // the global namespace is prepended to every key (optional) + $statsd->setNamespace('appwrite.usage'); + if($httpRequest >= 1) { + $statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod)); + } + + $statsd->count('network.inbound'.$tags, $networkRequestSize); + $statsd->count('network.outbound'.$tags, $networkResponseSize); + $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize); + + if($storage >= 1) { + $statsd->count('storage.all'.$tags, $storage); + } } -}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'statsd', 'usage', 'deletes', 'database', 'mode'], 'api'); - -function statsdUpdate($statsd, $usage): void -{ - /** @var Appwrite\Event\Event $usage */ - - $projectId = $usage->getParam('projectId') ?? ''; - - $storage = $usage->getParam('storage') ?? 0; - - $networkRequestSize = $usage->getParam('networkRequestSize') ?? 0; - $networkResponseSize = $usage->getParam('networkResponseSize') ?? 0; - - $httpMethod = $usage->getParam('httpMethod') ?? ''; - $httpRequest = $usage->getParam('httpRequest') ?? 0; - - $functionId = $usage->getParam('functionId') ?? ''; - $functionExecution = $usage->getParam('functionExecution') ?? 0; - $functionExecutionTime = $usage->getParam('functionExecutionTime') ?? 0; - $functionStatus = $usage->getParam('functionStatus') ?? ''; - - $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); - - // the global namespace is prepended to every key (optional) - $statsd->setNamespace('appwrite.usage'); - - if($httpRequest >= 1) { - $statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod)); - } - - if($functionExecution >= 1) { - $statsd->increment('executions.all'.$tags.',functionId='.$functionId.',functionStatus='.$functionStatus); - $statsd->count('executions.time'.$tags.',functionId='.$functionId, $functionExecutionTime); - } - - $statsd->count('network.inbound'.$tags, $networkRequestSize); - $statsd->count('network.outbound'.$tags, $networkResponseSize); - $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize); - - if($storage >= 1) { - $statsd->count('storage.all'.$tags, $storage); - } -} \ No newline at end of file +}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'statsd', 'usage', 'deletes', 'database', 'mode'], 'api'); \ No newline at end of file diff --git a/app/workers/functions.php b/app/workers/functions.php index 510346fb48..6941c69765 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -134,8 +134,6 @@ class FunctionsV1 extends Worker public function run(): void { - global $register; - $projectId = $this->args['projectId'] ?? ''; $functionId = $this->args['functionId'] ?? ''; $webhooks = $this->args['webhooks'] ?? []; @@ -279,7 +277,7 @@ class FunctionsV1 extends Worker */ public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void { - global $list; + global $list, $register; $runtimes = Config::getParam('runtimes'); @@ -477,21 +475,26 @@ class FunctionsV1 extends Worker ->setParam('eventData', $execution->getArrayCopy(array_keys($executionModel->getRules()))); $executionUpdate->trigger(); - - $usage = new Event('v1-usage', 'UsageV1'); - - $usage - ->setParam('projectId', $projectId) - ->setParam('functionId', $function->getId()) - ->setParam('functionExecution', 1) - ->setParam('functionStatus', $functionStatus) - ->setParam('functionExecutionTime', $executionTime * 1000) // ms - ->setParam('networkRequestSize', 0) - ->setParam('networkResponseSize', 0) - ; if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $usage->trigger(); + $statsd = $register->get('statsd'); + + $storage = 0; + + $functionExecutionTime = $executionTime * 1000; + + $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); + + // the global namespace is prepended to every key (optional) + $statsd->setNamespace('appwrite.usage'); + + $statsd->increment('executions.all'.$tags.',functionId='.$function->getId().',functionStatus='.$functionStatus); + $statsd->count('executions.time'.$tags.',functionId='.$function->getId(), $functionExecutionTime); + + if($storage >= 1) { + $statsd->count('storage.all'.$tags, $storage); + } + } $this->cleanup(); diff --git a/docker-compose.yml b/docker-compose.yml index 89c88c68d2..51ecde502e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -315,6 +315,8 @@ services: - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP - _APP_USAGE_STATS + - _APP_STATSD_HOST + - _APP_STATSD_PORT - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD From afe772f8fc02d7ead00a75f74de0cfe2b2cbe8f3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 2 Aug 2021 14:51:56 +0545 Subject: [PATCH 05/45] fix worker statsd not found issue --- app/controllers/shared/api.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 29333f6488..c1a1f8c827 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -169,7 +169,7 @@ App::init(function ($utopia, $request, $response, $project, $user) { }, ['utopia', 'request', 'response', 'project', 'user'], 'auth'); -App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $statsd, $usage, $deletes, $database, $mode) { +App::shutdown(function ($utopia, $request, $response, $project, $register, $events, $audits, $usage, $deletes, $database, $mode) { /** @var Utopia\App $utopia */ /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ @@ -228,7 +228,8 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits $httpRequest = $usage->getParam('httpRequest') ?? 0; $tags = ",project={$project->getId()},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); - + + $statsd = $register->get('statsd'); // the global namespace is prepended to every key (optional) $statsd->setNamespace('appwrite.usage'); @@ -245,4 +246,4 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits } } -}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'statsd', 'usage', 'deletes', 'database', 'mode'], 'api'); \ No newline at end of file +}, ['utopia', 'request', 'response', 'project', 'register', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api'); \ No newline at end of file From 23caa625505706deda9f2da3094f0deebd0edbed Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 2 Aug 2021 15:14:53 +0545 Subject: [PATCH 06/45] remove unused params --- app/workers/functions.php | 7 ------- 1 file changed, 7 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 6941c69765..676c784e5a 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -479,8 +479,6 @@ class FunctionsV1 extends Worker if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $statsd = $register->get('statsd'); - $storage = 0; - $functionExecutionTime = $executionTime * 1000; $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); @@ -490,11 +488,6 @@ class FunctionsV1 extends Worker $statsd->increment('executions.all'.$tags.',functionId='.$function->getId().',functionStatus='.$functionStatus); $statsd->count('executions.time'.$tags.',functionId='.$function->getId(), $functionExecutionTime); - - if($storage >= 1) { - $statsd->count('storage.all'.$tags, $storage); - } - } $this->cleanup(); From 01d0ce698825c16a233a1c3944763a0de650973d Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 5 Aug 2021 21:01:00 +0200 Subject: [PATCH 07/45] feat(database): after pagination --- app/controllers/api/database.php | 14 ++- composer.json | 2 +- composer.lock | 30 +++-- tests/e2e/Services/Database/DatabaseBase.php | 109 +++++++++++++++++++ tests/e2e/Services/Storage/StorageBase.php | 4 +- 5 files changed, 143 insertions(+), 16 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 649c346452..00d86b7007 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1056,12 +1056,12 @@ App::get('/v1/database/collections/:collectionId/documents') ->param('queries', [], new ArrayList(new Text(128)), 'Array of query strings.', true) ->param('limit', 25, new Range(0, 100), 'Maximum number of documents to return in response. Use this value to manage pagination. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 900000000), 'Offset value. The default value is 0. Use this param to manage pagination.', true) - // TODO@kodumbeats 'after' param for pagination ->param('orderAttributes', [], new ArrayList(new Text(128)), 'Array of attributes used to sort results.', true) ->param('orderTypes', [], new ArrayList(new WhiteList(['DESC', 'ASC'], true)), 'Array of order directions for sorting attribtues. Possible values are DESC for descending order, or ASC for ascending order.', true) + ->param('orderAfter', '', new UID(), 'ID of the document used to return documents listed after. Should be used for efficient pagination working with many documents.', true) ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $response, $dbForExternal) { + ->action(function ($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $orderAfter, $response, $dbForExternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ @@ -1087,7 +1087,15 @@ App::get('/v1/database/collections/:collectionId/documents') throw new Exception($validator->getDescription(), 400); } - $documents = $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes); + if (!empty($orderAfter)) { + $orderAfterDocument = $dbForExternal->getDocument($collectionId, $orderAfter); + + if ($orderAfterDocument->isEmpty()) { + throw new Exception('Document for orderAfter not found', 400); + } + } + + $documents = $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $orderAfterDocument ?? null); $response->dynamic(new Document([ 'sum' => \count($documents), diff --git a/composer.json b/composer.json index f595bfd1a8..47cfe2d005 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "utopia-php/cache": "0.4.*", "utopia-php/cli": "0.11.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.6.*", + "utopia-php/database": "dev-main as 0.6.0", "utopia-php/locale": "0.3.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index b43adc9166..a1bd64bb84 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f9727be2725e573a92b2d1aeeb77b421", + "content-hash": "1669de851a29702eba426a09a6871921", "packages": [ { "name": "adhocore/jwt", @@ -1984,16 +1984,16 @@ }, { "name": "utopia-php/database", - "version": "0.6.0", + "version": "dev-main", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "561adc215fce3bd3b8c3ebb971ca354fb1526f26" + "reference": "c48b60884f63a547520ecef35714827af1870bc4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/561adc215fce3bd3b8c3ebb971ca354fb1526f26", - "reference": "561adc215fce3bd3b8c3ebb971ca354fb1526f26", + "url": "https://api.github.com/repos/utopia-php/database/zipball/c48b60884f63a547520ecef35714827af1870bc4", + "reference": "c48b60884f63a547520ecef35714827af1870bc4", "shasum": "" }, "require": { @@ -2011,6 +2011,7 @@ "utopia-php/cli": "^0.11.0", "vimeo/psalm": "4.0.1" }, + "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -2041,9 +2042,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.6.0" + "source": "https://github.com/utopia-php/database/tree/main" }, - "time": "2021-08-03T15:13:48+00:00" + "time": "2021-08-04T16:53:44+00:00" }, { "name": "utopia-php/domains", @@ -6254,9 +6255,18 @@ "time": "2015-12-17T08:42:14+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-main", + "alias": "0.6.0", + "alias_normalized": "0.6.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -6278,5 +6288,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.1.0" + "plugin-api-version": "2.0.0" } diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index 755321ed73..cae46e3ffe 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -289,6 +289,115 @@ trait DatabaseBase return []; } + /** + * @depends testCreateDocument + */ + public function testListDocumentsAfterPagination(array $data):array + { + /** + * Test after without order. + */ + $base = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['moviesId'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals('Captain America', $base['body']['documents'][0]['title']); + $this->assertEquals('Spider-Man: Far From Home', $base['body']['documents'][1]['title']); + $this->assertEquals('Spider-Man: Homecoming', $base['body']['documents'][2]['title']); + $this->assertCount(3, $base['body']['documents']); + + $documents = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['moviesId'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'orderAfter' => $base['body']['documents'][0]['$id'] + ]); + + $this->assertEquals($base['body']['documents'][1]['$id'], $documents['body']['documents'][0]['$id']); + $this->assertEquals($base['body']['documents'][2]['$id'], $documents['body']['documents'][1]['$id']); + $this->assertCount(2, $documents['body']['documents']); + + $documents = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['moviesId'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'orderAfter' => $base['body']['documents'][2]['$id'] + ]); + + $this->assertEmpty($documents['body']['documents']); + + /** + * Test with ASC order and after. + */ + $base = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['moviesId'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'orderAttributes' => ['releaseYear'], + 'orderTypes' => ['ASC'], + ]); + + $this->assertEquals(1944, $base['body']['documents'][0]['releaseYear']); + $this->assertEquals(2017, $base['body']['documents'][1]['releaseYear']); + $this->assertEquals(2019, $base['body']['documents'][2]['releaseYear']); + $this->assertCount(3, $base['body']['documents']); + + $documents = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['moviesId'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'orderAttributes' => ['releaseYear'], + 'orderTypes' => ['ASC'], + 'orderAfter' => $base['body']['documents'][1]['$id'] + ]); + + $this->assertEquals($base['body']['documents'][2]['$id'], $documents['body']['documents'][0]['$id']); + $this->assertCount(1, $documents['body']['documents']); + + /** + * Test with DESC order and after. + */ + $base = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['moviesId'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'orderAttributes' => ['releaseYear'], + 'orderTypes' => ['DESC'], + ]); + + $this->assertEquals(1944, $base['body']['documents'][2]['releaseYear']); + $this->assertEquals(2017, $base['body']['documents'][1]['releaseYear']); + $this->assertEquals(2019, $base['body']['documents'][0]['releaseYear']); + $this->assertCount(3, $base['body']['documents']); + + $documents = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['moviesId'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'orderAttributes' => ['releaseYear'], + 'orderTypes' => ['DESC'], + 'orderAfter' => $base['body']['documents'][1]['$id'] + ]); + + $this->assertEquals($base['body']['documents'][2]['$id'], $documents['body']['documents'][0]['$id']); + $this->assertCount(1, $documents['body']['documents']); + + /** + * Test after with unknown document. + */ + $documents = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['moviesId'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'orderAfter' => 'unknown' + ]); + + $this->assertEquals($documents['headers']['status-code'], 400); + + return []; + } + /** * @depends testCreateDocument */ diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 508866b6a1..6a614c02eb 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -34,7 +34,7 @@ trait StorageBase */ return ['fileId' => $file['body']['$id']]; } - + /** * @depends testCreateFile */ @@ -169,7 +169,7 @@ trait StorageBase /** * Test for FAILURE */ - + return $data; } From fb1546cfccce3df14849338774e8e09b1ba86a89 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 6 Aug 2021 12:34:42 +0200 Subject: [PATCH 08/45] rename orderAfter to after --- app/controllers/api/database.php | 12 ++++++------ composer.lock | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 00d86b7007..283bb77e4d 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1056,12 +1056,12 @@ App::get('/v1/database/collections/:collectionId/documents') ->param('queries', [], new ArrayList(new Text(128)), 'Array of query strings.', true) ->param('limit', 25, new Range(0, 100), 'Maximum number of documents to return in response. Use this value to manage pagination. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 900000000), 'Offset value. The default value is 0. Use this param to manage pagination.', true) + ->param('after', '', new UID(), 'ID of the document used to return documents listed after. Should be used for efficient pagination working with many documents.', true) ->param('orderAttributes', [], new ArrayList(new Text(128)), 'Array of attributes used to sort results.', true) ->param('orderTypes', [], new ArrayList(new WhiteList(['DESC', 'ASC'], true)), 'Array of order directions for sorting attribtues. Possible values are DESC for descending order, or ASC for ascending order.', true) - ->param('orderAfter', '', new UID(), 'ID of the document used to return documents listed after. Should be used for efficient pagination working with many documents.', true) ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $orderAfter, $response, $dbForExternal) { + ->action(function ($collectionId, $queries, $limit, $offset, $after, $orderAttributes, $orderTypes, $response, $dbForExternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ @@ -1087,15 +1087,15 @@ App::get('/v1/database/collections/:collectionId/documents') throw new Exception($validator->getDescription(), 400); } - if (!empty($orderAfter)) { - $orderAfterDocument = $dbForExternal->getDocument($collectionId, $orderAfter); + if (!empty($after)) { + $afterDocument = $dbForExternal->getDocument($collectionId, $after); - if ($orderAfterDocument->isEmpty()) { + if ($afterDocument->isEmpty()) { throw new Exception('Document for orderAfter not found', 400); } } - $documents = $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $orderAfterDocument ?? null); + $documents = $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null); $response->dynamic(new Document([ 'sum' => \count($documents), diff --git a/composer.lock b/composer.lock index d670606358..a56ff12590 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c2bda60e7b774a0c813f52033a268c6c", + "content-hash": "7233a857d191a94d96aa5d0fa59d7bce", "packages": [ { "name": "adhocore/jwt", @@ -1988,12 +1988,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "c48b60884f63a547520ecef35714827af1870bc4" + "reference": "59d9d34164b6fb896bc43085a9a82a292b43473a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/c48b60884f63a547520ecef35714827af1870bc4", - "reference": "c48b60884f63a547520ecef35714827af1870bc4", + "url": "https://api.github.com/repos/utopia-php/database/zipball/59d9d34164b6fb896bc43085a9a82a292b43473a", + "reference": "59d9d34164b6fb896bc43085a9a82a292b43473a", "shasum": "" }, "require": { @@ -2042,9 +2042,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/main" + "source": "https://github.com/utopia-php/database/tree/0.6.1" }, - "time": "2021-08-04T16:53:44+00:00" + "time": "2021-08-05T17:19:16+00:00" }, { "name": "utopia-php/domains", From 99442bdfc399903866358518323c046f8c0fbf42 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 6 Aug 2021 12:36:32 +0200 Subject: [PATCH 09/45] revert utopia-php/database from dev to tag --- composer.json | 2 +- composer.lock | 18 ++++-------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/composer.json b/composer.json index 20f9624768..b3593e34d9 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "utopia-php/cache": "0.4.*", "utopia-php/cli": "0.11.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-main as 0.6.0", + "utopia-php/database": "0.6.*", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index a56ff12590..9a8a5a4f19 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7233a857d191a94d96aa5d0fa59d7bce", + "content-hash": "c2bda60e7b774a0c813f52033a268c6c", "packages": [ { "name": "adhocore/jwt", @@ -1984,7 +1984,7 @@ }, { "name": "utopia-php/database", - "version": "dev-main", + "version": "0.6.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", @@ -2011,7 +2011,6 @@ "utopia-php/cli": "^0.11.0", "vimeo/psalm": "4.0.1" }, - "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -6255,18 +6254,9 @@ "time": "2015-12-17T08:42:14+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-main", - "alias": "0.6.0", - "alias_normalized": "0.6.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From dac093c645647d03ce0d7a18d31602644b0ac972 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 6 Aug 2021 14:35:57 +0200 Subject: [PATCH 10/45] feat(database): add after pagination --- app/controllers/api/database.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 283bb77e4d..ff1edd8a33 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -180,17 +180,26 @@ App::get('/v1/database/collections') ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) ->param('limit', 25, new Range(0, 100), 'Results limit value. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 40000), 'Results offset. The default value is 0. Use this param to manage pagination.', true) + ->param('after', '', new UID(), 'ID of the collection used to return collection listed after. Should be used for efficient pagination working with many collections.', true) ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForExternal') - ->action(function ($search, $limit, $offset, $orderType, $response, $dbForExternal) { + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForExternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; + if (!empty($after)) { + $afterCollection = $dbForExternal->getDocument('collections', $after); + + if ($afterCollection->isEmpty()) { + throw new Exception('Collection for after not found', 400); + } + } + $response->dynamic(new Document([ - 'collections' => $dbForExternal->find(Database::COLLECTIONS, $queries, $limit, $offset, ['_id'], [$orderType]), + 'collections' => $dbForExternal->find(Database::COLLECTIONS, $queries, $limit, $offset, [], [$orderType], $afterCollection ?? null), 'sum' => $dbForExternal->count(Database::COLLECTIONS, $queries, APP_LIMIT_COUNT), ]), Response::MODEL_COLLECTION_LIST); }); From a49b12c542679556000637cea3a49b354f7096dd Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 6 Aug 2021 14:36:05 +0200 Subject: [PATCH 11/45] feat(functions): add after pagination --- app/controllers/api/functions.php | 45 ++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 8d2f28a87a..4adfd81fee 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -85,17 +85,26 @@ App::get('/v1/functions') ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) ->param('limit', 25, new Range(0, 100), 'Results limit value. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 2000), 'Results offset. The default value is 0. Use this param to manage pagination.', true) + ->param('after', '', new UID(), 'ID of the function used to return functions listed after. Should be used for efficient pagination working with many functions.', true) ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $orderType, $response, $dbForInternal) { + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; + if (!empty($after)) { + $afterFunction = $dbForInternal->getDocument('functions', $after); + + if ($afterFunction->isEmpty()) { + throw new Exception('Function for after not found', 400); + } + } + $response->dynamic(new Document([ - 'functions' => $dbForInternal->find('functions', $queries, $limit, $offset, ['_id'], [$orderType]), + 'functions' => $dbForInternal->find('functions', $queries, $limit, $offset, [], [$orderType], $afterFunction ?? null), 'sum' => $dbForInternal->count('functions', $queries, APP_LIMIT_COUNT), ]), Response::MODEL_FUNCTION_LIST); }); @@ -502,10 +511,11 @@ App::get('/v1/functions/:functionId/tags') ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) ->param('limit', 25, new Range(0, 100), 'Results limit value. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 2000), 'Results offset. The default value is 0. Use this param to manage pagination.', true) + ->param('after', '', new UID(), 'ID of the tag used to return tags listed after. Should be used for efficient pagination working with many tags.', true) ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($functionId, $search, $limit, $offset, $orderType, $response, $dbForInternal) { + ->action(function ($functionId, $search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -516,8 +526,16 @@ App::get('/v1/functions/:functionId/tags') } $queries[] = new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]); - - $results = $dbForInternal->find('tags', $queries, $limit, $offset, ['_id'], [$orderType]); + + if (!empty($after)) { + $afterTag = $dbForInternal->getDocument('tags', $after); + + if ($afterTag->isEmpty()) { + throw new Exception('Tag for after not found', 400); + } + } + + $results = $dbForInternal->find('tags', $queries, $limit, $offset, [], [$orderType], $afterTag ?? null); $sum = $dbForInternal->count('tags', $queries, APP_LIMIT_COUNT); $response->dynamic(new Document([ @@ -743,12 +761,13 @@ App::get('/v1/functions/:functionId/executions') ->param('functionId', '', new UID(), 'Function unique ID.') ->param('limit', 25, new Range(0, 100), 'Results limit value. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 2000), 'Results offset. The default value is 0. Use this param to manage pagination.', true) + ->param('after', '', new UID(), 'ID of the execution used to return executions listed after. Should be used for efficient pagination working with many executions.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($functionId, $limit, $offset, $response, $dbForInternal) { + ->action(function ($functionId, $limit, $offset, $after, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - + Authorization::disable(); $function = $dbForInternal->getDocument('functions', $functionId); Authorization::reset(); @@ -757,10 +776,18 @@ App::get('/v1/functions/:functionId/executions') throw new Exception('Function not found', 404); } + if (!empty($after)) { + $afterExecution = $dbForInternal->getDocument('executions', $after); + + if ($afterExecution->isEmpty()) { + throw new Exception('Execution for after not found', 400); + } + } + $results = $dbForInternal->find('executions', [ new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]), - ], $limit, $offset, ['_id'], [Database::ORDER_DESC]); - + ], $limit, $offset, [], [Database::ORDER_DESC], $afterExecution ?? null); + $sum = $dbForInternal->count('executions', [ new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]), ], APP_LIMIT_COUNT); From ca9dafddae229532d2a6d9e4aa26273e5db81ad8 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 6 Aug 2021 14:36:17 +0200 Subject: [PATCH 12/45] feat(projects): add after pagination --- app/controllers/api/projects.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 44fa614adb..030ec3a76e 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -166,16 +166,25 @@ App::get('/v1/projects') ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) ->param('limit', 25, new Range(0, 100), 'Results limit value. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 2000), 'Results offset. The default value is 0. Use this param to manage pagination.', true) + ->param('after', '', new UID(), 'ID of the project used to return projects listed after. Should be used for efficient pagination working with many projects.', true) ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForConsole') - ->action(function ($search, $limit, $offset, $orderType, $response, $dbForConsole) { + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForConsole) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; - $results = $dbForConsole->find('projects', $queries, $limit, $offset, ['_id'], [$orderType]); + if (!empty($after)) { + $afterProject = $dbForConsole->getDocument('projects', $after); + + if ($afterProject->isEmpty()) { + throw new Exception('Project for after not found', 400); + } + } + + $results = $dbForConsole->find('projects', $queries, $limit, $offset, [], [$orderType], $afterProject ?? null); $sum = $dbForConsole->count('projects', $queries, APP_LIMIT_COUNT); $response->dynamic(new Document([ From 8e6c415d01f88474cd1244db721c5b40ee8aa42d Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 6 Aug 2021 14:36:27 +0200 Subject: [PATCH 13/45] feat(storage): add after pagination --- app/controllers/api/storage.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 552b0a9df1..cf7ec10fbd 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -168,17 +168,26 @@ App::get('/v1/storage/files') ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) ->param('limit', 25, new Range(0, 100), 'Results limit value. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 2000), 'Results offset. The default value is 0. Use this param to manage pagination.', true) + ->param('after', '', new UID(), 'ID of the file used to return files listed after. Should be used for efficient pagination working with many files.', true) ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $orderType, $response, $dbForInternal) { + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, $search)] : []; + if (!empty($after)) { + $afterFile = $dbForInternal->getDocument('files', $after); + + if ($afterFile->isEmpty()) { + throw new Exception('File for after not found', 400); + } + } + $response->dynamic(new Document([ - 'files' => $dbForInternal->find('files', $queries, $limit, $offset, ['_id'], [$orderType]), + 'files' => $dbForInternal->find('files', $queries, $limit, $offset, [], [$orderType], $afterFile ?? null), 'sum' => $dbForInternal->count('files', $queries, APP_LIMIT_COUNT), ]), Response::MODEL_FILE_LIST); }); From 79971330f2b739a338b2815f517509e54fa87ae1 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 6 Aug 2021 14:36:35 +0200 Subject: [PATCH 14/45] feat(teams): add after pagination --- app/controllers/api/teams.php | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index b9015267a7..ee08f8cae1 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -97,16 +97,25 @@ App::get('/v1/teams') ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) ->param('limit', 25, new Range(0, 100), 'Results limit value. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 2000), 'Results offset. The default value is 0. Use this param to manage pagination.', true) + ->param('after', '', new UID(), 'ID of the team used to return teams listed after. Should be used for efficient pagination working with many teams.', true) ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $orderType, $response, $dbForInternal) { + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; - $results = $dbForInternal->find('teams', $queries, $limit, $offset, ['_id'], [$orderType]); + if (!empty($after)) { + $afterTeam = $dbForInternal->getDocument('teams', $after); + + if ($afterTeam->isEmpty()) { + throw new Exception('Team for after not found', 400); + } + } + + $results = $dbForInternal->find('teams', $queries, $limit, $offset, [], [$orderType], $afterTeam ?? null); $sum = $dbForInternal->count('teams', $queries, APP_LIMIT_COUNT); $response->dynamic(new Document([ @@ -413,10 +422,11 @@ App::get('/v1/teams/:teamId/memberships') ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) ->param('limit', 25, new Range(0, 100), 'Results limit value. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 2000), 'Results offset. The default value is 0. Use this param to manage pagination.', true) + ->param('after', '', new UID(), 'ID of the file used to return files listed after. Should be used for efficient pagination working with many files.', true) ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($teamId, $search, $limit, $offset, $orderType, $response, $dbForInternal) { + ->action(function ($teamId, $search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -426,7 +436,15 @@ App::get('/v1/teams/:teamId/memberships') throw new Exception('Team not found', 404); } - $memberships = $dbForInternal->find('memberships', [new Query('teamId', Query::TYPE_EQUAL, [$teamId])], $limit, $offset, ['_id'], [$orderType]); + if (!empty($after)) { + $afterMembership = $dbForInternal->getDocument('memberships', $after); + + if ($afterMembership->isEmpty()) { + throw new Exception('Membership for after not found', 400); + } + } + + $memberships = $dbForInternal->find('memberships', [new Query('teamId', Query::TYPE_EQUAL, [$teamId])], $limit, $offset, [], [$orderType], $afterMembership ?? null); $sum = $dbForInternal->count('memberships', [new Query('teamId', Query::TYPE_EQUAL, [$teamId])], APP_LIMIT_COUNT); $users = []; From af570076122eba98863d1a40814620a2bffd100e Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 6 Aug 2021 14:36:48 +0200 Subject: [PATCH 15/45] feat(users): add after pagination --- app/controllers/api/users.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 8c30b8b1c4..bff1373f3e 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -81,14 +81,23 @@ App::get('/v1/users') ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) ->param('limit', 25, new Range(0, 100), 'Results limit value. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 2000), 'Results offset. The default value is 0. Use this param to manage pagination.', true) + ->param('after', '', new UID(), 'ID of the user used to return users listed after. Should be used for efficient pagination working with many users.', true) ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $orderType, $response, $dbForInternal) { + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $results = $dbForInternal->find('users', [], $limit, $offset, ['_id'], [$orderType]); + if (!empty($after)) { + $afterUser = $dbForInternal->getDocument('users', $after); + + if ($afterUser->isEmpty()) { + throw new Exception('User for after not found', 400); + } + } + + $results = $dbForInternal->find('users', [], $limit, $offset, [], [$orderType], $afterUser ?? null); $sum = $dbForInternal->count('users', [], APP_LIMIT_COUNT); $response->dynamic(new Document([ From 24614f1fe9f376f7f85795d5977d312419b99b41 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 7 Aug 2021 15:46:22 +0300 Subject: [PATCH 16/45] Simplified Permission handling --- app/controllers/general.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 91b96edb83..fa8f7b9cb5 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -53,7 +53,6 @@ App::init(function ($utopia, $request, $response, $console, $project, $dbForCons 'domain' => $domain->get(), ]); $certificate = $dbForConsole->createDocument('certificates', $certificate); - Authorization::enable(); Console::info('Issuing a TLS certificate for the master domain (' . $domain->get() . ') in a few seconds...'); @@ -63,10 +62,11 @@ App::init(function ($utopia, $request, $response, $console, $project, $dbForCons 'validateTarget' => false, 'validateCNAME' => false, ]); - } else { - Authorization::enable(); // ensure authorization is reenabled } + $domains[$domain->get()] = true; + + Authorization::reset(); // ensure authorization is re-enabled } Config::setParam('domains', $domains); } From 9ad1917072df15ced1b50336d7d73917da279351 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 7 Aug 2021 15:49:36 +0300 Subject: [PATCH 17/45] Permission fix --- app/views/console/functions/function.phtml | 2 +- app/workers/database.php | 4 ++++ src/Appwrite/Utopia/Response/Model/Func.php | 12 ++++++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/app/views/console/functions/function.phtml b/app/views/console/functions/function.phtml index 405fd28eb0..0ec16db908 100644 --- a/app/views/console/functions/function.phtml +++ b/app/views/console/functions/function.phtml @@ -452,7 +452,7 @@ $usageStatsEnabled = $this->getParam('usageStatsEnabled', true); - +
Add * for wildcard access
diff --git a/app/workers/database.php b/app/workers/database.php index 42df89e1b5..60c060ee7a 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -3,6 +3,7 @@ use Appwrite\Resque\Worker; use Utopia\CLI\Console; use Utopia\Database\Document; +use Utopia\Database\Validator\Authorization; require_once __DIR__.'/../workers.php'; @@ -21,6 +22,8 @@ class DatabaseV1 extends Worker { $projectId = $this->args['projectId'] ?? ''; $type = $this->args['type'] ?? ''; + + Authorization::disable(); switch (strval($type)) { case CREATE_TYPE_ATTRIBUTE: @@ -49,6 +52,7 @@ class DatabaseV1 extends Worker break; } + Authorization::reset(); } public function shutdown(): void diff --git a/src/Appwrite/Utopia/Response/Model/Func.php b/src/Appwrite/Utopia/Response/Model/Func.php index 6d5bf8c01a..6c1ba151f6 100644 --- a/src/Appwrite/Utopia/Response/Model/Func.php +++ b/src/Appwrite/Utopia/Response/Model/Func.php @@ -16,12 +16,12 @@ class Func extends Model 'default' => '', 'example' => '5e5ea5c16897e', ]) - ->addRule('$permissions', [ - 'type' => Response::MODEL_PERMISSIONS, - 'description' => 'Function permissions.', - 'default' => new \stdClass, - 'example' => new \stdClass, - 'array' => false, + ->addRule('execute', [ + 'type' => self::TYPE_STRING, + 'description' => 'Document execute permissions.', + 'default' => '', + 'example' => 'role:all', + 'array' => true, ]) ->addRule('name', [ 'type' => self::TYPE_STRING, From ffe9c99157ada92750f048f9205831be42717f1f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 8 Aug 2021 11:21:08 +0545 Subject: [PATCH 18/45] new statsd class and implementation --- app/controllers/shared/api.php | 34 +++---------- app/init.php | 3 +- app/workers/functions.php | 20 +++++--- src/Appwrite/Statsd/Statsd.php | 93 ++++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 36 deletions(-) create mode 100644 src/Appwrite/Statsd/Statsd.php diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index c1a1f8c827..9655b768c1 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -18,7 +18,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e /** @var Utopia\Registry\Registry $register */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Statsd\Statsd $usage */ /** @var Appwrite\Event\Event $deletes */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $functions */ @@ -176,7 +176,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $register, $even /** @var Utopia\Database\Document $project */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Statsd\Statsd $usage */ /** @var Appwrite\Event\Event $deletes */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $functions */ @@ -218,32 +218,12 @@ App::shutdown(function ($utopia, $request, $response, $project, $register, $even && $project->getId() && $mode !== APP_MODE_ADMIN //TODO: add check to make sure user is admin && !empty($route->getLabel('sdk.namespace', null))) { // Don't calculate console usage on admin mode - - $storage = $usage->getParam('storage') ?? 0; - - $networkRequestSize = $request->getSize() + $usage->getParam('storage'); - $networkResponseSize = $response->getSize(); - $httpMethod = $usage->getParam('httpMethod') ?? ''; - $httpRequest = $usage->getParam('httpRequest') ?? 0; - - $tags = ",project={$project->getId()},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); - - $statsd = $register->get('statsd'); - // the global namespace is prepended to every key (optional) - $statsd->setNamespace('appwrite.usage'); - - if($httpRequest >= 1) { - $statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod)); - } - - $statsd->count('network.inbound'.$tags, $networkRequestSize); - $statsd->count('network.outbound'.$tags, $networkResponseSize); - $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize); - - if($storage >= 1) { - $statsd->count('storage.all'.$tags, $storage); - } + $usage + ->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage')) + ->setParam('networkResponseSize', $response->getSize()) + ->save() + ; } }, ['utopia', 'request', 'response', 'project', 'register', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api'); \ No newline at end of file diff --git a/app/init.php b/app/init.php index 9d8413e64d..bb5278445d 100644 --- a/app/init.php +++ b/app/init.php @@ -26,6 +26,7 @@ use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Database\Document; use Appwrite\Event\Event; use Appwrite\OpenSSL\OpenSSL; +use Appwrite\Statsd\Statsd; use Utopia\App; use Utopia\View; use Utopia\Config\Config; @@ -378,7 +379,7 @@ App::setResource('audits', function($register) { }, ['register']); App::setResource('usage', function($register) { - return new Event(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME); + return new Statsd($register->get('statsd')); }, ['register']); App::setResource('mails', function($register) { diff --git a/app/workers/functions.php b/app/workers/functions.php index 676c784e5a..75dc3ca5e9 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -2,6 +2,7 @@ use Appwrite\Event\Event; use Appwrite\Resque\Worker; +use Appwrite\Statsd\Statsd; use Appwrite\Utopia\Response\Model\Execution; use Cron\CronExpression; use Swoole\Runtime; @@ -479,15 +480,18 @@ class FunctionsV1 extends Worker if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $statsd = $register->get('statsd'); - $functionExecutionTime = $executionTime * 1000; + $usage = new Statsd($statsd); - $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); - - // the global namespace is prepended to every key (optional) - $statsd->setNamespace('appwrite.usage'); - - $statsd->increment('executions.all'.$tags.',functionId='.$function->getId().',functionStatus='.$functionStatus); - $statsd->count('executions.time'.$tags.',functionId='.$function->getId(), $functionExecutionTime); + $usage + ->setParam('projectId', $projectId) + ->setParam('functionId', $function->getId()) + ->setParam('functionExecution', 1) + ->setParam('functionStatus', $functionStatus) + ->setParam('functionExecutionTime', $executionTime * 1000) // ms + ->setParam('networkRequestSize', 0) + ->setParam('networkResponseSize', 0) + ->save() + ; } $this->cleanup(); diff --git a/src/Appwrite/Statsd/Statsd.php b/src/Appwrite/Statsd/Statsd.php new file mode 100644 index 0000000000..134b890565 --- /dev/null +++ b/src/Appwrite/Statsd/Statsd.php @@ -0,0 +1,93 @@ +statsd = $statsd; + } + + /** + * @param string $key + * @param mixed $value + * + * @return $this + */ + public function setParam(string $key, $value): self + { + $this->params[$key] = $value; + + return $this; + } + + /** + * Save to statsd. + */ + public function save(): void + { + $projectId = $this->params['projectId'] ?? ''; + + $storage = $this->params['storage'] ?? 0; + + $networkRequestSize = $this->params['networkRequestSize'] ?? 0; + $networkResponseSize = $this->params['networkResponseSize'] ?? 0; + + $httpMethod = $this->params['httpMethod'] ?? ''; + $httpRequest = $this->params['httpRequest'] ?? 0; + + $functionId = $this->params['functionId'] ?? ''; + $functionExecution = $this->params['functionExecution'] ?? 0; + $functionExecutionTime = $this->params['functionExecutionTime'] ?? 0; + $functionStatus = $this->params['functionStatus'] ?? ''; + + $tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); + + // the global namespace is prepended to every key (optional) + $this->statsd->setNamespace('appwrite.usage'); + + if ($httpRequest >= 1) { + $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod)); + } + + if ($functionExecution >= 1) { + $this->statsd->increment('executions.all' . $tags . ',functionId=' . $functionId . ',functionStatus=' . $functionStatus); + $this->statsd->count('executions.time' . $tags . ',functionId=' . $functionId, $functionExecutionTime); + } + + $this->statsd->count('network.inbound' . $tags, $networkRequestSize); + $this->statsd->count('network.outbound' . $tags, $networkResponseSize); + $this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize); + + if ($storage >= 1) { + $this->statsd->count('storage.all' . $tags, $storage); + } + + $this->reset(); + } + + public function reset(): self + { + $this->params = []; + + return $this; + } +} From a8f864a07ddc9b5bcf3ce0e7f2c4a320e98f946e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 8 Aug 2021 12:13:50 +0545 Subject: [PATCH 19/45] Update src/Appwrite/Statsd/Statsd.php Co-authored-by: Eldad A. Fux --- src/Appwrite/Statsd/Statsd.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Statsd/Statsd.php b/src/Appwrite/Statsd/Statsd.php index 134b890565..e1fdbc46e9 100644 --- a/src/Appwrite/Statsd/Statsd.php +++ b/src/Appwrite/Statsd/Statsd.php @@ -40,7 +40,7 @@ class Statsd } /** - * Save to statsd. + * Submit data to StatsD. */ public function save(): void { From fa46f14092b1d097d5b7fa8fa4cda2fa04327e16 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 8 Aug 2021 12:16:20 +0545 Subject: [PATCH 20/45] refactor the class --- app/controllers/shared/api.php | 6 +++--- app/init.php | 1 + app/workers/functions.php | 6 +++--- .../{Statsd/Statsd.php => Stats/Stats.php} | 16 +++++++++++++--- 4 files changed, 20 insertions(+), 9 deletions(-) rename src/Appwrite/{Statsd/Statsd.php => Stats/Stats.php} (89%) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 9655b768c1..a8d5f20cee 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -18,7 +18,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e /** @var Utopia\Registry\Registry $register */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Statsd\Statsd $usage */ + /** @var Appwrite\Stats\Stats $usage */ /** @var Appwrite\Event\Event $deletes */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $functions */ @@ -176,7 +176,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $register, $even /** @var Utopia\Database\Document $project */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Statsd\Statsd $usage */ + /** @var Appwrite\Stats\Stats $usage */ /** @var Appwrite\Event\Event $deletes */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $functions */ @@ -222,7 +222,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $register, $even $usage ->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage')) ->setParam('networkResponseSize', $response->getSize()) - ->save() + ->submit() ; } diff --git a/app/init.php b/app/init.php index bb5278445d..5c20816e43 100644 --- a/app/init.php +++ b/app/init.php @@ -249,6 +249,7 @@ $register->set('statsd', function () { // Register DB connection return $statsd; }); + $register->set('smtp', function () { $mail = new PHPMailer(true); diff --git a/app/workers/functions.php b/app/workers/functions.php index 75dc3ca5e9..1b36b7cdc4 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -2,7 +2,7 @@ use Appwrite\Event\Event; use Appwrite\Resque\Worker; -use Appwrite\Statsd\Statsd; +use Appwrite\Stats\Stats; use Appwrite\Utopia\Response\Model\Execution; use Cron\CronExpression; use Swoole\Runtime; @@ -480,7 +480,7 @@ class FunctionsV1 extends Worker if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $statsd = $register->get('statsd'); - $usage = new Statsd($statsd); + $usage = new Stats($statsd); $usage ->setParam('projectId', $projectId) @@ -490,7 +490,7 @@ class FunctionsV1 extends Worker ->setParam('functionExecutionTime', $executionTime * 1000) // ms ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) - ->save() + ->submit() ; } diff --git a/src/Appwrite/Statsd/Statsd.php b/src/Appwrite/Stats/Stats.php similarity index 89% rename from src/Appwrite/Statsd/Statsd.php rename to src/Appwrite/Stats/Stats.php index e1fdbc46e9..4b9eddf85f 100644 --- a/src/Appwrite/Statsd/Statsd.php +++ b/src/Appwrite/Stats/Stats.php @@ -1,10 +1,10 @@ params[$key])) ? $this->params[$key] : null; + } + /** * Submit data to StatsD. */ - public function save(): void + public function submit(): void { $projectId = $this->params['projectId'] ?? ''; From 8255bdb06a5e9b7a1631646acbb8f644759f299f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 8 Aug 2021 12:24:16 +0545 Subject: [PATCH 21/45] fix issue --- app/init.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index 53272c0276..2394d778c0 100644 --- a/app/init.php +++ b/app/init.php @@ -29,7 +29,7 @@ use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; -use Appwrite\Statsd\Statsd; +use Appwrite\Stats\Stats; use Utopia\App; use Utopia\View; use Utopia\Config\Config; @@ -423,7 +423,7 @@ App::setResource('audits', function($register) { }, ['register']); App::setResource('usage', function($register) { - return new Statsd($register->get('statsd')); + return new Stats($register->get('statsd')); }, ['register']); App::setResource('mails', function($register) { From 72948624ee6cc416d27524c97031557ca8d49cbc Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 8 Aug 2021 14:09:23 +0545 Subject: [PATCH 22/45] simple unit test for stats class --- tests/unit/Stats/StatsTest.php | 61 ++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/unit/Stats/StatsTest.php diff --git a/tests/unit/Stats/StatsTest.php b/tests/unit/Stats/StatsTest.php new file mode 100644 index 0000000000..ad660330ab --- /dev/null +++ b/tests/unit/Stats/StatsTest.php @@ -0,0 +1,61 @@ +object = new Stats($statsd); + } + + public function tearDown(): void + { + } + + public function testParams() + { + $this->object + ->setParam('statsKey1', 'statsValue1') + ->setParam('statsKey2', 'statsValue2') + ; + + $this->object->submit(); + + $this->assertEquals(null, $this->object->getParam('statsKey1')); + $this->assertEquals(null, $this->object->getParam('statsKey2')); + $this->assertEquals(null, $this->object->getParam('statsKey3')); + } + + public function testReset() + { + $this->object + ->setParam('statsKey1', 'statsValue1') + ->setParam('statsKey2', 'statsValue2') + ; + + $this->assertEquals('statsValue1', $this->object->getParam('statsKey1')); + $this->assertEquals('statsValue2', $this->object->getParam('statsKey2')); + + $this->object->reset(); + + $this->assertEquals(null, $this->object->getParam('statsKey1')); + $this->assertEquals(null, $this->object->getParam('statsKey2')); + $this->assertEquals(null, $this->object->getParam('statsKey3')); + } +} From d73ddb581c1fd5f759442dec0108fcd2f9de11c2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 8 Aug 2021 15:08:44 +0545 Subject: [PATCH 23/45] added namespace and modified tests --- src/Appwrite/Stats/Stats.php | 28 +++++++++++++++++++++++++++- tests/unit/Stats/StatsTest.php | 34 +++++++++++++++++++++------------- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 4b9eddf85f..b07c4c85c5 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -16,6 +16,11 @@ class Stats */ protected $statsd; + /** + * @var string + */ + protected $namespace = 'appwrite.usage'; + /** * Event constructor. * @@ -49,6 +54,26 @@ class Stats return (isset($this->params[$key])) ? $this->params[$key] : null; } + /** + * @param string $namespace + * + * @return $this + */ + public function setNamespace(string $namespace): self + { + $this->namespace = $namespace; + + return $this; + } + + /** + * @return string + */ + public function getNamespace() + { + return $this->namespace; + } + /** * Submit data to StatsD. */ @@ -72,7 +97,7 @@ class Stats $tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); // the global namespace is prepended to every key (optional) - $this->statsd->setNamespace('appwrite.usage'); + $this->statsd->setNamespace($this->namespace); if ($httpRequest >= 1) { $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod)); @@ -97,6 +122,7 @@ class Stats public function reset(): self { $this->params = []; + $this->namespace = 'appwrite.usage'; return $this; } diff --git a/tests/unit/Stats/StatsTest.php b/tests/unit/Stats/StatsTest.php index ad660330ab..784ac77fb4 100644 --- a/tests/unit/Stats/StatsTest.php +++ b/tests/unit/Stats/StatsTest.php @@ -20,7 +20,7 @@ class StatsTest extends TestCase $connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port); $statsd = new \Domnikl\Statsd\Client($connection); - + $this->object = new Stats($statsd); } @@ -28,34 +28,42 @@ class StatsTest extends TestCase { } + public function testNamespace() + { + $this->object->setNamespace('appwritetest.usage'); + $this->assertEquals('appwritetest.usage', $this->object->getNamespace()); + } + public function testParams() { $this->object - ->setParam('statsKey1', 'statsValue1') - ->setParam('statsKey2', 'statsValue2') + ->setParam('projectId', 'appwrite_test') + ->setParam('networkRequestSize', 100) ; + $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); + $this->assertEquals(100, $this->object->getParam('networkRequestSize')); + $this->object->submit(); - $this->assertEquals(null, $this->object->getParam('statsKey1')); - $this->assertEquals(null, $this->object->getParam('statsKey2')); - $this->assertEquals(null, $this->object->getParam('statsKey3')); + $this->assertEquals(null, $this->object->getParam('projectId')); + $this->assertEquals(null, $this->object->getParam('networkRequestSize')); } public function testReset() { $this->object - ->setParam('statsKey1', 'statsValue1') - ->setParam('statsKey2', 'statsValue2') + ->setParam('projectId', 'appwrite_test') + ->setParam('networkRequestSize', 100) ; - $this->assertEquals('statsValue1', $this->object->getParam('statsKey1')); - $this->assertEquals('statsValue2', $this->object->getParam('statsKey2')); + $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); + $this->assertEquals(100, $this->object->getParam('networkRequestSize')); $this->object->reset(); - $this->assertEquals(null, $this->object->getParam('statsKey1')); - $this->assertEquals(null, $this->object->getParam('statsKey2')); - $this->assertEquals(null, $this->object->getParam('statsKey3')); + $this->assertEquals(null, $this->object->getParam('projectId')); + $this->assertEquals(null, $this->object->getParam('networkRequestSize')); + $this->assertEquals('appwrite.usage', $this->object->getNamespace()); } } From e6099546597ef5e915fcdc240c8d1b173d766c42 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Sun, 8 Aug 2021 18:07:59 -0400 Subject: [PATCH 24/45] Fix logic for finding matching attribute --- app/controllers/api/database.php | 24 ++++++++++++------------ bin/worker-database | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 1ebb857da3..939265c270 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -717,21 +717,21 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') throw new Exception('Collection not found', 404); } - $attributes = $collection->getAttributes(); + /** @var Document[] $attributes */ + $attributes = $collection->getAttribute('attributes'); - // Search for attribute - $attributeIndex = array_search($attributeId, array_column($attributes, '$id')); - - if ($attributeIndex === false) { - throw new Exception('Attribute not found', 404); + // find attribute in collection + $attribute = null; + foreach ($attributes as $a) { + if ($a->getId() === $attributeId) { + $attribute = $a->setAttribute('$collection', $collectionId); // set the collectionId + break; // break once attribute is found + } } - $attribute = new Document([\array_merge($attributes[$attributeIndex], [ - 'collectionId' => $collectionId, - ])]); - - $type = $attribute->getAttribute('type', ''); - $format = $attribute->getAttribute('format', ''); + if (\is_null($attribute)) { + throw new Exception('Attribute not found', 404); + } $database ->setParam('type', DELETE_TYPE_ATTRIBUTE) diff --git a/bin/worker-database b/bin/worker-database index 97e067d0d5..3dfbeaaad4 100644 --- a/bin/worker-database +++ b/bin/worker-database @@ -7,4 +7,4 @@ else REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" fi -QUEUE='v1-database' APP_INCLUDE='/usr/src/code/app/workers/database.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file +INTERVAL=0.1 QUEUE='v1-database' APP_INCLUDE='/usr/src/code/app/workers/database.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file From a42588f3340f9d94a15e305c3cf879c9b5656e02 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Sun, 8 Aug 2021 18:08:10 -0400 Subject: [PATCH 25/45] Test for attribute deletion --- .../Database/DatabaseCustomServerTest.php | 80 +++++++++++++++++-- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index dea3b588c7..4e559fad8f 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -13,7 +13,7 @@ class DatabaseCustomServerTest extends Scope use ProjectCustom; use SideServer; - public function testDeleteCollection() + public function testDeleteAttribute() { /** * Test for SUCCESS @@ -54,7 +54,59 @@ class DatabaseCustomServerTest extends Scope 'required' => true, ]); - // wait for database worker to finish creating attributes + $unneeded = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'unneeded', + 'size' => 256, + 'required' => true, + ]); + + // Wait for database worker to finish creating attributes + sleep(5); + + $index = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'indexId' => 'key_lastName', + 'type' => 'key', + 'attributes' => [ + 'lastName', + ], + ]); + + // Wait for database worker to finish creating index + sleep(5); + + $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $actors['body']['$id'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), []); + + $unneededId = $unneeded['body']['$id']; + + $this->assertEquals($collection['body']['$id'], $firstName['body']['$collection']); + $this->assertEquals($collection['body']['$id'], $lastName['body']['$collection']); + $this->assertIsArray($collection['body']['attributes']); + $this->assertCount(3, $collection['body']['attributes']); + $this->assertEquals($collection['body']['attributes'][0]['$id'], $firstName['body']['$id']); + $this->assertEquals($collection['body']['attributes'][1]['$id'], $lastName['body']['$id']); + $this->assertEquals($collection['body']['attributes'][2]['$id'], $unneeded['body']['$id']); + $this->assertCount(1, $collection['body']['indexes']); + $this->assertEquals($collection['body']['indexes'][0]['$id'], $index['body']['$id']); + + // Delete attribute + $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $actors ['body']['$id'] . '/attributes/' . $unneededId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + sleep(5); $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $actors['body']['$id'], array_merge([ @@ -70,8 +122,20 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($collection['body']['attributes'][0]['$id'], $firstName['body']['$id']); $this->assertEquals($collection['body']['attributes'][1]['$id'], $lastName['body']['$id']); + return [ + 'collectionId' => $actors['body']['$id'], + ]; + } + + /** + * @depends testDeleteAttribute + */ + public function testDeleteCollection($data) + { + $collectionId = $data['collectionId']; + // Add Documents to the collection - $document1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/documents', array_merge([ + $document1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/documents', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ @@ -84,7 +148,7 @@ class DatabaseCustomServerTest extends Scope 'write' => ['user:'.$this->getUser()['$id']], ]); - $document2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/documents', array_merge([ + $document2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/documents', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ @@ -98,7 +162,7 @@ class DatabaseCustomServerTest extends Scope ]); $this->assertEquals($document1['headers']['status-code'], 201); - $this->assertEquals($document1['body']['$collection'], $actors['body']['$id']); + $this->assertEquals($document1['body']['$collection'], $collectionId); $this->assertIsArray($document1['body']['$read']); $this->assertIsArray($document1['body']['$write']); $this->assertCount(1, $document1['body']['$read']); @@ -107,7 +171,7 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($document1['body']['lastName'], 'Holland'); $this->assertEquals($document2['headers']['status-code'], 201); - $this->assertEquals($document2['body']['$collection'], $actors['body']['$id']); + $this->assertEquals($document2['body']['$collection'], $collectionId); $this->assertIsArray($document2['body']['$read']); $this->assertIsArray($document2['body']['$write']); $this->assertCount(1, $document2['body']['$read']); @@ -116,7 +180,7 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($document2['body']['lastName'], 'Jackson'); // Delete the actors collection - $response = $this->client->call(Client::METHOD_DELETE, '/database/collections/'.$actors['body']['$id'], array_merge([ + $response = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId , array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -126,7 +190,7 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($response['body'],""); // Try to get the collection and check if it has been deleted - $response = $this->client->call(Client::METHOD_GET, '/database/collections/'.$actors['body']['$id'], array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId , array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] ], $this->getHeaders())); From 4c08097e92e1d0440daf6a61d6925f1a2d0ef965 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Sun, 8 Aug 2021 19:42:08 -0400 Subject: [PATCH 26/45] Use system for naming worker constants --- app/init.php | 14 +++++++------- app/workers/database.php | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/app/init.php b/app/init.php index 72b3dedc7e..4ed09857fa 100644 --- a/app/init.php +++ b/app/init.php @@ -76,18 +76,18 @@ const APP_SOCIAL_DISCORD = 'https://appwrite.io/discord'; const APP_SOCIAL_DISCORD_CHANNEL = '564160730845151244'; const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; -// Creation Types -const CREATE_TYPE_ATTRIBUTE = 'newAttribute'; -const CREATE_TYPE_INDEX = 'newIndex'; -// Deletion Types -const DELETE_TYPE_ATTRIBUTE = 'attribute'; -const DELETE_TYPE_INDEX = 'index'; +// Database Worker Types +const DATABASE_TYPE_CREATE_ATTRIBUTE = 'createAttribute'; +const DATABASE_TYPE_CREATE_INDEX = 'createIndex'; +const DATABASE_TYPE_DELETE_ATTRIBUTE = 'deleteAttribute'; +const DATABASE_TYPE_DELETE_INDEX = 'deleteIndex'; +// Deletes Worker Types const DELETE_TYPE_DOCUMENT = 'document'; const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; -// Mail Types +// Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery'; const MAIL_TYPE_INVITATION = 'invitation'; diff --git a/app/workers/database.php b/app/workers/database.php index 42df89e1b5..ca03511191 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -23,22 +23,22 @@ class DatabaseV1 extends Worker $type = $this->args['type'] ?? ''; switch (strval($type)) { - case CREATE_TYPE_ATTRIBUTE: + case DATABASE_TYPE_CREATE_ATTRIBUTE: $attribute = $this->args['document'] ?? ''; $attribute = new Document($attribute); $this->createAttribute($attribute, $projectId); break; - case DELETE_TYPE_ATTRIBUTE: + case DATABASE_TYPE_DELETE_ATTRIBUTE: $attribute = $this->args['document'] ?? ''; $attribute = new Document($attribute); $this->deleteAttribute($attribute, $projectId); break; - case CREATE_TYPE_INDEX: + case DATABASE_TYPE_CREATE_INDEX: $index = $this->args['document'] ?? ''; $index = new Document($index); $this->createIndex($index, $projectId); break; - case DELETE_TYPE_INDEX: + case DATABASE_TYPE_DELETE_INDEX: $index = $this->args['document'] ?? ''; $index = new Document($index); $this->deleteIndex($index, $projectId); From ae95d1b83a7a7ad5e69a44fa4595edd70a30147e Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Sun, 8 Aug 2021 19:46:14 -0400 Subject: [PATCH 27/45] Fix logic for known indexes --- app/controllers/api/database.php | 33 +++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 939265c270..75eab37142 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -106,7 +106,7 @@ $attributesCallback = function ($attribute, $response, $dbForExternal, $database ]); $database - ->setParam('type', CREATE_TYPE_ATTRIBUTE) + ->setParam('type', DATABASE_TYPE_CREATE_ATTRIBUTE) ->setParam('document', $attribute) ; @@ -734,7 +734,7 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') } $database - ->setParam('type', DELETE_TYPE_ATTRIBUTE) + ->setParam('type', DATABASE_TYPE_DELETE_ATTRIBUTE) ->setParam('document', $attribute) ; @@ -764,7 +764,7 @@ App::post('/v1/database/collections/:collectionId/indexes') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_INDEX) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') - ->param('id', null, new Key(), 'Index ID.') + ->param('indexId', null, new Key(), 'Index ID.') ->param('type', null, new WhiteList([Database::INDEX_KEY, Database::INDEX_FULLTEXT, Database::INDEX_UNIQUE, Database::INDEX_SPATIAL, Database::INDEX_ARRAY]), 'Index type.') ->param('attributes', null, new ArrayList(new Key()), 'Array of attributes to index.') ->param('orders', [], new ArrayList(new WhiteList(['ASC', 'DESC'], false, Database::VAR_STRING)), 'Array of index orders.', true) @@ -772,7 +772,7 @@ App::post('/v1/database/collections/:collectionId/indexes') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $id, $type, $attributes, $orders, $response, $dbForExternal, $database, $audits) { + ->action(function ($collectionId, $indexId, $type, $attributes, $orders, $response, $dbForExternal, $database, $audits) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $database */ @@ -808,7 +808,7 @@ App::post('/v1/database/collections/:collectionId/indexes') $lengths[$key] = ($attributeType === Database::VAR_STRING) ? $attributeSize : null; } - $success = $dbForExternal->addIndexInQueue($collectionId, $id, $type, $attributes, $lengths, $orders); + $success = $dbForExternal->addIndexInQueue($collectionId, $indexId, $type, $attributes, $lengths, $orders); // Database->createIndex() does not return a document // So we need to create one for the response @@ -816,7 +816,7 @@ App::post('/v1/database/collections/:collectionId/indexes') // TODO@kodumbeats should $lengths be a part of the response model? $index = new Document([ '$collection' => $collectionId, - '$id' => $id, + '$id' => $indexId, 'type' => $type, 'attributes' => $attributes, 'lengths' => $lengths, @@ -824,7 +824,7 @@ App::post('/v1/database/collections/:collectionId/indexes') ]); $database - ->setParam('type', CREATE_TYPE_INDEX) + ->setParam('type', DATABASE_TYPE_CREATE_INDEX) ->setParam('document', $index) ; @@ -949,21 +949,24 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') throw new Exception('Collection not found', 404); } + /** @var Document[] $indexes */ $indexes = $collection->getAttribute('indexes'); - // // Search for index - $indexIndex = array_search($indexId, array_column($indexes, '$id')); + // find attribute in collection + $index= null; + foreach ($indexes as $i) { + if ($i->getId() === $indexId) { + $index = $i->setAttribute('$collection', $collectionId); // set the collectionId + break; // break once index is found + } + } - if ($indexIndex === false) { + if (\is_null($index)) { throw new Exception('Index not found', 404); } - $index = new Document([\array_merge($indexes[$indexIndex], [ - 'collectionId' => $collectionId, - ])]); - $database - ->setParam('type', DELETE_TYPE_INDEX) + ->setParam('type', DATABASE_TYPE_DELETE_INDEX) ->setParam('document', $index) ; From 83c4001f508da064214ff40140eb2771ea9e84c4 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Sun, 8 Aug 2021 19:51:18 -0400 Subject: [PATCH 28/45] Fix tests --- tests/e2e/Services/Database/DatabaseBase.php | 10 ++++++---- .../e2e/Services/Webhooks/WebhooksCustomServerTest.php | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index ab6b081203..ef30311639 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -43,16 +43,19 @@ trait DatabaseBase 'required' => true, ]); + sleep(2); + $releaseYear = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['moviesId'] . '/attributes/integer', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'attributeId' => 'releaseYear', - 'size' => 0, 'required' => true, ]); + sleep(2); + $actors = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['moviesId'] . '/attributes/string', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -61,7 +64,6 @@ trait DatabaseBase 'attributeId' => 'actors', 'size' => 256, 'required' => false, - 'default' => null, 'array' => true, ]); @@ -88,7 +90,7 @@ trait DatabaseBase $this->assertEquals($actors['body']['array'], true); // wait for database worker to create attributes - sleep(10); + sleep(5); $movies = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['moviesId'], array_merge([ 'content-type' => 'application/json', @@ -120,7 +122,7 @@ trait DatabaseBase 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ - 'id' => 'titleIndex', + 'indexId' => 'titleIndex', 'type' => 'fulltext', 'attributes' => ['title'], ]); diff --git a/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php b/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php index 923039fa1d..bcb473b4a8 100644 --- a/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php +++ b/tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php @@ -63,7 +63,7 @@ class WebhooksCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ - 'id' => 'fullname', + 'indexId' => 'fullname', 'type' => 'key', 'attributes' => ['lastName', 'firstName'], 'orders' => ['ASC', 'ASC'], From 5296e40b07b71ab2a5d0e13fff7dfcf11cc176c9 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Sun, 8 Aug 2021 19:56:31 -0400 Subject: [PATCH 29/45] Test for deleting index --- .../Database/DatabaseCustomServerTest.php | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index 4e559fad8f..421055fde9 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -13,7 +13,7 @@ class DatabaseCustomServerTest extends Scope use ProjectCustom; use SideServer; - public function testDeleteAttribute() + public function testDeleteAttribute(): array { /** * Test for SUCCESS @@ -124,11 +124,36 @@ class DatabaseCustomServerTest extends Scope return [ 'collectionId' => $actors['body']['$id'], + 'indexId' => $index['body']['$id'], ]; } + /** + * @depends testDeleteAttribute + */ + public function testDeleteIndex($data): array + { + $index = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $data['collectionId'] . '/indexes/'. $data['indexId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + // Wait for database worker to finish deleting index + sleep(5); + + $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['collectionId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), []); + + $this->assertCount(0, $collection['body']['indexes']); + + return $data; + } /** - * @depends testDeleteAttribute + * @depends testDeleteIndex */ public function testDeleteCollection($data) { From d6335f3cfe80acdc16b4b20a8e24c3e47a3a5acb Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 9 Aug 2021 10:57:21 +0545 Subject: [PATCH 30/45] support disabling switch --- public/dist/scripts/app-all.js | 6 ++++-- public/dist/scripts/app.js | 6 ++++-- public/scripts/views/forms/custom-id.js | 9 +++++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/public/dist/scripts/app-all.js b/public/dist/scripts/app-all.js index 52170473f2..b0433ae497 100644 --- a/public/dist/scripts/app-all.js +++ b/public/dist/scripts/app-all.js @@ -2372,8 +2372,10 @@ code.innerHTML=value;Prism.highlightElement(code);div.scrollTop=0;};element.addE function syncA(){element.value=picker.value;update();} function syncB(){picker.value=element.value;} element.parentNode.insertBefore(preview,element);update();syncB();}});})(window);(function(window){"use strict";window.ls.container.get("view").add({selector:"data-forms-copy",controller:function(element,alerts,document,window){var button=window.document.createElement("i");button.type="button";button.className="icon-docs note copy";button.style.cursor="pointer";element.parentNode.insertBefore(button,element.nextSibling);var copy=function(event){let disabled=element.disabled;element.disabled=false;element.focus();element.select();document.execCommand("Copy");if(document.selection){document.selection.empty();}else if(window.getSelection){window.getSelection().removeAllRanges();} -element.disabled=disabled;element.blur();alerts.add({text:"Copied to clipboard",class:""},3000);};button.addEventListener("click",copy);}});})(window);(function(window){"use strict";window.ls.container.get("view").add({selector:"data-custom-id",controller:function(element,sdk,console,window){let prevData="";let idType=element.getAttribute('data-id-type');const div=window.document.createElement("div");div.className="input-copy";const button=window.document.createElement("i");button.type="button";button.style.cursor="pointer";const writer=window.document.createElement("input");writer.type="text";writer.setAttribute("maxlength",element.getAttribute("maxlength"));const placeholder=element.getAttribute("placeholder");if(placeholder){writer.setAttribute("placeholder",placeholder);} -const info=window.document.createElement("div");info.className="text-fade text-size-xs margin-top-negative-small margin-bottom";div.appendChild(writer);div.appendChild(button);element.parentNode.insertBefore(div,element);element.parentNode.insertBefore(info,div.nextSibling);const switchType=function(event){if(idType=="custom"){idType="auto";setIdType(idType);}else{idType="custom";setIdType(idType);}} +element.disabled=disabled;element.blur();alerts.add({text:"Copied to clipboard",class:""},3000);};button.addEventListener("click",copy);}});})(window);(function(window){"use strict";window.ls.container.get("view").add({selector:"data-custom-id",controller:function(element,sdk,console,window){let prevData="";let idType=element.getAttribute('data-id-type');let disableSwitch=element.getAttribute('data-disable-switch');const div=window.document.createElement("div");if(disableSwitch!=="true"){div.className="input-copy";} +const button=window.document.createElement("i");button.type="button";button.style.cursor="pointer";const writer=window.document.createElement("input");writer.type="text";writer.setAttribute("maxlength",element.getAttribute("maxlength"));const placeholder=element.getAttribute("placeholder");if(placeholder){writer.setAttribute("placeholder",placeholder);} +const info=window.document.createElement("div");info.className="text-fade text-size-xs margin-top-negative-small margin-bottom";div.appendChild(writer);if(disableSwitch!=="true"){div.appendChild(button);} +element.parentNode.insertBefore(div,element);element.parentNode.insertBefore(info,div.nextSibling);const switchType=function(event){if(idType=="custom"){idType="auto";setIdType(idType);}else{idType="custom";setIdType(idType);}} const validate=function(event){const[service,method]=element.dataset["validator"].split('.');const value=event.target.value;if(value.length<1){event.target.setCustomValidity("ID is required");}else{switch(service){case'projects':setValidity(console[service][method](value),event.target);break;default:setValidity(sdk[service][method](value),event.target);}}} const setValidity=async function(promise,target){try{await promise;target.setCustomValidity("ID already exists");}catch(e){target.setCustomValidity("");}} const setIdType=function(idType){if(idType=="custom"){element.setAttribute("data-id-type",idType);info.innerHTML="Allowed Characters A-Z, a-z, 0-9, and non-leading underscore";if(prevData==='auto-generated'){prevData=""} diff --git a/public/dist/scripts/app.js b/public/dist/scripts/app.js index f4cec9f70c..fbaf23c857 100644 --- a/public/dist/scripts/app.js +++ b/public/dist/scripts/app.js @@ -356,8 +356,10 @@ code.innerHTML=value;Prism.highlightElement(code);div.scrollTop=0;};element.addE function syncA(){element.value=picker.value;update();} function syncB(){picker.value=element.value;} element.parentNode.insertBefore(preview,element);update();syncB();}});})(window);(function(window){"use strict";window.ls.container.get("view").add({selector:"data-forms-copy",controller:function(element,alerts,document,window){var button=window.document.createElement("i");button.type="button";button.className="icon-docs note copy";button.style.cursor="pointer";element.parentNode.insertBefore(button,element.nextSibling);var copy=function(event){let disabled=element.disabled;element.disabled=false;element.focus();element.select();document.execCommand("Copy");if(document.selection){document.selection.empty();}else if(window.getSelection){window.getSelection().removeAllRanges();} -element.disabled=disabled;element.blur();alerts.add({text:"Copied to clipboard",class:""},3000);};button.addEventListener("click",copy);}});})(window);(function(window){"use strict";window.ls.container.get("view").add({selector:"data-custom-id",controller:function(element,sdk,console,window){let prevData="";let idType=element.getAttribute('data-id-type');const div=window.document.createElement("div");div.className="input-copy";const button=window.document.createElement("i");button.type="button";button.style.cursor="pointer";const writer=window.document.createElement("input");writer.type="text";writer.setAttribute("maxlength",element.getAttribute("maxlength"));const placeholder=element.getAttribute("placeholder");if(placeholder){writer.setAttribute("placeholder",placeholder);} -const info=window.document.createElement("div");info.className="text-fade text-size-xs margin-top-negative-small margin-bottom";div.appendChild(writer);div.appendChild(button);element.parentNode.insertBefore(div,element);element.parentNode.insertBefore(info,div.nextSibling);const switchType=function(event){if(idType=="custom"){idType="auto";setIdType(idType);}else{idType="custom";setIdType(idType);}} +element.disabled=disabled;element.blur();alerts.add({text:"Copied to clipboard",class:""},3000);};button.addEventListener("click",copy);}});})(window);(function(window){"use strict";window.ls.container.get("view").add({selector:"data-custom-id",controller:function(element,sdk,console,window){let prevData="";let idType=element.getAttribute('data-id-type');let disableSwitch=element.getAttribute('data-disable-switch');const div=window.document.createElement("div");if(disableSwitch!=="true"){div.className="input-copy";} +const button=window.document.createElement("i");button.type="button";button.style.cursor="pointer";const writer=window.document.createElement("input");writer.type="text";writer.setAttribute("maxlength",element.getAttribute("maxlength"));const placeholder=element.getAttribute("placeholder");if(placeholder){writer.setAttribute("placeholder",placeholder);} +const info=window.document.createElement("div");info.className="text-fade text-size-xs margin-top-negative-small margin-bottom";div.appendChild(writer);if(disableSwitch!=="true"){div.appendChild(button);} +element.parentNode.insertBefore(div,element);element.parentNode.insertBefore(info,div.nextSibling);const switchType=function(event){if(idType=="custom"){idType="auto";setIdType(idType);}else{idType="custom";setIdType(idType);}} const validate=function(event){const[service,method]=element.dataset["validator"].split('.');const value=event.target.value;if(value.length<1){event.target.setCustomValidity("ID is required");}else{switch(service){case'projects':setValidity(console[service][method](value),event.target);break;default:setValidity(sdk[service][method](value),event.target);}}} const setValidity=async function(promise,target){try{await promise;target.setCustomValidity("ID already exists");}catch(e){target.setCustomValidity("");}} const setIdType=function(idType){if(idType=="custom"){element.setAttribute("data-id-type",idType);info.innerHTML="Allowed Characters A-Z, a-z, 0-9, and non-leading underscore";if(prevData==='auto-generated'){prevData=""} diff --git a/public/scripts/views/forms/custom-id.js b/public/scripts/views/forms/custom-id.js index c444912c29..304f780000 100644 --- a/public/scripts/views/forms/custom-id.js +++ b/public/scripts/views/forms/custom-id.js @@ -5,9 +5,12 @@ controller: function (element, sdk, console, window) { let prevData = ""; let idType = element.getAttribute('data-id-type'); + let disableSwitch = element.getAttribute('data-disable-switch'); const div = window.document.createElement("div"); - div.className = "input-copy"; + if(disableSwitch !== "true") { + div.className = "input-copy"; + } const button = window.document.createElement("i"); button.type = "button"; @@ -25,7 +28,9 @@ info.className = "text-fade text-size-xs margin-top-negative-small margin-bottom"; div.appendChild(writer); - div.appendChild(button); + if(disableSwitch !== "true") { + div.appendChild(button); + } element.parentNode.insertBefore(div, element); element.parentNode.insertBefore(info, div.nextSibling); From 0988423a676bfba80879041c35677d2a9f79a702 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 9 Aug 2021 11:32:05 +0545 Subject: [PATCH 31/45] update ui --- app/views/console/database/collection.phtml | 100 ++++---- app/views/console/database/document.phtml | 248 ++++++++++---------- app/views/console/functions/function.phtml | 2 +- app/views/console/storage/index.phtml | 8 +- 4 files changed, 179 insertions(+), 179 deletions(-) diff --git a/app/views/console/database/collection.phtml b/app/views/console/database/collection.phtml index 72d75dc3ee..8485e44053 100644 --- a/app/views/console/database/collection.phtml +++ b/app/views/console/database/collection.phtml @@ -22,8 +22,8 @@ $maxCells = 10;
- - + +

JSON View

@@ -80,52 +80,52 @@ $maxCells = 10;
documents found
- +
- $rule): - if($i > $maxCells) { - break; - } - $label = (isset($rule['label'])) ? $rule['label'] : ''; - ?> - - + $rule): + if ($i > $maxCells) { + break; + } + $label = (isset($rule['label'])) ? $rule['label'] : ''; + ?> + + - $rule): - if($i > $maxCells) { - break; - } - $label = $rule['label'] ?? ''; - $key = $rule['key'] ?? ''; - $type = $rule['type'] ?? ''; - $array = $rule['array'] ?? ''; - ?> - - +
escape($label); ?>escape($label); ?>
- - - - - - - {...} - - - - - - + $rule): + if ($i > $maxCells) { + break; + } + $label = $rule['label'] ?? ''; + $key = $rule['key'] ?? ''; + $type = $rule['type'] ?? ''; + $array = $rule['array'] ?? ''; + ?> + + + + + + + + {...} + + + + + + [...] - +
@@ -215,7 +215,7 @@ $maxCells = 10;
- +

@@ -248,7 +248,7 @@ $maxCells = 10;
- +