Merge branch 'feat-database-indexing' of github.com:appwrite/appwrite into feat-new-usage-endpoints

2026-05-26 13:51:13 +00:00 · 2021-08-28 21:39:01 +05:30
parent 1b074a3c38 06096b4168
commit f00fead07b
6 changed files with 218 additions and 25 deletions
@@ -15,6 +15,7 @@ use Utopia\Validator\JSON;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Query;
+use Utopia\Database\Adapter\MariaDB;
 use Utopia\Database\Validator\Key;
 use Utopia\Database\Validator\Permissions;
 use Utopia\Database\Validator\QueryValidator;
@@ -23,6 +24,7 @@ use Utopia\Database\Validator\Structure;
 use Utopia\Database\Validator\UID;
 use Utopia\Database\Exception\Authorization as AuthorizationException;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
+use Utopia\Database\Exception\Limit as LimitException;
 use Utopia\Database\Exception\Structure as StructureException;
 use Appwrite\Utopia\Response;
 use Appwrite\Database\Validator\CustomId;
@@ -1116,6 +1118,16 @@ App::post('/v1/database/collections/:collectionId/indexes')
            throw new Exception('Collection not found', 404);
        }

+        $count = $dbForInternal->count('indexes', [
+            new Query('collectionId', Query::TYPE_EQUAL, [$collectionId])
+        ], 61);
+
+        $limit = 64 - MariaDB::getNumberOfDefaultIndexes();
+
+        if ($count >= $limit) {
+            throw new Exception('Index limit exceeded', 400);
+        }
+
        // Convert Document[] to array of attribute metadata
        $oldAttributes = \array_map(function ($a) {
            return $a->getArrayCopy();
@@ -1140,7 +1152,6 @@ App::post('/v1/database/collections/:collectionId/indexes')
            $lengths[$key] = ($attributeType === Database::VAR_STRING) ? $attributeSize : null;
        }

-        // TODO@kodumbeats should $lengths be a part of the response model?
        try {
            $index = $dbForInternal->createDocument('indexes', new Document([
                '$id' => $collectionId.'_'.$indexId,
@@ -1153,7 +1164,7 @@ App::post('/v1/database/collections/:collectionId/indexes')
                'orders' => $orders,
            ]));
        } catch (DuplicateException $th) {
-            throw new Exception('Attribute already exists', 409);
+            throw new Exception('Index already exists', 409);
        }

        $dbForInternal->purgeDocument('collections', $collectionId);
@@ -1571,8 +1582,11 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId')
        catch (AuthorizationException $exception) {
            throw new Exception('Unauthorized permissions', 401);
        }
+        catch (DuplicateException $exception) {
+            throw new Exception('Document already exists', 409);
+        }
        catch (StructureException $exception) {
-            throw new Exception('Bad structure. '.$exception->getMessage(), 400);
+            throw new Exception($exception->getMessage(), 400);
        }
        
        $usage
@@ -1645,4 +1659,4 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId')
        ;

        $response->noContent();
-    });
+    });
@@ -423,13 +423,13 @@ $auth = $this->getParam('auth', []);
                data-scope="console">
                <ul class="tiles cell-3 margin-bottom-small">
                    <?php foreach ($providers as $provider => $data):
-    if (isset($data['enabled']) && !$data['enabled']) {continue;}
-    if (isset($data['mock']) && $data['mock']) {continue;}
-    $sandbox = $data['sandbox'] ?? false;
-    $form = $data['form'] ?? false;
-    $name = $data['name'] ?? 'Unknown';
-    $beta = $data['beta'] ?? false;
-    ?>
+                        if (isset($data['enabled']) && !$data['enabled']) {continue;}
+                        // if (isset($data['mock']) && $data['mock']) {continue;}
+                        $sandbox = $data['sandbox'] ?? false;
+                        $form = $data['form'] ?? false;
+                        $name = $data['name'] ?? 'Unknown';
+                        $beta = $data['beta'] ?? false;
+                        ?>
 	                        <li class="<?php echo (isset($data['enabled']) && !$data['enabled']) ? 'dev-feature' : ''; ?>">
 	                            <div data-ui-modal class="modal close" data-button-alias="none" data-open-event="provider-update-<?php echo $provider; ?>">
 	                                <button type="button" class="close pull-end" data-ui-modal-close=""><i class="icon-cancel"></i></button>
@@ -63,7 +63,12 @@
        "adhocore/jwt": "1.1.2",
        "slickdeals/statsd": "3.1.0"
    },
-    "repositories": [],
+    "repositories": [
+        {
+            "type": "git",
+            "url": "https://github.com/utopia-php/database"
+        }
+    ],
    "require-dev": {
        "appwrite/sdk-generator": "0.13.0",
        "swoole/ide-helper": "4.6.7",
@@ -4,7 +4,7 @@
        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
        "This file is @generated automatically"
    ],
-    "content-hash": "aac413429914bd9299a7ae722f00cef8",
+    "content-hash": "175f077512c575216c4c88f1d33c6d00",
    "packages": [
        {
            "name": "adhocore/jwt",
@@ -1987,15 +1987,9 @@
            "version": "dev-feat-adjusted-query-validator",
            "source": {
                "type": "git",
-                "url": "https://github.com/utopia-php/database.git",
+                "url": "https://github.com/utopia-php/database",
                "reference": "cb73391371f70ddb54bc0000064b15c5f173cb7a"
            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/database/zipball/cb73391371f70ddb54bc0000064b15c5f173cb7a",
-                "reference": "cb73391371f70ddb54bc0000064b15c5f173cb7a",
-                "shasum": ""
-            },
            "require": {
                "ext-mongodb": "*",
                "ext-pdo": "*",
@@ -2017,7 +2011,11 @@
                    "Utopia\\Database\\": "src/Database"
                }
            },
-            "notification-url": "https://packagist.org/downloads/",
+            "autoload-dev": {
+                "psr-4": {
+                    "Utopia\\Tests\\": "tests/Database"
+                }
+            },
            "license": [
                "MIT"
            ],
@@ -2039,10 +2037,6 @@
                "upf",
                "utopia"
            ],
-            "support": {
-                "issues": "https://github.com/utopia-php/database/issues",
-                "source": "https://github.com/utopia-php/database/tree/feat-adjusted-query-validator"
-            },
            "time": "2021-08-23T14:18:47+00:00"
        },
        {
@@ -1089,4 +1089,86 @@ trait DatabaseBase

        return $data;
    }
+
+    /**
+     * @depends testDefaultPermissions
+     */
+    public function testUniqueIndexDuplicate(array $data): array
+    {
+        $uniqueIndex = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['moviesId'] . '/indexes', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ]), [
+            'indexId' => 'unique_title',
+            'type' => 'unique',
+            'attributes' => ['title'],
+        ]);
+
+        $this->assertEquals($uniqueIndex['headers']['status-code'], 201);
+
+        sleep(2);
+
+        // test for failure
+        $duplicate = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['moviesId'] . '/documents', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'documentId' => 'unique()',
+            'data' => [
+                'title' => 'Captain America',
+                'releaseYear' => 1944,
+                'actors' => [
+                    'Chris Evans',
+                    'Samuel Jackson',
+                ]
+            ],
+            'read' => ['user:'.$this->getUser()['$id']],
+            'write' => ['user:'.$this->getUser()['$id']],
+        ]);
+
+        $this->assertEquals(409, $duplicate['headers']['status-code']);
+
+        // Test for exception when updating document to conflict
+        $document = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['moviesId'] . '/documents', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'documentId' => 'unique()',
+            'data' => [
+                'title' => 'Captain America 5',
+                'releaseYear' => 1944,
+                'actors' => [
+                    'Chris Evans',
+                    'Samuel Jackson',
+                ]
+            ],
+            'read' => ['user:'.$this->getUser()['$id']],
+            'write' => ['user:'.$this->getUser()['$id']],
+        ]);
+
+        $this->assertEquals(201, $document['headers']['status-code']);
+
+        // Test for exception when updating document to conflict
+        $duplicate = $this->client->call(Client::METHOD_PATCH, '/database/collections/' . $data['moviesId'] . '/documents/' . $document['body']['$id'], array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'documentId' => 'unique()',
+            'data' => [
+                'title' => 'Captain America',
+                'releaseYear' => 1944,
+                'actors' => [
+                    'Chris Evans',
+                    'Samuel Jackson',
+                ]
+            ],
+            'read' => ['user:'.$this->getUser()['$id']],
+            'write' => ['user:'.$this->getUser()['$id']],
+        ]);
+
+        $this->assertEquals(409, $duplicate['headers']['status-code']);
+
+        return $data;
+    }
 }
@@ -306,4 +306,102 @@ class DatabaseCustomServerTest extends Scope

        $this->assertEquals($response['headers']['status-code'], 404);
    }
+
+    public function testIndexLimitException()
+    {
+        $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ]), [
+            'collectionId' => 'testLimitException',
+            'name' => 'testLimitException',
+            'read' => ['role:all'],
+            'write' => ['role:all'],
+            'permission' => 'document',
+        ]);
+
+        $this->assertEquals($collection['headers']['status-code'], 201);
+        $this->assertEquals($collection['body']['name'], 'testLimitException');
+
+        $collectionId = $collection['body']['$id'];
+
+        // add unique attributes for indexing
+        for ($i=0; $i < 64; $i++) {
+            // $this->assertEquals(true, static::getDatabase()->createAttribute('indexLimit', "test{$i}", Database::VAR_STRING, 16, true));
+            $attribute = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+                'x-appwrite-key' => $this->getProject()['apiKey']
+            ]), [
+                'attributeId' => "attribute{$i}",
+                'size' => 64,
+                'required' => true,
+            ]);
+
+            $this->assertEquals($attribute['headers']['status-code'], 201);
+        }
+
+        sleep(5);
+
+        $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ]));
+
+        $this->assertEquals($collection['headers']['status-code'], 200);
+        $this->assertEquals($collection['body']['name'], 'testLimitException');
+        $this->assertIsArray($collection['body']['attributes']);
+        $this->assertIsArray($collection['body']['indexes']);
+        $this->assertCount(64, $collection['body']['attributes']);
+        $this->assertCount(0, $collection['body']['indexes']);
+
+        // testing for indexLimit = 64
+        // MariaDB, MySQL, and MongoDB create 3 indexes per new collection
+        // Add up to the limit, then check if the next index throws IndexLimitException
+        for ($i=0; $i < 61; $i++) {
+            // $this->assertEquals(true, static::getDatabase()->createIndex('indexLimit', "index{$i}", Database::INDEX_KEY, ["test{$i}"], [16]));
+            $index = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+                'x-appwrite-key' => $this->getProject()['apiKey']
+            ]), [
+                'indexId' => "key_attribute{$i}",
+                'type' => 'key',
+                'attributes' => ["attribute{$i}"],
+            ]);
+
+            $this->assertEquals(201, $index['headers']['status-code']);
+            $this->assertEquals("key_attribute{$i}", $index['body']['key']);
+        }
+
+        sleep(5);
+
+        $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ]));
+
+        $this->assertEquals($collection['headers']['status-code'], 200);
+        $this->assertEquals($collection['body']['name'], 'testLimitException');
+        $this->assertIsArray($collection['body']['attributes']);
+        $this->assertIsArray($collection['body']['indexes']);
+        $this->assertCount(64, $collection['body']['attributes']);
+        $this->assertCount(61, $collection['body']['indexes']);
+
+        $tooMany = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ]), [
+            'indexId' => 'tooMany',
+            'type' => 'key',
+            'attributes' => ['attribute61'],
+        ]);
+
+        $this->assertEquals(400, $tooMany['headers']['status-code']);
+        $this->assertEquals('Index limit exceeded', $tooMany['body']['message']);
+    }
 }