Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

(refactor): convert collection configs to Attribute/Index value objects

Browse Source
This commit is contained in:
Jake Barnby
2026-03-27 00:51:08 +13:00
parent 1230f8201f
commit c791012bf7
12 changed files with 4479 additions and 7599 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/config/collections/common.php
+1536 -2627
View File
File diff suppressed because it is too large Load Diff
app/config/collections/databases.php
+76 -114
View File
@@ -1,126 +1,88 @@
<?php
use Utopia\Database\Attribute;
use Utopia\Database\Database;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Index;
use Utopia\Query\Schema\ColumnType;
use Utopia\Query\Schema\IndexType;
return [
'collections' => [
'$collection' => ID::custom('databases'),
'$id' => ID::custom('collections'),
'$collection' => 'databases',
'$id' => 'collections',
'name' => 'Collections',
'attributes' => [
[
'$id' => ID::custom('databaseInternalId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('databaseId'),
'type' => Database::VAR_STRING,
'signed' => true,
'size' => Database::LENGTH_KEY,
'format' => '',
'filters' => [],
'required' => true,
'default' => null,
'array' => false,
],
[
'$id' => ID::custom('name'),
'type' => Database::VAR_STRING,
'size' => 256,
'required' => true,
'signed' => true,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('enabled'),
'type' => Database::VAR_BOOLEAN,
'signed' => true,
'size' => 0,
'format' => '',
'filters' => [],
'required' => true,
'default' => null,
'array' => false,
],
[
'$id' => ID::custom('documentSecurity'),
'type' => Database::VAR_BOOLEAN,
'signed' => true,
'size' => 0,
'format' => '',
'filters' => [],
'required' => true,
'default' => null,
'array' => false,
],
[
'$id' => ID::custom('attributes'),
'type' => Database::VAR_STRING,
'size' => 1000000,
'required' => false,
'signed' => true,
'array' => false,
'filters' => ['subQueryAttributes'],
],
[
'$id' => ID::custom('indexes'),
'type' => Database::VAR_STRING,
'size' => 1000000,
'required' => false,
'signed' => true,
'array' => false,
'filters' => ['subQueryIndexes'],
],
[
'$id' => ID::custom('search'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 16384,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
new Attribute(
key: 'databaseInternalId',
type: ColumnType::String,
size: Database::LENGTH_KEY,
required: true,
),
new Attribute(
key: 'databaseId',
type: ColumnType::String,
size: Database::LENGTH_KEY,
required: true,
),
new Attribute(
key: 'name',
type: ColumnType::String,
size: 256,
required: true,
),
new Attribute(
key: 'enabled',
type: ColumnType::Boolean,
required: true,
),
new Attribute(
key: 'documentSecurity',
type: ColumnType::Boolean,
required: true,
),
new Attribute(
key: 'attributes',
type: ColumnType::String,
size: 1000000,
filters: ['subQueryAttributes'],
),
new Attribute(
key: 'indexes',
type: ColumnType::String,
size: 1000000,
filters: ['subQueryIndexes'],
),
new Attribute(
key: 'search',
type: ColumnType::String,
size: 16384,
),
],
'indexes' => [
[
'$id' => ID::custom('_fulltext_search'),
'type' => Database::INDEX_FULLTEXT,
'attributes' => ['search'],
'lengths' => [],
'orders' => [],
],
[
'$id' => ID::custom('_key_name'),
'type' => Database::INDEX_KEY,
'attributes' => ['name'],
'lengths' => [256],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_enabled'),
'type' => Database::INDEX_KEY,
'attributes' => ['enabled'],
'lengths' => [],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_documentSecurity'),
'type' => Database::INDEX_KEY,
'attributes' => ['documentSecurity'],
'lengths' => [],
'orders' => [Database::ORDER_ASC],
],
new Index(
key: '_fulltext_search',
type: IndexType::Fulltext,
attributes: ['search'],
),
new Index(
key: '_key_name',
type: IndexType::Key,
attributes: ['name'],
lengths: [256],
orders: ['ASC'],
),
new Index(
key: '_key_enabled',
type: IndexType::Key,
attributes: ['enabled'],
orders: ['ASC'],
),
new Index(
key: '_key_documentSecurity',
type: IndexType::Key,
attributes: ['documentSecurity'],
orders: ['ASC'],
),
],
]
],
];
app/config/collections/logs.php
+59 -86
View File
@@ -1,94 +1,67 @@
<?php
use Utopia\Database\Attribute;
use Utopia\Database\Database;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Index;
use Utopia\Query\Schema\ColumnType;
use Utopia\Query\Schema\IndexType;
$logsCollection = [];
$logsCollection['stats'] = [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('stats'),
'name' => 'stats',
'attributes' => [
[
'$id' => ID::custom('metric'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
return [
'stats' => [
'$collection' => '_metadata',
'$id' => 'stats',
'name' => 'stats',
'attributes' => [
new Attribute(
key: 'metric',
type: ColumnType::String,
size: Database::LENGTH_KEY,
required: true,
),
new Attribute(
key: 'region',
type: ColumnType::String,
size: Database::LENGTH_KEY,
required: true,
),
new Attribute(
key: 'value',
type: ColumnType::Integer,
size: 8,
required: true,
),
new Attribute(
key: 'time',
type: ColumnType::Datetime,
signed: false,
filters: ['datetime'],
),
new Attribute(
key: 'period',
type: ColumnType::String,
size: 4,
required: true,
),
],
[
'$id' => ID::custom('region'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('value'),
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 8,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('time'),
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => ID::custom('period'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 4,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
'$id' => ID::custom('_key_time'),
'type' => Database::INDEX_KEY,
'attributes' => ['time'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
],
[
'$id' => ID::custom('_key_period_time'),
'type' => Database::INDEX_KEY,
'attributes' => ['period', 'time'],
'lengths' => [],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_metric_period_time'),
'type' => Database::INDEX_UNIQUE,
'attributes' => ['metric', 'period', 'time'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
'indexes' => [
new Index(
key: '_key_time',
type: IndexType::Key,
attributes: ['time'],
orders: ['DESC'],
),
new Index(
key: '_key_period_time',
type: IndexType::Key,
attributes: ['period', 'time'],
orders: ['ASC'],
),
new Index(
key: '_key_metric_period_time',
type: IndexType::Unique,
attributes: ['metric', 'period', 'time'],
orders: ['DESC'],
),
],
],
];
return $logsCollection;
app/config/collections/platform.php
+1136 -1875
View File
File diff suppressed because it is too large Load Diff
app/config/collections/projects.php
+1537 -2666
View File
File diff suppressed because it is too large Load Diff
app/config/collections/vectorsdb.php
+100 -150
View File
@@ -1,165 +1,115 @@
<?php
use Utopia\Database\Attribute;
use Utopia\Database\Database;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Index;
use Utopia\Query\Schema\ColumnType;
use Utopia\Query\Schema\IndexType;
return [
'collections' => [
'$collection' => ID::custom('databases'),
'$id' => ID::custom('collections'),
'$collection' => 'databases',
'$id' => 'collections',
'name' => 'Collections',
'attributes' => [
[
'$id' => ID::custom('databaseInternalId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('databaseId'),
'type' => Database::VAR_STRING,
'signed' => true,
'size' => Database::LENGTH_KEY,
'format' => '',
'filters' => [],
'required' => true,
'default' => null,
'array' => false,
],
[
'$id' => ID::custom('name'),
'type' => Database::VAR_STRING,
'size' => 256,
'required' => true,
'signed' => true,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('dimension'),
'type' => Database::VAR_INTEGER,
'size' => 0,
'required' => true,
'signed' => false,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('enabled'),
'type' => Database::VAR_BOOLEAN,
'signed' => true,
'size' => 0,
'format' => '',
'filters' => [],
'required' => true,
'default' => null,
'array' => false,
],
[
'$id' => ID::custom('documentSecurity'),
'type' => Database::VAR_BOOLEAN,
'signed' => true,
'size' => 0,
'format' => '',
'filters' => [],
'required' => true,
'default' => null,
'array' => false,
],
[
'$id' => ID::custom('attributes'),
'type' => Database::VAR_STRING,
'size' => 1000000,
'required' => false,
'signed' => true,
'array' => false,
'filters' => ['subQueryAttributes'],
],
[
'$id' => ID::custom('indexes'),
'type' => Database::VAR_STRING,
'size' => 1000000,
'required' => false,
'signed' => true,
'array' => false,
'filters' => ['subQueryIndexes'],
],
[
'$id' => ID::custom('search'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 16384,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
new Attribute(
key: 'databaseInternalId',
type: ColumnType::String,
size: Database::LENGTH_KEY,
required: true,
),
new Attribute(
key: 'databaseId',
type: ColumnType::String,
size: Database::LENGTH_KEY,
required: true,
),
new Attribute(
key: 'name',
type: ColumnType::String,
size: 256,
required: true,
),
new Attribute(
key: 'dimension',
type: ColumnType::Integer,
required: true,
signed: false,
),
new Attribute(
key: 'enabled',
type: ColumnType::Boolean,
required: true,
),
new Attribute(
key: 'documentSecurity',
type: ColumnType::Boolean,
required: true,
),
new Attribute(
key: 'attributes',
type: ColumnType::String,
size: 1000000,
filters: ['subQueryAttributes'],
),
new Attribute(
key: 'indexes',
type: ColumnType::String,
size: 1000000,
filters: ['subQueryIndexes'],
),
new Attribute(
key: 'search',
type: ColumnType::String,
size: 16384,
),
],
'defaultAttributes' => [
[
'$id' => ID::custom('embeddings'),
'type' => Database::VAR_VECTOR,
'required' => true,
'signed' => false,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('metadata'),
'type' => Database::VAR_OBJECT,
'default' => [],
'required' => false,
'size' => 0,
'signed' => false,
'array' => false,
'filters' => [],
],
new Attribute(
key: 'embeddings',
type: ColumnType::Vector,
required: true,
signed: false,
),
new Attribute(
key: 'metadata',
type: ColumnType::Object,
default: [],
signed: false,
),
],
'indexes' => [
[
'$id' => ID::custom('_fulltext_search'),
'type' => Database::INDEX_FULLTEXT,
'attributes' => ['search'],
'lengths' => [],
'orders' => [],
],
[
'$id' => ID::custom('_key_name'),
'type' => Database::INDEX_KEY,
'attributes' => ['name'],
'lengths' => [256],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_enabled'),
'type' => Database::INDEX_KEY,
'attributes' => ['enabled'],
'lengths' => [],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_documentSecurity'),
'type' => Database::INDEX_KEY,
'attributes' => ['documentSecurity'],
'lengths' => [],
'orders' => [Database::ORDER_ASC],
],
new Index(
key: '_fulltext_search',
type: IndexType::Fulltext,
attributes: ['search'],
),
new Index(
key: '_key_name',
type: IndexType::Key,
attributes: ['name'],
lengths: [256],
orders: ['ASC'],
),
new Index(
key: '_key_enabled',
type: IndexType::Key,
attributes: ['enabled'],
orders: ['ASC'],
),
new Index(
key: '_key_documentSecurity',
type: IndexType::Key,
attributes: ['documentSecurity'],
orders: ['ASC'],
),
],
'defaultIndexes' => [
// not creating default indexes on the embeddings as it depends on the type of query users using the most
[
'$id' => ID::custom('_key_metadata'),
'type' => Database::INDEX_OBJECT,
'attributes' => ['metadata'],
'lengths' => [],
'orders' => [],
],
]
]
new Index(
key: '_key_metadata',
type: IndexType::Object,
attributes: ['metadata'],
),
],
],
];
app/http.php
+4 -21
View File
@@ -255,25 +255,8 @@ function createDatabase(Http $app, string $resourceKey, string $dbName, array $c
continue;
}
$attributes = array_map(fn ($attr) => new Document([
'$id' => ID::custom($attr['$id']),
'type' => $attr['type'],
'size' => $attr['size'],
'required' => $attr['required'],
'signed' => $attr['signed'],
'array' => $attr['array'],
'filters' => $attr['filters'],
'default' => $attr['default'] ?? null,
'format' => $attr['format'] ?? ''
]), $collection['attributes']);
$indexes = array_map(fn ($index) => new Document([
'$id' => ID::custom($index['$id']),
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]), $collection['indexes']);
$attributes = $collection['attributes'];
$indexes = $collection['indexes'];
$database->createCollection($key, $attributes, $indexes);
$collectionsCreated++;
@@ -481,8 +464,8 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $tot
continue;
}
$attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']);
$indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']);
$attributes = $collection['attributes'];
$indexes = $collection['indexes'];
$dbForProject->createCollection($key, $attributes, $indexes);
$collectionsCreated++;
src/Appwrite/Migration/Migration.php
+18 -38
View File
@@ -5,6 +5,7 @@ namespace Appwrite\Migration;
use Exception;
use Utopia\Config\Config;
use Utopia\Console;
use Utopia\Database\Attribute;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception\Conflict;
@@ -227,15 +228,8 @@ abstract class Migration
$collection = $this->collections[$collectionType][$id];
$attributes = [];
foreach ($collection['attributes'] as $attribute) {
$attributes[] = new Document($attribute);
}
$indexes = [];
foreach ($collection['indexes'] as $index) {
$indexes[] = new Document($index);
}
$attributes = $collection['attributes'];
$indexes = $collection['indexes'];
try {
$this->dbForProject->createCollection($name, $attributes, $indexes);
@@ -284,7 +278,7 @@ abstract class Migration
$attributesToCreate = [];
$attributes = $collection['attributes'];
$attributeKeys = \array_column($collection['attributes'], '$id');
$attributeKeys = \array_map(fn ($a) => $a->key, $collection['attributes']);
foreach ($attributeIds as $attributeId) {
$attributeKey = \array_search($attributeId, $attributeKeys);
@@ -293,12 +287,11 @@ abstract class Migration
throw new Exception("Attribute {$attributeId} not found");
}
$attribute = $attributes[$attributeKey];
$attribute['filters'] ??= [];
$attribute['default'] ??= null;
$attribute['default'] = \in_array('json', $attribute['filters'])
? \json_encode($attribute['default'])
: $attribute['default'];
$attribute = clone $attributes[$attributeKey];
if (\in_array('json', $attribute->filters) && $attribute->default !== null) {
$attribute->default = \json_encode($attribute->default);
}
$attributesToCreate[] = $attribute;
}
@@ -349,28 +342,21 @@ abstract class Migration
$attributes = $collection['attributes'];
$attributeKey = \array_search($attributeId, \array_column($attributes, '$id'));
$attributeKey = \array_search($attributeId, \array_map(fn ($a) => $a->key, $attributes));
if ($attributeKey === false) {
throw new Exception("Attribute {$attributeId} not found");
}
$attribute = $attributes[$attributeKey];
$filters = $attribute['filters'] ?? [];
$default = $attribute['default'] ?? null;
$attribute = clone $attributes[$attributeKey];
if (\in_array('json', $attribute->filters) && $attribute->default !== null) {
$attribute->default = \json_encode($attribute->default);
}
$database->createAttribute(
collection: $collectionId,
id: $attributeId,
type: $attribute['type'],
size: $attribute['size'],
required: $attribute['required'],
default: \in_array('json', $filters) ? \json_encode($default) : $default,
signed: $attribute['signed'] ?? true,
array: $attribute['array'] ?? false,
format: $attribute['format'] ?? '',
formatOptions: $attribute['formatOptions'] ?? [],
filters: $filters,
attribute: $attribute,
);
}
@@ -407,21 +393,15 @@ abstract class Migration
$indexes = $collection['indexes'];
$indexKey = \array_search($indexId, \array_column($indexes, '$id'));
$indexKey = \array_search($indexId, \array_map(fn ($i) => $i->key, $indexes));
if ($indexKey === false) {
throw new Exception("Index {$indexId} not found");
}
$index = $indexes[$indexKey];
$database->createIndex(
collection: $collectionId,
id: $indexId,
type: $index['type'],
attributes: $index['attributes'],
lengths: $index['lengths'] ?? [],
orders: $index['orders'] ?? []
index: $indexes[$indexKey],
);
}
src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php
+2 -9
View File
@@ -216,15 +216,8 @@ class Create extends Action
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'The "collections" collection is not configured.');
}
$attributes = [];
foreach ($collections['attributes'] as $attribute) {
$attributes[] = new Document($attribute);
}
$indexes = [];
foreach ($collections['indexes'] as $index) {
$indexes[] = new Document($index);
}
$attributes = $collections['attributes'];
$indexes = $collections['indexes'];
try {
$dbForProject->createCollection('database_' . $database->getSequence(), $attributes, $indexes);
src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Collections/Create.php
+8 -10
View File
@@ -12,6 +12,7 @@ use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Database\Validator\CustomId;
use Appwrite\Utopia\Response as UtopiaResponse;
use Utopia\Config\Config;
use Utopia\Database\Attribute;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate as DuplicateException;
@@ -117,18 +118,15 @@ class Create extends CollectionAction
/** @var Database $dbForDatabases */
$dbForDatabases = $getDatabasesDB($database);
$attributes = [];
$indexes = [];
$collections = (Config::getParam('collections', [])['vectorsdb'] ?? [])['collections'] ?? [];
foreach ($collections['defaultAttributes'] as $attribute) {
if ($attribute['$id'] === 'embeddings') {
$attribute['size'] = $dimension;
$attributes = \array_map(function (Attribute $attribute) use ($dimension) {
if ($attribute->key === 'embeddings') {
$attribute = clone $attribute;
$attribute->size = $dimension;
}
$attributes[] = new Document($attribute);
}
foreach ($collections['defaultIndexes'] as $index) {
$indexes[] = new Document($index);
}
return $attribute;
}, $collections['defaultAttributes']);
$indexes = $collections['defaultIndexes'];
try {
// passing null in creates only creates the metadata collection
if (!$dbForDatabases->exists(null, Database::METADATA)) {
src/Appwrite/Platform/Modules/Projects/Http/Projects/Create.php
+2 -2
View File
@@ -270,8 +270,8 @@ class Create extends Action
continue;
}
$attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']);
$indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']);
$attributes = $collection['attributes'];
$indexes = $collection['indexes'];
try {
$dbForProject->createCollection($key, $attributes, $indexes);
tests/unit/General/CollectionsTest.php
+1 -1
View File
@@ -21,7 +21,7 @@ class CollectionsTest extends TestCase
foreach ($collection['attributes'] as $check) {
$occurrences = 0;
foreach ($collection['attributes'] as $attribute) {
if ($attribute['$id'] == $check['$id']) {
if ($attribute->key == $check->key) {
$occurrences++;
}
}