Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat-custom-cf-hostnames' into lohanidamodar-patch-3

Browse Source
This commit is contained in:
Damodar Lohani
2024-11-17 10:55:30 +05:45
committed by GitHub
parent 25ae2ff173 9a7e5a3153
commit 48d13f31b8
24 changed files with 2285 additions and 512 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/tests.yml
+25 -5
View File
@@ -127,6 +127,11 @@ jobs:
Messaging,
Migrations
]
tables-mode: [
'Project',
'Shared V1',
'Shared V2',
]
steps:
- name: checkout
@@ -145,11 +150,26 @@ jobs:
docker compose up -d
sleep 30
- name: Run ${{matrix.service}} Tests
run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{matrix.service}} --debug
- name: Run ${{matrix.service}} Shared Tables Tests
run: _APP_DATABASE_SHARED_TABLES=database_db_main docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{matrix.service}} --debug
- name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode
run: |
if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then
echo "Using shared tables V1"
export _APP_DATABASE_SHARED_TABLES=database_db_main
export _APP_DATABASE_SHARED_TABLES_V1=database_db_main
elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then
echo "Using shared tables V2"
export _APP_DATABASE_SHARED_TABLES=database_db_main
export _APP_DATABASE_SHARED_TABLES_V1=
else
echo "Using project tables"
export _APP_DATABASE_SHARED_TABLES=
export _APP_DATABASE_SHARED_TABLES_V1=
fi
docker compose exec -T \
-e _APP_DATABASE_SHARED_TABLES \
-e _APP_DATABASE_SHARED_TABLES_V1 \
appwrite test /usr/src/code/tests/e2e/Services/${{ matrix.service }} --debug
benchmarking:
name: Benchmark
.gitignore
+1
View File
@@ -16,3 +16,4 @@ dev/yasd_init.php
.phpunit.result.cache
Makefile
appwrite.json
.zed/
app/cli.php
+4 -3
View File
@@ -114,8 +114,9 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
if (isset($databases[$dsn->getHost()])) {
$database = $databases[$dsn->getHost()];
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
@@ -136,10 +137,10 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
->getResource();
$database = new Database($dbAdapter, $cache);
$databases[$dsn->getHost()] = $database;
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
app/config/errors.php
+1 -1
View File
@@ -686,7 +686,7 @@ return [
],
Exception::ATTRIBUTE_LIMIT_EXCEEDED => [
'name' => Exception::ATTRIBUTE_LIMIT_EXCEEDED,
'description' => 'The maximum number of attributes has been reached.',
'description' => 'The maximum number or size of attributes for this collection has been reached.',
'code' => 400,
],
Exception::ATTRIBUTE_VALUE_INVALID => [
app/controllers/api/databases.php
+6 -3
View File
@@ -153,7 +153,7 @@ function createAttribute(string $databaseId, string $collectionId, Document $att
} catch (DuplicateException) {
throw new Exception(Exception::ATTRIBUTE_ALREADY_EXISTS);
} catch (LimitException) {
throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED, 'Attribute limit exceeded');
throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED);
} catch (\Throwable $e) {
$dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $collectionId);
$dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $collection->getInternalId());
@@ -197,7 +197,7 @@ function createAttribute(string $databaseId, string $collectionId, Document $att
throw new Exception(Exception::ATTRIBUTE_ALREADY_EXISTS);
} catch (LimitException) {
$dbForProject->deleteDocument('attributes', $attribute->getId());
throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED, 'Attribute limit exceeded');
throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED);
} catch (\Throwable $e) {
$dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $relatedCollection->getId());
$dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $relatedCollection->getInternalId());
@@ -393,6 +393,8 @@ function updateAttribute(
throw new Exception(Exception::ATTRIBUTE_INVALID_RESIZE);
} catch (NotFoundException) {
throw new Exception(Exception::ATTRIBUTE_NOT_FOUND);
} catch (LimitException) {
throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED);
}
}
@@ -2627,7 +2629,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
$validator = new IndexValidator(
$collection->getAttribute('attributes'),
$dbForProject->getAdapter()->getMaxIndexLength()
$dbForProject->getAdapter()->getMaxIndexLength(),
$dbForProject->getAdapter()->getInternalIndexesKeys(),
);
if (!$validator->isValid($index)) {
throw new Exception(Exception::INDEX_INVALID, $validator->getDescription());
app/controllers/api/projects.php
+74 -29
View File
@@ -122,6 +122,10 @@ App::post('/v1/projects')
$projectId = ($projectId == 'unique()') ? ID::unique() : $projectId;
if ($projectId === 'console') {
throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project.");
}
$databases = Config::getParam('pools-database', []);
$databaseOverride = System::getEnv('_APP_DATABASE_OVERRIDE');
@@ -132,16 +136,14 @@ App::post('/v1/projects')
$dsn = $databases[array_rand($databases)];
}
if ($projectId === 'console') {
throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project.");
}
// TODO: Temporary until all projects are using shared tables.
if ($dsn === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn, $sharedTables)) {
$schema = 'appwrite';
$database = 'appwrite';
$namespace = System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', '');
$dsn = $schema . '://' . System::getEnv('_APP_DATABASE_SHARED_TABLES', '') . '?database=' . $database;
$dsn = $schema . '://' . $dsn . '?database=' . $database;
if (!empty($namespace)) {
$dsn .= '&namespace=' . $namespace;
@@ -195,11 +197,18 @@ App::post('/v1/projects')
$adapter = $pools->get($dsn->getHost())->pop()->getResource();
$dbForProject = new Database($adapter, $cache);
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
$sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', ''));
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$projectTables = !\in_array($dsn->getHost(), $sharedTables);
$sharedTablesV1 = \in_array($dsn->getHost(), $sharedTablesV1);
$sharedTablesV2 = !$projectTables && !$sharedTablesV1;
$sharedTables = $sharedTablesV1 || $sharedTablesV2;
if ($sharedTables) {
$dbForProject
->setSharedTables(true)
->setTenant($project->getInternalId())
->setTenant($sharedTablesV1 ? $project->getInternalId() : null)
->setNamespace($dsn->getParam('namespace'));
} else {
$dbForProject
@@ -208,34 +217,70 @@ App::post('/v1/projects')
->setNamespace('_' . $project->getInternalId());
}
$dbForProject->create();
$create = true;
$audit = new Audit($dbForProject);
$audit->setup();
try {
$dbForProject->create();
} catch (Duplicate) {
$create = false;
}
$abuse = new TimeLimit('', 0, 1, $dbForProject);
$abuse->setup();
if ($create || $projectTables) {
$audit = new Audit($dbForProject);
$audit->setup();
/** @var array $collections */
$collections = Config::getParam('collections', [])['projects'] ?? [];
$abuse = new TimeLimit('', 0, 1, $dbForProject);
$abuse->setup();
}
foreach ($collections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
if (!$create && $sharedTablesV1) {
$attributes = \array_map(fn ($attribute) => new Document($attribute), Audit::ATTRIBUTES);
$indexes = \array_map(fn (array $index) => new Document($index), Audit::INDEXES);
$dbForProject->createDocument(Database::METADATA, new Document([
'$id' => ID::custom('audit'),
'$permissions' => [Permission::create(Role::any())],
'name' => 'audit',
'attributes' => $attributes,
'indexes' => $indexes,
'documentSecurity' => true
]));
$attributes = \array_map(function (array $attribute) {
return new Document($attribute);
}, $collection['attributes']);
$attributes = \array_map(fn ($attribute) => new Document($attribute), TimeLimit::ATTRIBUTES);
$indexes = \array_map(fn (array $index) => new Document($index), TimeLimit::INDEXES);
$dbForProject->createDocument(Database::METADATA, new Document([
'$id' => ID::custom('abuse'),
'$permissions' => [Permission::create(Role::any())],
'name' => 'abuse',
'attributes' => $attributes,
'indexes' => $indexes,
'documentSecurity' => true
]));
}
$indexes = \array_map(function (array $index) {
return new Document($index);
}, $collection['indexes']);
if ($create || $sharedTablesV1) {
/** @var array $collections */
$collections = Config::getParam('collections', [])['projects'] ?? [];
try {
$dbForProject->createCollection($key, $attributes, $indexes);
} catch (Duplicate) {
// Collection already exists
foreach ($collections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
$attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']);
$indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']);
try {
$dbForProject->createCollection($key, $attributes, $indexes);
} catch (Duplicate) {
$dbForProject->createDocument(Database::METADATA, new Document([
'$id' => ID::custom($key),
'$permissions' => [Permission::create(Role::any())],
'name' => $key,
'attributes' => $attributes,
'indexes' => $indexes,
'documentSecurity' => true
]));
}
}
}
app/controllers/api/teams.php
+8 -3
View File
@@ -805,8 +805,11 @@ App::get('/v1/teams/:teamId/memberships')
}, $membershipsPrivacy);
$memberships = array_map(function ($membership) use ($dbForProject, $team, $membershipsPrivacy) {
$user = !empty(array_filter($membershipsPrivacy))
? $dbForProject->getDocument('users', $membership->getAttribute('userId'))
: new Document();
if ($membershipsPrivacy['mfa']) {
$user = $dbForProject->getDocument('users', $membership->getAttribute('userId'));
$mfa = $user->getAttribute('mfa', false);
if ($mfa) {
@@ -888,9 +891,11 @@ App::get('/v1/teams/:teamId/memberships/:membershipId')
return $privacy || $isPrivilegedUser || $isAppUser;
}, $membershipsPrivacy);
if ($membershipsPrivacy['mfa']) {
$user = $dbForProject->getDocument('users', $membership->getAttribute('userId'));
$user = !empty(array_filter($membershipsPrivacy))
? $dbForProject->getDocument('users', $membership->getAttribute('userId'))
: new Document();
if ($membershipsPrivacy['mfa']) {
$mfa = $user->getAttribute('mfa', false);
if ($mfa) {
app/controllers/general.php
+2 -2
View File
@@ -63,7 +63,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
}
if (System::getEnv('_APP_OPTIONS_ROUTER_PROTECTION', 'disabled') === 'enabled') {
if ($host !== 'localhost' && $host !== APP_HOSTNAME_INTERNAL) { // localhost allowed for proxy, APP_HOSTNAME_INTERNAL allowed for migrations
if ($host !== 'localhost' && $host !== APP_HOSTNAME_INTERNAL && $host !== System::getEnv('_APP_CONSOLE_DOMAIN', '')) {
throw new AppwriteException(AppwriteException::GENERAL_ACCESS_FORBIDDEN, 'Router protection does not allow accessing Appwrite over this domain. Please add it as custom domain to your project or disable _APP_OPTIONS_ROUTER_PROTECTION environment variable.');
}
}
@@ -438,7 +438,7 @@ App::init()
});
App::init()
->groups(['api', 'web'])
->groups(['api'])
->inject('utopia')
->inject('swooleRequest')
->inject('request')
app/http.php
+3 -2
View File
@@ -16,6 +16,7 @@ use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
@@ -91,7 +92,7 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg
try {
Console::success('[Setup] - Creating database: appwrite...');
$dbForConsole->create();
} catch (\Throwable $e) {
} catch (Duplicate) {
Console::success('[Setup] - Skip: metadata table already exists');
}
@@ -223,7 +224,7 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg
});
});
$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($register) {
$http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($register) {
App::setResource('swooleRequest', fn () => $swooleRequest);
App::setResource('swooleResponse', fn () => $swooleResponse);
app/init.php
+6 -2
View File
@@ -1432,7 +1432,9 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole,
$dsn = new DSN('mysql://' . $project->getAttribute('database'));
}
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
@@ -1487,7 +1489,9 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
app/realtime.php
+30 -5
View File
@@ -29,6 +29,7 @@ use Utopia\Database\Validator\Authorization;
use Utopia\DSN\DSN;
use Utopia\Logger\Log;
use Utopia\System\System;
use Utopia\Telemetry\Adapter\None as NoTelemetry;
use Utopia\WebSocket\Adapter;
use Utopia\WebSocket\Server;
@@ -92,7 +93,9 @@ if (!function_exists('getProjectDB')) {
$database = new Database($adapter, getCache());
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
@@ -142,6 +145,13 @@ if (!function_exists('getRealtime')) {
}
}
if (!function_exists('getTelemetry')) {
function getTelemetry(int $workerId): Utopia\Telemetry\Adapter
{
return new NoTelemetry();
}
}
$realtime = getRealtime();
/**
@@ -274,6 +284,12 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
$server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $realtime, $logError) {
Console::success('Worker ' . $workerId . ' started successfully');
$telemetry = getTelemetry($workerId);
$register->set('telemetry', fn () => $telemetry);
$register->set('telemetry.connectionCounter', fn () => $telemetry->createUpDownCounter('realtime.server.open_connections'));
$register->set('telemetry.connectionCreatedCounter', fn () => $telemetry->createCounter('realtime.server.connection.created'));
$register->set('telemetry.messageSentCounter', fn () => $telemetry->createCounter('realtime.server.message.sent'));
$attempts = 0;
$start = time();
@@ -416,6 +432,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
);
if (($num = count($receivers)) > 0) {
$register->get('telemetry.messageSentCounter')->add($num);
$stats->incr($event['project'], 'messages', $num);
}
});
@@ -519,6 +536,9 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
]
]));
$register->get('telemetry.connectionCounter')->add(1);
$register->get('telemetry.connectionCreatedCounter')->add(1);
$stats->set($project->getId(), [
'projectId' => $project->getId(),
'teamId' => $project->getAttribute('teamId')
@@ -592,9 +612,12 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
}
switch ($message['type']) {
/**
* This type is used to authenticate.
*/
case 'ping':
$server->send([$connection], json_encode([
'type' => 'pong'
]));
break;
case 'authentication':
if (!array_key_exists('session', $message['data'])) {
throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Payload is not valid.');
@@ -652,12 +675,14 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
}
});
$server->onClose(function (int $connection) use ($realtime, $stats) {
$server->onClose(function (int $connection) use ($realtime, $stats, $register) {
if (array_key_exists($connection, $realtime->connections)) {
$stats->decr($realtime->connections[$connection]['projectId'], 'connectionsTotal');
}
$realtime->unsubscribe($connection);
$register->get('telemetry.connectionCounter')->add(-1);
Console::info('Connection close: ' . $connection);
});
app/worker.php
+9 -3
View File
@@ -93,7 +93,9 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
$dsn = new DSN('mysql://' . $project->getAttribute('database'));
}
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
@@ -126,7 +128,9 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
if (isset($databases[$dsn->getHost()])) {
$database = $databases[$dsn->getHost()];
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
@@ -150,7 +154,9 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
$databases[$dsn->getHost()] = $database;
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
composer.json
+8 -3
View File
@@ -45,13 +45,13 @@
"ext-sockets": "*",
"appwrite/php-runtimes": "0.16.*",
"appwrite/php-clamav": "2.0.*",
"utopia-php/abuse": "0.43.0",
"utopia-php/abuse": "0.43.*",
"utopia-php/analytics": "0.10.*",
"utopia-php/audit": "0.43.0",
"utopia-php/audit": "0.43.*",
"utopia-php/cache": "0.11.*",
"utopia-php/cli": "0.15.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.53.16",
"utopia-php/database": "0.53.20",
"utopia-php/domains": "0.5.*",
"utopia-php/dsn": "0.2.1",
"utopia-php/framework": "0.33.*",
@@ -70,6 +70,7 @@
"utopia-php/storage": "0.18.*",
"utopia-php/swoole": "0.8.*",
"utopia-php/system": "0.9.*",
"utopia-php/telemetry": "0.1.*",
"utopia-php/vcs": "0.8.*",
"utopia-php/websocket": "0.1.*",
"matomo/device-detector": "6.1.*",
@@ -96,6 +97,10 @@
"config": {
"platform": {
"php": "8.3"
},
"allow-plugins": {
"php-http/discovery": false,
"tbachert/spi": false
}
}
}
composer.lock
Generated
+1832 -328
View File
File diff suppressed because it is too large Load Diff
docker-compose.yml
+1
View File
@@ -193,6 +193,7 @@ services:
- _APP_EXPERIMENT_LOGGING_PROVIDER
- _APP_EXPERIMENT_LOGGING_CONFIG
- _APP_DATABASE_SHARED_TABLES
- _APP_DATABASE_SHARED_TABLES_V1
appwrite-console:
<<: *x-logging
docs/references/account/create-token-magic-url.md
+1 -1
View File
@@ -1,3 +1,3 @@
Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST /v1/account/sessions/token](https://appwrite.io/docs/references/cloud/client-web/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.
Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST /v1/account/sessions/token](https://appwrite.io/docs/references/cloud/client-web/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.
A user is limited to 10 active sessions at a time by default. [Learn more about session limits](https://appwrite.io/docs/authentication-security#limits).
src/Appwrite/Platform/Workers/Databases.php
+106 -99
View File
@@ -92,6 +92,7 @@ class Databases extends Action
* @throws Authorization
* @throws Conflict
* @throws \Exception
* @throws \Throwable
*/
private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void
{
@@ -134,7 +135,6 @@ class Databases extends Action
$options = $attribute->getAttribute('options', []);
$project = $dbForConsole->getDocument('projects', $projectId);
try {
switch ($type) {
case Database::VAR_RELATIONSHIP:
@@ -170,7 +170,6 @@ class Databases extends Action
$dbForProject->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'available'));
} catch (\Throwable $e) {
// TODO: Send non DatabaseExceptions to Sentry
Console::error($e->getMessage());
if ($e instanceof DatabaseException) {
@@ -193,15 +192,17 @@ class Databases extends Action
$relatedAttribute->setAttribute('status', 'failed')
);
}
throw $e;
} finally {
$this->trigger($database, $collection, $attribute, $project, $projectId, $events);
}
if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) {
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId());
}
if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) {
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId());
}
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
}
}
/**
@@ -215,6 +216,7 @@ class Databases extends Action
* @throws Authorization
* @throws Conflict
* @throws \Exception
* @throws \Throwable
**/
private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void
{
@@ -248,113 +250,116 @@ class Databases extends Action
// - stuck: attribute was available but cannot be removed
try {
if ($status !== 'failed') {
if ($type === Database::VAR_RELATIONSHIP) {
if ($options['twoWay']) {
$relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']);
if ($relatedCollection->isEmpty()) {
throw new DatabaseException('Collection not found');
try {
if ($status !== 'failed') {
if ($type === Database::VAR_RELATIONSHIP) {
if ($options['twoWay']) {
$relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']);
if ($relatedCollection->isEmpty()) {
throw new DatabaseException('Collection not found');
}
$relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']);
}
$relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']);
}
if (!$dbForProject->deleteRelationship('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
$dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'stuck'));
throw new DatabaseException('Failed to delete Relationship');
if (!$dbForProject->deleteRelationship('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
$dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'stuck'));
throw new DatabaseException('Failed to delete Relationship');
}
} elseif (!$dbForProject->deleteAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
throw new DatabaseException('Failed to delete Attribute');
}
} elseif (!$dbForProject->deleteAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
throw new DatabaseException('Failed to delete Attribute');
}
}
$dbForProject->deleteDocument('attributes', $attribute->getId());
$dbForProject->deleteDocument('attributes', $attribute->getId());
if (!$relatedAttribute->isEmpty()) {
$dbForProject->deleteDocument('attributes', $relatedAttribute->getId());
}
} catch (\Throwable $e) {
// TODO: Send non DatabaseExceptions to Sentry
Console::error($e->getMessage());
if ($e instanceof DatabaseException) {
$attribute->setAttribute('error', $e->getMessage());
if (!$relatedAttribute->isEmpty()) {
$relatedAttribute->setAttribute('error', $e->getMessage());
$dbForProject->deleteDocument('attributes', $relatedAttribute->getId());
}
} catch (\Throwable $e) {
Console::error($e->getMessage());
if ($e instanceof DatabaseException) {
$attribute->setAttribute('error', $e->getMessage());
if (!$relatedAttribute->isEmpty()) {
$relatedAttribute->setAttribute('error', $e->getMessage());
}
}
}
$dbForProject->updateDocument(
'attributes',
$attribute->getId(),
$attribute->setAttribute('status', 'stuck')
);
if (!$relatedAttribute->isEmpty()) {
$dbForProject->updateDocument(
'attributes',
$relatedAttribute->getId(),
$relatedAttribute->setAttribute('status', 'stuck')
$attribute->getId(),
$attribute->setAttribute('status', 'stuck')
);
if (!$relatedAttribute->isEmpty()) {
$dbForProject->updateDocument(
'attributes',
$relatedAttribute->getId(),
$relatedAttribute->setAttribute('status', 'stuck')
);
}
throw $e;
} finally {
$this->trigger($database, $collection, $attribute, $project, $projectId, $events);
}
} finally {
$this->trigger($database, $collection, $attribute, $project, $projectId, $events);
}
// The underlying database removes/rebuilds indexes when attribute is removed
// Update indexes table with changes
/** @var Document[] $indexes */
$indexes = $collection->getAttribute('indexes', []);
// The underlying database removes/rebuilds indexes when attribute is removed
// Update indexes table with changes
/** @var Document[] $indexes */
$indexes = $collection->getAttribute('indexes', []);
foreach ($indexes as $index) {
/** @var string[] $attributes */
$attributes = $index->getAttribute('attributes');
$lengths = $index->getAttribute('lengths');
$orders = $index->getAttribute('orders');
foreach ($indexes as $index) {
/** @var string[] $attributes */
$attributes = $index->getAttribute('attributes');
$lengths = $index->getAttribute('lengths');
$orders = $index->getAttribute('orders');
$found = \array_search($key, $attributes);
$found = \array_search($key, $attributes);
if ($found !== false) {
// If found, remove entry from attributes, lengths, and orders
// array_values wraps array_diff to reindex array keys
// when found attribute is removed from array
$attributes = \array_values(\array_diff($attributes, [$attributes[$found]]));
$lengths = \array_values(\array_diff($lengths, isset($lengths[$found]) ? [$lengths[$found]] : []));
$orders = \array_values(\array_diff($orders, isset($orders[$found]) ? [$orders[$found]] : []));
if ($found !== false) {
// If found, remove entry from attributes, lengths, and orders
// array_values wraps array_diff to reindex array keys
// when found attribute is removed from array
$attributes = \array_values(\array_diff($attributes, [$attributes[$found]]));
$lengths = \array_values(\array_diff($lengths, isset($lengths[$found]) ? [$lengths[$found]] : []));
$orders = \array_values(\array_diff($orders, isset($orders[$found]) ? [$orders[$found]] : []));
if (empty($attributes)) {
$dbForProject->deleteDocument('indexes', $index->getId());
} else {
$index
->setAttribute('attributes', $attributes, Document::SET_TYPE_ASSIGN)
->setAttribute('lengths', $lengths, Document::SET_TYPE_ASSIGN)
->setAttribute('orders', $orders, Document::SET_TYPE_ASSIGN);
// Check if an index exists with the same attributes and orders
$exists = false;
foreach ($indexes as $existing) {
if (
$existing->getAttribute('key') !== $index->getAttribute('key') // Ignore itself
&& $existing->getAttribute('attributes') === $index->getAttribute('attributes')
&& $existing->getAttribute('orders') === $index->getAttribute('orders')
) {
$exists = true;
break;
}
}
if ($exists) { // Delete the duplicate if created, else update in db
$this->deleteIndex($database, $collection, $index, $project, $dbForConsole, $dbForProject);
if (empty($attributes)) {
$dbForProject->deleteDocument('indexes', $index->getId());
} else {
$dbForProject->updateDocument('indexes', $index->getId(), $index);
$index
->setAttribute('attributes', $attributes, Document::SET_TYPE_ASSIGN)
->setAttribute('lengths', $lengths, Document::SET_TYPE_ASSIGN)
->setAttribute('orders', $orders, Document::SET_TYPE_ASSIGN);
// Check if an index exists with the same attributes and orders
$exists = false;
foreach ($indexes as $existing) {
if (
$existing->getAttribute('key') !== $index->getAttribute('key') // Ignore itself
&& $existing->getAttribute('attributes') === $index->getAttribute('attributes')
&& $existing->getAttribute('orders') === $index->getAttribute('orders')
) {
$exists = true;
break;
}
}
if ($exists) { // Delete the duplicate if created, else update in db
$this->deleteIndex($database, $collection, $index, $project, $dbForConsole, $dbForProject);
} else {
$dbForProject->updateDocument('indexes', $index->getId(), $index);
}
}
}
}
}
} finally {
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
$dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
$dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) {
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId());
$dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId());
if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) {
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId());
$dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId());
}
}
}
@@ -370,6 +375,7 @@ class Databases extends Action
* @throws Conflict
* @throws Structure
* @throws DatabaseException
* @throws \Throwable
*/
private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void
{
@@ -401,9 +407,7 @@ class Databases extends Action
}
$dbForProject->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'available'));
} catch (\Throwable $e) {
// TODO: Send non DatabaseExceptions to Sentry
Console::error($e->getMessage());
if ($e instanceof DatabaseException) {
$index->setAttribute('error', $e->getMessage());
}
@@ -412,11 +416,12 @@ class Databases extends Action
$index->getId(),
$index->setAttribute('status', 'failed')
);
throw $e;
} finally {
$this->trigger($database, $collection, $index, $project, $projectId, $events);
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
}
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
}
/**
@@ -431,6 +436,7 @@ class Databases extends Action
* @throws Conflict
* @throws Structure
* @throws DatabaseException
* @throws \Throwable
*/
private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void
{
@@ -459,7 +465,6 @@ class Databases extends Action
$dbForProject->deleteDocument('indexes', $index->getId());
$index->setAttribute('status', 'deleted');
} catch (\Throwable $e) {
// TODO: Send non DatabaseExceptions to Sentry
Console::error($e->getMessage());
if ($e instanceof DatabaseException) {
@@ -470,11 +475,13 @@ class Databases extends Action
$index->getId(),
$index->setAttribute('status', 'stuck')
);
throw $e;
} finally {
$this->trigger($database, $collection, $index, $project, $projectId, $events);
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId());
}
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId());
}
/**
src/Appwrite/Platform/Workers/Deletes.php
+11 -17
View File
@@ -494,30 +494,24 @@ class Deletes extends Action
];
$limit = \count($projectCollectionIds) + 25;
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
while (true) {
$collections = $dbForProject->listCollections($limit);
foreach ($collections as $collection) {
if ($dsn->getHost() !== System::getEnv('_APP_DATABASE_SHARED_TABLES', '') || !\in_array($collection->getId(), $projectCollectionIds)) {
try {
try {
if (!\in_array($dsn->getHost(), $sharedTables) || !\in_array($collection->getId(), $projectCollectionIds)) {
$dbForProject->deleteCollection($collection->getId());
} catch (Throwable $e) {
Console::error('Error deleting '.$collection->getId().' '.$e->getMessage());
/**
* Ignore junction tables;
*/
if (!preg_match('/^_\d+_\d+$/', $collection->getId())) {
throw $e;
}
} else {
$this->deleteByGroup($collection->getId(), [], database: $dbForProject);
}
} else {
$this->deleteByGroup($collection->getId(), [], database: $dbForProject);
} catch (Throwable $e) {
Console::error('Error deleting '.$collection->getId().' '.$e->getMessage());
}
}
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
if (\in_array($dsn->getHost(), $sharedTables)) {
$collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections);
if (empty(\array_diff($collectionsIds, $projectCollectionIds))) {
@@ -571,10 +565,10 @@ class Deletes extends Action
], $dbForConsole);
// Delete metadata table
if ($dsn->getHost() !== System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$dbForProject->deleteCollection('_metadata');
if (!\in_array($dsn->getHost(), $sharedTables)) {
$dbForProject->deleteCollection(Database::METADATA);
} else {
$this->deleteByGroup('_metadata', [], $dbForProject);
$this->deleteByGroup(Database::METADATA, [], $dbForProject);
}
// Delete all storage directories
src/Appwrite/Platform/Workers/Migrations.php
-2
View File
@@ -268,8 +268,6 @@ class Migrations extends Action
$transfer = $source = $destination = null;
try {
$migration = $this->dbForProject->getDocument('migrations', $migration->getId());
if (
$migration->getAttribute('source') === SourceAppwrite::getName() &&
empty($migration->getAttribute('credentials', []))
src/Appwrite/Utopia/Response/Model/Func.php
+1 -1
View File
@@ -94,7 +94,7 @@ class Func extends Model
])
->addRule('schedule', [
'type' => self::TYPE_STRING,
'description' => 'Function execution schedult in CRON format.',
'description' => 'Function execution schedule in CRON format.',
'default' => '',
'example' => '5 4 * * *',
])
tests/e2e/Services/Databases/DatabasesCustomServerTest.php
+1 -1
View File
@@ -1362,7 +1362,7 @@ class DatabasesCustomServerTest extends Scope
]);
$this->assertEquals(400, $tooWide['headers']['status-code']);
$this->assertEquals('Attribute limit exceeded', $tooWide['body']['message']);
$this->assertEquals('attribute_limit_exceeded', $tooWide['body']['type']);
}
public function testIndexLimitException()
tests/e2e/Services/Projects/ProjectsConsoleClientTest.php
+99 -2
View File
@@ -3727,11 +3727,11 @@ class ProjectsConsoleClientTest extends Scope
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'teamId' => ID::unique(),
'name' => 'Amating Team',
'name' => 'Amazing Team',
]);
$this->assertEquals(201, $team['headers']['status-code']);
$this->assertEquals('Amating Team', $team['body']['name']);
$this->assertEquals('Amazing Team', $team['body']['name']);
$this->assertNotEmpty($team['body']['$id']);
$teamId = $team['body']['$id'];
@@ -3794,6 +3794,103 @@ class ProjectsConsoleClientTest extends Scope
return $data;
}
public function testDeleteSharedProject(): void
{
$team = $this->client->call(Client::METHOD_POST, '/teams', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'teamId' => ID::unique(),
'name' => 'Amazing Team',
]);
$teamId = $team['body']['$id'];
// Ensure deleting one project does not affect another project
$project1 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'projectId' => ID::unique(),
'name' => 'Amazing Project 1',
'teamId' => $teamId,
'region' => 'default'
]);
$project2 = $this->client->call(Client::METHOD_POST, '/projects', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'projectId' => ID::unique(),
'name' => 'Amazing Project 2',
'teamId' => $teamId,
'region' => 'default'
]);
$project1Id = $project1['body']['$id'];
$project2Id = $project2['body']['$id'];
// Create user in each project
$key1 = $this->client->call(Client::METHOD_POST, '/projects/' . $project1Id . '/keys', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Key Test',
'scopes' => ['users.read', 'users.write'],
]);
$user1 = $this->client->call(Client::METHOD_POST, '/users', [
'content-type' => 'application/json',
'x-appwrite-project' => $project1Id,
'x-appwrite-key' => $key1['body']['secret'],
], [
'userId' => ID::unique(),
'email' => 'test1@appwrite.io',
'password' => 'password',
]);
$this->assertEquals(201, $user1['headers']['status-code']);
$key2 = $this->client->call(Client::METHOD_POST, '/projects/' . $project2Id . '/keys', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Key Test',
'scopes' => ['users.read', 'users.write'],
]);
$user2 = $this->client->call(Client::METHOD_POST, '/users', [
'content-type' => 'application/json',
'x-appwrite-project' => $project2Id,
'x-appwrite-key' => $key2['body']['secret'],
], [
'userId' => ID::unique(),
'email' => 'test2@appwrite.io',
'password' => 'password',
]);
$this->assertEquals(201, $user2['headers']['status-code']);
// Delete project 1
$project1 = $this->client->call(Client::METHOD_DELETE, '/projects/' . $project1Id, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$this->assertEquals(204, $project1['headers']['status-code']);
\sleep(3);
// Ensure project 2 user is still there
$user2 = $this->client->call(Client::METHOD_GET, '/users/' . $user2['body']['$id'], [
'content-type' => 'application/json',
'x-appwrite-project' => $project2Id,
'x-appwrite-key' => $key2['body']['secret'],
]);
$this->assertEquals(200, $user2['headers']['status-code']);
}
/**
* @depends testCreateProject
*/
tests/e2e/Services/Realtime/RealtimeCustomClientTest.php
+24
View File
@@ -111,6 +111,30 @@ class RealtimeCustomClientTest extends Scope
$client->close();
}
public function testPingPong()
{
$client = $this->getWebsocket(['files'], [
'origin' => 'http://localhost'
]);
$response = json_decode($client->receive(), true);
$this->assertArrayHasKey('type', $response);
$this->assertArrayHasKey('data', $response);
$this->assertEquals('connected', $response['type']);
$this->assertNotEmpty($response['data']);
$this->assertCount(1, $response['data']['channels']);
$this->assertContains('files', $response['data']['channels']);
$client->send(\json_encode([
'type' => 'ping'
]));
$response = json_decode($client->receive(), true);
$this->assertEquals('pong', $response['type']);
$client->close();
}
public function testManualAuthentication()
{
$user = $this->getUser();
tests/e2e/Services/Teams/TeamsCustomClientTest.php
+32
View File
@@ -83,6 +83,38 @@ class TeamsCustomClientTest extends Scope
$this->assertNotEmpty($response['body']['memberships'][0]['userName']);
$this->assertNotEmpty($response['body']['memberships'][0]['userEmail']);
$this->assertArrayHasKey('mfa', $response['body']['memberships'][0]);
/**
* Update project settings to show only MFA
*/
$response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $this->getProject()['$id'] . '/auth/memberships-privacy', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => 'console',
'cookie' => 'a_session_console=' . $this->getRoot()['session'],
]), [
'userName' => false,
'userEmail' => false,
'mfa' => true,
]);
$this->assertEquals(200, $response['headers']['status-code']);
/**
* Test that sensitive fields are not shown
*/
$response = $this->client->call(Client::METHOD_GET, '/teams/' . $teamUid . '/memberships', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $projectId,
], $this->getHeaders()));
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertIsInt($response['body']['total']);
$this->assertNotEmpty($response['body']['memberships'][0]['$id']);
// Assert that sensitive fields are present
$this->assertEmpty($response['body']['memberships'][0]['userName']);
$this->assertEmpty($response['body']['memberships'][0]['userEmail']);
$this->assertArrayHasKey('mfa', $response['body']['memberships'][0]);
}
/**