Merge branch '1.6.x' into fix-db-perf-improvements

This commit is contained in:
Luke B. Silver
2025-03-05 20:06:31 +00:00
committed by GitHub
11 changed files with 165 additions and 155 deletions
+1 -1
View File
@@ -2400,7 +2400,7 @@ App::put('/v1/account/sessions/phone')
App::post('/v1/account/tokens/phone')
->alias('/v1/account/sessions/phone')
->desc('Create phone token')
->groups(['api', 'account'])
->groups(['api', 'account', 'auth'])
->label('scope', 'sessions.write')
->label('auth.type', 'phone')
->label('audits.event', 'session.create')
+13 -10
View File
@@ -243,8 +243,8 @@ function updateAttribute(
string $filter = null,
string|bool|int|float $default = null,
bool $required = null,
int|float $min = null,
int|float $max = null,
int|float|null $min = null,
int|float|null $max = null,
array $elements = null,
array $options = [],
string $newKey = null,
@@ -300,6 +300,9 @@ function updateAttribute(
switch ($attribute->getAttribute('format')) {
case APP_DATABASE_ATTRIBUTE_INT_RANGE:
case APP_DATABASE_ATTRIBUTE_FLOAT_RANGE:
$min ??= $attribute->getAttribute('formatOptions')['min'];
$max ??= $attribute->getAttribute('formatOptions')['max'];
if ($min > $max) {
throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Minimum value must be lesser than maximum value');
}
@@ -1560,8 +1563,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege
->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?int $min, ?int $max, ?int $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) {
// Ensure attribute default is within range
$min = \is_null($min) ? PHP_INT_MIN : $min;
$max = \is_null($max) ? PHP_INT_MAX : $max;
$min ??= PHP_INT_MIN;
$max ??= PHP_INT_MAX;
if ($min > $max) {
throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Minimum value must be lesser than maximum value');
@@ -1637,8 +1640,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float'
->action(function (string $databaseId, string $collectionId, string $key, ?bool $required, ?float $min, ?float $max, ?float $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) {
// Ensure attribute default is within range
$min = \is_null($min) ? -PHP_FLOAT_MAX : $min;
$max = \is_null($max) ? PHP_FLOAT_MAX : $max;
$min ??= -PHP_FLOAT_MAX;
$max ??= PHP_FLOAT_MAX;
if ($min > $max) {
throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Minimum value must be lesser than maximum value');
@@ -2349,8 +2352,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/integ
->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).')
->param('key', '', new Key(), 'Attribute Key.')
->param('required', null, new Boolean(), 'Is attribute required?')
->param('min', null, new Integer(), 'Minimum value to enforce on new documents')
->param('max', null, new Integer(), 'Maximum value to enforce on new documents')
->param('min', null, new Integer(), 'Minimum value to enforce on new documents', true)
->param('max', null, new Integer(), 'Maximum value to enforce on new documents', true)
->param('default', null, new Nullable(new Integer()), 'Default value for attribute when not provided. Cannot be set when attribute is required.')
->param('newKey', null, new Key(), 'New attribute key.', true)
->inject('response')
@@ -2408,8 +2411,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/float
->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).')
->param('key', '', new Key(), 'Attribute Key.')
->param('required', null, new Boolean(), 'Is attribute required?')
->param('min', null, new FloatValidator(), 'Minimum value to enforce on new documents')
->param('max', null, new FloatValidator(), 'Maximum value to enforce on new documents')
->param('min', null, new FloatValidator(), 'Minimum value to enforce on new documents', true)
->param('max', null, new FloatValidator(), 'Maximum value to enforce on new documents', true)
->param('default', null, new Nullable(new FloatValidator()), 'Default value for attribute when not provided. Cannot be set when attribute is required.')
->param('newKey', null, new Key(), 'New attribute key.', true)
->inject('response')
@@ -182,7 +182,7 @@ class StatsResources extends Action
}
try {
$this->countForDatabase($dbForProject, $dbForLogs, $region);
$this->countForDatabase($dbForProject, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]);
}
@@ -242,42 +242,54 @@ class StatsResources extends Action
$this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalImageTransformations);
}
protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region)
protected function countForDatabase(Database $dbForProject, string $region)
{
$totalCollections = 0;
$totalDocuments = 0;
$this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $dbForLogs, $region, &$totalCollections, &$totalDocuments) {
$totalDatabaseStorage = 0;
$this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $region, &$totalCollections, &$totalDocuments, &$totalDatabaseStorage) {
$collections = $dbForProject->count('database_' . $database->getInternalId());
$metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS);
$this->createStatsDocuments($region, $metric, $collections);
$documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region);
[$documents, $storage] = $this->countForCollections($dbForProject, $database, $region);
$totalDatabaseStorage += $storage;
$totalDocuments += $documents;
$totalCollections += $collections;
});
$this->createStatsDocuments($region, METRIC_COLLECTIONS, $totalCollections);
$this->createStatsDocuments($region, METRIC_DOCUMENTS, $totalDocuments);
$this->createStatsDocuments($region, METRIC_DATABASES_STORAGE, $totalDatabaseStorage);
}
protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int
protected function countForCollections(Database $dbForProject, Document $database, string $region): array
{
$databaseDocuments = 0;
$this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $dbForLogs, $database, $region, &$totalCollections, &$databaseDocuments) {
$databaseStorage = 0;
$this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $database, $region, &$databaseStorage, &$databaseDocuments) {
$documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
$metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS);
$this->createStatsDocuments($region, $metric, $documents);
$databaseDocuments += $documents;
$collectionStorage = $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
$metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE);
$this->createStatsDocuments($region, $metric, $collectionStorage);
$databaseStorage += $collectionStorage;
});
$metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS);
$this->createStatsDocuments($region, $metric, $databaseDocuments);
return $databaseDocuments;
$metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE);
$this->createStatsDocuments($region, $metric, $databaseStorage);
return [$databaseDocuments, $databaseStorage];
}
protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region)
@@ -48,6 +48,7 @@ class StatsUsageDump extends Action
METRIC_BUILDS => true,
METRIC_COLLECTIONS => true,
METRIC_DOCUMENTS => true,
METRIC_DATABASES_STORAGE => true,
];
/**
@@ -63,6 +64,7 @@ class StatsUsageDump extends Action
'.deployments.storage',
'.builds',
'.builds.storage',
'.databases.storage'
];
/**
@@ -197,13 +197,14 @@ class UsageProject extends Model
'example' => [],
'array' => true
])
->addRule('imageTransformationsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'An array of aggregated number of image transformations.',
'default' => 0,
'example' => 0,
])
->addRule('imageTransformations', [
'type' => Response::MODEL_METRIC,
'description' => 'An array of aggregated number of image transformations.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('imageTransformationsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of image transformations.',
'default' => 0,
+28 -39
View File
@@ -1142,49 +1142,38 @@ class UsageTest extends Scope
$tries = 0;
while (true) {
try {
// Compare new values with old values
$response = $this->client->call(
Client::METHOD_GET,
'/functions/' . $functionId . '/usage?range=30d',
$this->getConsoleHeaders()
);
$this->assertEventually(function () use ($functionId, $functionsMetrics, $projectMetrics) {
// Compare new values with old values
$response = $this->client->call(
Client::METHOD_GET,
'/functions/' . $functionId . '/usage?range=30d',
$this->getConsoleHeaders()
);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(19, count($response['body']));
$this->assertEquals('30d', $response['body']['range']);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(19, count($response['body']));
$this->assertEquals('30d', $response['body']['range']);
// Check if the new values are greater than the old values
$this->assertEquals($functionsMetrics['executionsTotal'] + 1, $response['body']['executionsTotal']);
$this->assertGreaterThan($functionsMetrics['executionsTimeTotal'], $response['body']['executionsTimeTotal']);
$this->assertGreaterThan($functionsMetrics['executionsMbSecondsTotal'], $response['body']['executionsMbSecondsTotal']);
// Check if the new values are greater than the old values
$this->assertEquals($functionsMetrics['executionsTotal'] + 1, $response['body']['executionsTotal']);
$this->assertGreaterThan($functionsMetrics['executionsTimeTotal'], $response['body']['executionsTimeTotal']);
$this->assertGreaterThan($functionsMetrics['executionsMbSecondsTotal'], $response['body']['executionsMbSecondsTotal']);
$response = $this->client->call(
Client::METHOD_GET,
'/project/usage',
$this->getConsoleHeaders(),
[
'period' => '1h',
'startDate' => self::getToday(),
'endDate' => self::getTomorrow(),
]
);
$response = $this->client->call(
Client::METHOD_GET,
'/project/usage',
$this->getConsoleHeaders(),
[
'period' => '1h',
'startDate' => self::getToday(),
'endDate' => self::getTomorrow(),
]
);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals($projectMetrics['executionsTotal'] + 1, $response['body']['executionsTotal']);
$this->assertGreaterThan($projectMetrics['executionsMbSecondsTotal'], $response['body']['executionsMbSecondsTotal']);
break;
} catch (ExpectationFailedException $th) {
if ($tries >= 5) {
throw $th;
} else {
$tries++;
sleep(5);
}
}
}
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals($projectMetrics['executionsTotal'] + 1, $response['body']['executionsTotal']);
$this->assertGreaterThan($projectMetrics['executionsMbSecondsTotal'], $response['body']['executionsMbSecondsTotal']);
});
}
public function tearDown(): void
@@ -2419,6 +2419,33 @@ class AccountCustomClientTest extends Scope
$message = $smsRequest['data']['message'];
$token = substr($message, 0, 6);
/**
* Test for FAILURE
*/
// disable phone sessions
$response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $this->getProject()['$id'] . '/auth/phone', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => 'console',
'cookie' => 'a_session_console=' . $this->getRoot()['session'],
]), [
'status' => false,
]);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(false, $response['body']['authPhone']);
$response = $this->client->call(Client::METHOD_POST, '/account/verification/phone', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]));
$this->assertEquals(501, $response['headers']['status-code']);
$this->assertEquals("Phone authentication is disabled for this project", $response['body']['message']);
return \array_merge($data, [
'token' => \substr($smsRequest['data']['message'], 0, 6)
]);
@@ -2309,6 +2309,30 @@ class DatabasesCustomServerTest extends Scope
$this->assertEquals(100, $new['body']['min']);
$this->assertEquals(2000, $new['body']['max']);
$update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'required' => false,
'default' => 100,
'min' => 0,
]);
$this->assertEquals(200, $update['headers']['status-code']);
$update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'required' => false,
'default' => 10,
'max' => 100,
]);
$this->assertEquals(200, $update['headers']['status-code']);
/**
* Test against failure
*/
@@ -2368,32 +2392,6 @@ class DatabasesCustomServerTest extends Scope
$this->assertEquals(400, $update['headers']['status-code']);
$this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']);
$update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'required' => false,
'default' => 100,
'min' => 0,
]);
$this->assertEquals(400, $update['headers']['status-code']);
$this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']);
$update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'required' => false,
'default' => 100,
'max' => 0,
]);
$this->assertEquals(400, $update['headers']['status-code']);
$this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']);
$update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer/' . $key, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
@@ -2572,6 +2570,30 @@ class DatabasesCustomServerTest extends Scope
$this->assertEquals(123.456, $new['body']['min']);
$this->assertEquals(2000, $new['body']['max']);
$update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'required' => false,
'default' => 123.456,
'min' => 0.0,
]);
$this->assertEquals(200, $update['headers']['status-code']);
$update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'required' => false,
'default' => 23.456,
'max' => 100.0,
]);
$this->assertEquals(200, $update['headers']['status-code']);
/**
* Test against failure
*/
@@ -2631,32 +2653,6 @@ class DatabasesCustomServerTest extends Scope
$this->assertEquals(400, $update['headers']['status-code']);
$this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']);
$update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'required' => false,
'default' => 123.456,
'min' => 0.0,
]);
$this->assertEquals(400, $update['headers']['status-code']);
$this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']);
$update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'required' => false,
'default' => 123.456,
'max' => 0.0,
]);
$this->assertEquals(400, $update['headers']['status-code']);
$this->assertEquals(AppwriteException::GENERAL_ARGUMENT_INVALID, $update['body']['type']);
$update = $this->client->call(Client::METHOD_PATCH, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/float/' . $key, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
@@ -53,8 +53,9 @@ trait MigrationsBase
$this->assertNotEmpty($migration['body']);
$this->assertNotEmpty($migration['body']['$id']);
$attempts = 0;
while ($attempts < 5) {
$migrationResult = [];
$this->assertEventually(function () use ($migration, &$migrationResult) {
$response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migration['body']['$id'], [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getDestinationProject()['$id'],
@@ -66,24 +67,18 @@ trait MigrationsBase
$this->assertNotEmpty($response['body']['$id']);
if ($response['body']['status'] === 'failed') {
$this->fail('Migration failed', json_encode($response['body'], JSON_PRETTY_PRINT));
$this->fail('Migration failed' . json_encode($response['body'], JSON_PRETTY_PRINT));
}
$this->assertNotEquals('failed', $response['body']['status']);
$this->assertEquals('completed', $response['body']['status']);
if ($response['body']['status'] === 'completed') {
return $response['body'];
}
$migrationResult = $response['body'];
if ($attempts === 4) {
$this->assertEquals('completed', $response['body']['status']);
}
return true;
});
$attempts++;
sleep(5);
}
return [];
return $migrationResult;
}
/**
@@ -1312,22 +1312,15 @@ class RealtimeCustomClientTest extends Scope
$this->assertNotEmpty($deployment['body']['$id']);
// Poll until deployment is built
while (true) {
$this->assertEventually(function () use ($function, $deploymentId) {
$deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]);
if (
$deployment['headers']['status-code'] >= 400
|| \in_array($deployment['body']['status'], ['ready', 'failed'])
) {
break;
}
\sleep(1);
}
$this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body']));
});
$response = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([
'content-type' => 'application/json',
+6 -14
View File
@@ -2,6 +2,7 @@
namespace Tests\E2E\Services\Webhooks;
use Appwrite\Tests\Async;
use Appwrite\Tests\Retry;
use CURLFile;
use Tests\E2E\Client;
@@ -12,29 +13,20 @@ use Utopia\Database\Validator\Datetime as DatetimeValidator;
trait WebhooksBase
{
protected function awaitDeploymentIsBuilt($functionId, $deploymentId, $checkForSuccess = true): void
use Async;
protected function awaitDeploymentIsBuilt($functionId, $deploymentId): void
{
while (true) {
$this->assertEventually(function () use ($functionId, $deploymentId) {
$deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/deployments/' . $deploymentId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]);
if (
$deployment['headers']['status-code'] >= 400
|| \in_array($deployment['body']['status'], ['ready', 'failed'])
) {
break;
}
\sleep(1);
}
if ($checkForSuccess) {
$this->assertEquals(200, $deployment['headers']['status-code']);
$this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body']));
}
});
}
public static function getWebhookSignature(array $webhook, string $signatureKey): string