Compare commits

...
143 Commits
Author SHA1 Message Date
Jake Barnby fffa8bdc7f Update lock 2025-01-30 22:58:48 +13:00
Jake Barnby 7f699910b6 Merge remote-tracking branch 'origin/1.6.x' into poc-timeout
# Conflicts:
#	composer.lock
2025-01-30 22:52:16 +13:00
Jake Barnby e8722c10e8 Update tests 2025-01-29 23:41:43 +13:00
Jake Barnby d32299fe1d Format 2025-01-28 22:12:13 +13:00
Jake Barnby 3d0884aaec Update hooks 2025-01-28 22:10:00 +13:00
Jake Barnby 3b71215b4f Fix merge 2025-01-28 21:55:11 +13:00
Jake Barnby 34074f74a7 Merge remote-tracking branch 'origin/1.6.x' into poc-timeout
# Conflicts:
#	.env
#	app/config/collections.php
#	app/config/errors.php
#	app/controllers/api/databases.php
#	app/views/install/compose.phtml
#	composer.lock
#	docker-compose.yml
#	src/Appwrite/Extend/Exception.php
#	src/Appwrite/Utopia/Response.php
2025-01-28 21:46:12 +13:00
fogelito cd5f6d6ecd uncomment appwrite-assistant 2023-08-15 11:50:27 +03:00
fogelito a0057a6e5d Merge branch '1.4.x' of github.com:appwrite/appwrite into poc-timeout 2023-08-15 11:49:35 +03:00
fogelito 943a7af885 Slow queries response model 2023-08-15 11:47:04 +03:00
Jake BarnbyandGitHub c93aea7669 Update docker-compose.yml 2023-08-15 01:35:59 -04:00
Jake Barnby 603c2083ef Uncomment service 2023-08-15 01:24:37 -04:00
Jake Barnby 0ceef6b24f Add slow query model 2023-08-15 01:06:52 -04:00
Jake Barnby 7cf899b302 Fix tests 2023-08-15 00:12:38 -04:00
Jake Barnby 5128c10348 Fix tests 2023-08-14 23:08:56 -04:00
Jake Barnby 479b41d914 Fix tests 2023-08-14 22:24:15 -04:00
fogelito 86abba7618 fix collections.php ID::custom 2023-08-15 00:50:40 +03:00
fogelito 0522affba0 fix collections.php 2023-08-15 00:41:34 +03:00
fogelito ab9ee84ed0 composer.lock 2023-08-14 23:36:15 +03:00
fogelito e67292a614 Merge branch '1.4.x' of github.com:appwrite/appwrite into poc-timeout
 Conflicts:
	composer.lock
	tests/e2e/Services/Databases/DatabasesBase.php
2023-08-14 22:01:31 +03:00
fogelito 7519e281a3 composer 2023-08-14 21:58:18 +03:00
fogelito 94c7fd74e4 composer versions 2023-08-09 18:27:31 +03:00
fogelito 41ca0c4165 composer.lock 2023-08-09 17:51:37 +03:00
fogelito 5578cea29f comment out appwrite-assistant 2023-08-09 17:20:05 +03:00
fogelito aa48432acd composer.json update 2023-08-09 13:10:29 +03:00
fogelito 7c7678b0cb composer.lock 2023-08-09 13:03:58 +03:00
fogelito a46a405b5c docker-compose.yml fixed 2023-08-09 13:02:08 +03:00
fogelito fa13eb1064 .env 2023-08-09 13:00:10 +03:00
fogelito c823e36183 remove unchanged setResource 2023-08-09 12:47:26 +03:00
ecb142fa50 Update app/controllers/api/databases.php
Co-authored-by: Jake Barnby <jakeb994@gmail.com>
2023-08-09 12:40:18 +03:00
fogelito aceb49505c pull changes 2023-08-09 12:38:34 +03:00
fogelito eed89b9fd3 Merge branch '1.4.x' of github.com:appwrite/appwrite into poc-timeout
 Conflicts:
	.env
	app/controllers/api/databases.php
	composer.json
	composer.lock
	docker-compose.yml
2023-08-09 12:36:23 +03:00
fogelito e6cf0a6ac3 addressing comments 2023-08-09 12:27:32 +03:00
fogelito 0f57d87c55 composer.lock 2023-08-01 11:32:38 +03:00
fogelito d61921573b composer 2023-08-01 11:32:23 +03:00
fogelito 82c9c074bc merge in master 2023-08-01 11:29:39 +03:00
fogelito 632d41ec6c Merge branch 'master' of github.com:appwrite/appwrite into poc-timeout
 Conflicts:
	app/controllers/api/databases.php
	composer.json
	composer.lock
	tests/e2e/Services/Databases/DatabasesBase.php
2023-08-01 11:25:21 +03:00
fogelito 59dd6bfbcc composer update 2023-08-01 11:13:14 +03:00
fogelito 54cae4ab00 composer update 2023-08-01 11:13:01 +03:00
fogelito 1e9daf98cb .env _APP_SLOW_QUERIES_MAX_HITS set to 5 2023-03-06 11:42:38 +02:00
fogelito 408e611b76 .env _APP_SLOW_QUERIES_MAX_HITS set to 5 2023-03-06 11:40:35 +02:00
fogelito bd582bd00d addressing comments 2023-03-06 10:57:32 +02:00
fogelito 25aa6049cd space removes 2023-03-06 10:45:56 +02:00
fogelito 2a7ca5fe67 set setResource return 2023-03-06 10:41:50 +02:00
fogelito 13b52a67a9 datetime validator name change 2023-03-06 10:26:20 +02:00
fogelito 466a469035 Merge remote-tracking branch 'origin/poc-timeout' into poc-timeout 2023-03-06 10:09:39 +02:00
fogelito 96a78a0763 collection require = true 2023-03-06 10:08:09 +02:00
33fb110bac Update docs/references/databases/delete-slow-query-document.md
Co-authored-by: Jake Barnby <jakeb994@gmail.com>
2023-03-06 09:57:53 +02:00
0936072e17 Update app/config/errors.php
Co-authored-by: Jake Barnby <jakeb994@gmail.com>
2023-03-02 09:05:59 +02:00
fogelito d34af2764d stopOnFailure="false" 2023-03-02 09:04:36 +02:00
fogelito 05485afb99 remove cursor hint + merge in 0.13.x 2023-03-02 08:58:58 +02:00
fogelito ef6a95252e Merge branch '1.3.x' of github.com:appwrite/appwrite into poc-timeout 2023-03-02 08:56:36 +02:00
fogelito 9322d1636e Last fixed before pull 2023-03-01 19:05:15 +02:00
fogelito 2f46d5c4a6 tests fixes 2023-03-01 18:39:38 +02:00
fogelito 3d3ffba93a console route change 2023-03-01 18:32:13 +02:00
72b30d4f60 Update app/config/errors.php
Co-authored-by: Jake Barnby <jakeb994@gmail.com>
2023-03-01 16:12:57 +02:00
962a44b3a8 Update app/controllers/api/databases.php
Co-authored-by: Jake Barnby <jakeb994@gmail.com>
2023-03-01 16:12:15 +02:00
6b884150cd Update app/controllers/api/databases.php
Co-authored-by: Jake Barnby <jakeb994@gmail.com>
2023-03-01 16:12:02 +02:00
fogelito c315240bce addressing comments 2023-03-01 15:56:16 +02:00
fogelito c0febfff5b Merge remote-tracking branch 'origin/poc-timeout' into poc-timeout 2023-02-28 19:34:17 +02:00
fogelito 3f9b8afbb4 addressing comments 2023-02-28 19:34:04 +02:00
707c8033b8 Update app/controllers/api/databases.php
Co-authored-by: Jake Barnby <jakeb994@gmail.com>
2023-02-28 19:21:12 +02:00
ae981d90c9 Update app/controllers/api/databases.php
Co-authored-by: Jake Barnby <jakeb994@gmail.com>
2023-02-28 18:39:11 +02:00
3964e86340 Update app/controllers/api/databases.php
Co-authored-by: Jake Barnby <jakeb994@gmail.com>
2023-02-28 18:34:38 +02:00
5a1ac151d9 Update app/controllers/api/databases.php
Co-authored-by: Jake Barnby <jakeb994@gmail.com>
2023-02-28 18:32:47 +02:00
fogelito 031d85e5d6 remove comment 2023-02-23 12:26:24 +02:00
fogelito ad320e97c6 fix empty queries 2023-02-23 12:15:11 +02:00
fogelito 497e505775 longtext.txt 2023-02-23 10:57:57 +02:00
fogelito 947193ef24 _APP_SLOW_QUERIES enabled 2023-02-23 10:09:43 +02:00
fogelito 79e60e84da collections.php changes 2023-02-23 10:06:16 +02:00
03e3522568 _APP_SLOW_QUERIES_BLOCK
Co-authored-by: Jake Barnby <jakeb994@gmail.com>
2023-02-23 10:05:25 +02:00
fogelito 8a30738f52 remove var_dump graphql 2023-02-23 09:46:40 +02:00
fogelito 188abb04a7 remove sleep 2023-02-21 13:33:09 +02:00
fogelito 04f2a0610d remove sleep 2023-02-21 13:22:28 +02:00
fogelito 89c9c293e8 Less docs for timeout? 2023-02-21 10:20:12 +02:00
fogelito 601e1648a7 Timeouts tests 2023-02-21 10:11:08 +02:00
fogelito 470ff2289d Timeouts tests 2023-02-21 10:05:23 +02:00
Jake Barnby 3a2ce3bf2b Fix graphql tests 2023-02-21 13:59:23 +13:00
fogelito 211ef70279 native timeouts try 2023-02-20 12:06:09 +02:00
fogelito 2f37057e7a native timeouts try 2023-02-20 11:36:57 +02:00
fogelito c3a6cefd0a native timeouts try 2023-02-20 11:06:00 +02:00
fogelito 82bcbf3306 native timeouts try 2023-02-16 19:48:15 +02:00
fogelito f6a27cbd84 remove line 2023-02-16 11:50:31 +02:00
fogelito 318076297b graphQL tests fails 2023-02-16 10:42:08 +02:00
fogelito 93c25e87d5 tests updates 2023-02-15 13:36:30 +02:00
fogelito ea0c7e9a4c composer.lock 2023-02-15 12:45:19 +02:00
fogelito 95e204b8d9 filter todo 2023-02-15 12:21:03 +02:00
fogelito 72a30cf65f Merge branch 'master' of github.com:appwrite/appwrite into poc-timeout
 Conflicts:
	app/controllers/api/databases.php
	app/init.php
	composer.json
	composer.lock
	tests/e2e/General/UsageTest.php
	tests/e2e/Services/Account/AccountCustomClientTest.php
	tests/e2e/Services/Databases/DatabasesBase.php
	tests/e2e/Services/Databases/DatabasesCustomServerTest.php
	tests/e2e/Services/Functions/FunctionsCustomServerTest.php
	tests/e2e/Services/Storage/StorageBase.php
	tests/e2e/Services/Storage/StorageCustomClientTest.php
	tests/e2e/Services/Storage/StorageCustomServerTest.php
	tests/e2e/Services/Teams/TeamsBase.php
	tests/e2e/Services/Teams/TeamsBaseClient.php
	tests/e2e/Services/Webhooks/WebhooksBase.php
	tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php
	tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php
2023-02-15 11:22:35 +02:00
fogelito 558869b118 update tests 2023-02-14 18:33:00 +02:00
fogelito ff2a300caf _APP_SLOW_QUERIES bool 2023-02-14 15:14:16 +02:00
fogelito 49fe9a9fea timeouts tests pass 2023-02-14 15:09:58 +02:00
fogelito 57c8cde568 timeouts tests pass 2023-02-14 14:42:05 +02:00
fogelito e2bb3270b3 composer .lock 2023-02-14 13:06:23 +02:00
fogelito 895c546884 403 error 2023-02-14 13:05:59 +02:00
fogelito 60c7d19be6 env variables 2023-02-14 09:39:19 +02:00
fogelito a71e23a943 env variables 2023-02-14 09:35:03 +02:00
fogelito dd1b9a5338 env variables 2023-02-13 19:24:29 +02:00
fogelito a90b3eb307 build after changes 2023-02-13 19:11:57 +02:00
fogelito 8779ce855d slow_queries 2023-02-13 18:38:57 +02:00
fogelito 84609e208f slow queries collection name to snake case 2023-02-13 18:34:22 +02:00
fogelito 9650d9867c _APP_SLOW_QUERIES_ACTIVE 2023-02-13 17:50:12 +02:00
fogelito d2142e7313 add helpers namespace 2023-02-13 17:43:58 +02:00
fogelito 238c405fc0 .env variables for slow queries 2023-02-13 17:23:05 +02:00
fogelito 7cd1ffc085 DatabasesCustomClientTest.php uncomment tests 2023-02-12 23:53:52 +02:00
fogelito bf84ac3adc Merge remote-tracking branch 'origin/poc-timeout' into poc-timeout 2023-02-12 23:51:35 +02:00
fogelito 3e66b764ec resolve comments $timeoutMilliseconds 2023-02-12 23:51:24 +02:00
167d127481 Update app/init.php
Co-authored-by: Eldad A. Fux <eldad.fux@gmail.com>
2023-02-12 23:47:31 +02:00
09128267e0 Update docker-compose.yml
Co-authored-by: Eldad A. Fux <eldad.fux@gmail.com>
2023-02-12 23:47:23 +02:00
a32019e8dc Update app/controllers/api/databases.php
Co-authored-by: Eldad A. Fux <eldad.fux@gmail.com>
2023-02-12 23:46:48 +02:00
87bc7ad8b0 Update app/controllers/api/databases.php
Co-authored-by: Eldad A. Fux <eldad.fux@gmail.com>
2023-02-12 23:35:09 +02:00
fogelito cf96f8baca change timeout colllection to slow queries 2023-02-12 23:33:42 +02:00
fogelito 3d76da1f25 change timeout colllection to slow queries 2023-02-12 23:05:10 +02:00
fogelito e5cd1dd842 Merge remote-tracking branch 'origin/poc-timeout' into poc-timeout 2023-02-12 22:49:37 +02:00
fogelito 3841287fc2 composer changes 2023-02-12 22:49:12 +02:00
fogelito 9cfeab7c64 default 2023-02-05 17:36:14 +02:00
fogelito 3a88457052 default 2023-02-05 17:21:08 +02:00
fogelito 00abaef65f add method key 2023-02-05 16:51:04 +02:00
fogelito d01be37a9c md5 2023-02-05 16:49:22 +02:00
fogelito 898f02e1c8 Update list with databaseId 2023-02-02 10:55:23 +02:00
fogelito 593f7154a4 Update list with databaseId 2023-01-31 18:17:07 +02:00
fogelito 3ac2adf13f Update list with databaseId 2023-01-31 18:16:24 +02:00
fogelito 76592b8ea4 Tests for new routes 2023-01-30 17:04:26 +02:00
fogelito b19365a35f Tests for new routes 2023-01-30 16:59:39 +02:00
fogelito 6aab350669 change route names 2023-01-30 12:36:15 +02:00
fogelito f58581663e framework upgrade 0.27.0 2023-01-30 10:38:29 +02:00
fogelito b95f52edf7 Some tests 2023-01-29 14:40:27 +02:00
fogelito 73a2d103fb Query filters + cursor for list slow queries 2023-01-29 14:33:52 +02:00
fogelito 14a6f407b8 timeout get 2023-01-25 11:07:19 +02:00
fogelito 1bcb2b561a timeout get 2023-01-25 11:05:49 +02:00
fogelito bad3b40ae9 timeout patch 2023-01-25 10:48:44 +02:00
fogelito a69d6fd3f8 New timeout rout for list blocked docs 2023-01-24 19:30:59 +02:00
fogelito 55ca917edf Code clean up 2023-01-22 14:01:22 +02:00
fogelito d610a14388 Code clean up 2023-01-19 17:51:23 +02:00
fogelito 017ef1f13c remove comments 2023-01-19 11:20:42 +02:00
fogelito 1a7e65d339 error resource changes 2023-01-19 11:19:16 +02:00
fogelito 317469c8fc debugging var dumps 2023-01-18 19:25:18 +02:00
fogelito d615674da6 Some changes 2023-01-17 19:10:41 +02:00
fogelito c39235410e remove the hardcoded throw Timeout Exception; 2023-01-16 08:41:20 +02:00
fogelito 40bd8d54e5 remove the hardcoded throw Timeout Exception; 2023-01-16 08:32:47 +02:00
fogelito 3c5b14abe7 add tests for sleep operator 2023-01-15 12:57:41 +02:00
fogelito e746634491 validate the sleep operator 2023-01-12 18:30:34 +02:00
fogelito 9241d37c26 starting using hooks 2023-01-11 17:38:28 +02:00
fogelito 347d4afa24 Some POC for exceed timeout case 2023-01-08 12:05:50 +02:00
17 changed files with 55195 additions and 30 deletions
+3
View File
@@ -110,3 +110,6 @@ _APP_MESSAGE_PUSH_TEST_DSN=
_APP_WEBHOOK_MAX_FAILED_ATTEMPTS=10
_APP_PROJECT_REGIONS=default
_APP_FUNCTIONS_CREATION_ABUSE_LIMIT=5000
_APP_SLOW_QUERIES_MAX_HITS=5
_APP_SLOW_QUERIES_TIMEOUT=15000
_APP_SLOW_QUERIES_BLOCK=enabled
+89
View File
@@ -2637,4 +2637,93 @@ return [
]
]
],
'slowQueries' => [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('slowQueries'),
'name' => 'slowQueries',
'attributes' => [
[
'$id' => ID::custom('blocked'),
'type' => Database::VAR_BOOLEAN,
'format' => '',
'size' => 0,
'signed' => true,
'required' => true,
'default' => false,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('count'),
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('queries'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 410002,
'signed' => false,
'required' => true,
'default' => [],
'array' => false,
'filters' => ['json'],
],
[
'$id' => ID::custom('path'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2500,
'signed' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('databaseId'),
'type' => Database::VAR_STRING,
'signed' => true,
'size' => Database::LENGTH_KEY,
'format' => '',
'filters' => [],
'required' => true,
'default' => null,
'array' => false,
],
[
'$id' => ID::custom('collectionId'),
'type' => Database::VAR_STRING,
'signed' => true,
'size' => Database::LENGTH_KEY,
'format' => '',
'filters' => [],
'required' => true,
'default' => null,
'array' => false,
],
],
'indexes' => [
[
'$id' => '_key_database_id',
'type' => Database::INDEX_KEY,
'attributes' => ['databaseId', 'collectionId'],
'lengths' => [],
'orders' => []
],
[
'$id' => '_key_blocked',
'type' => Database::INDEX_KEY,
'attributes' => ['blocked', 'databaseId', 'collectionId'],
'lengths' => [],
'orders' => []
]
],
],
];
+12
View File
@@ -1023,4 +1023,16 @@ return [
'description' => 'Target has an invalid provider type.',
'code' => 400,
],
/* Slow queries */
Exception::QUERY_BLOCKED => [
'name' => Exception::QUERY_BLOCKED,
'description' => 'Request blocked after exceeding the maximum number of timeouts',
'code' => 403,
],
Exception::QUERY_TIMEOUT => [
'name' => Exception::QUERY_TIMEOUT,
'description' => 'Request timed out, this query will be blocked if it keeps timing out, try adjusting the "queries" parameter or adding indexes for the queried attributes.',
'code' => 408,
],
];
+207 -7
View File
@@ -32,6 +32,7 @@ use Utopia\Database\Exception\NotFound as NotFoundException;
use Utopia\Database\Exception\Query as QueryException;
use Utopia\Database\Exception\Restricted as RestrictedException;
use Utopia\Database\Exception\Structure as StructureException;
use Utopia\Database\Exception\Timeout as TimeoutException;
use Utopia\Database\Exception\Truncate as TruncateException;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
@@ -449,6 +450,88 @@ App::init()
}
});
App::init()
->groups(['timeout'])
->inject('request')
->inject('dbForProject')
->action(function (Request $request, Database $dbForProject) {
$key = md5(json_encode([
$request->getMethod(),
$request->getURI(), // Contains databaseId & collectionId
$request->getParam('queries')
]));
/** @var Document $document */
$document = Authorization::skip(fn () => $dbForProject->getDocument(
'slowQueries',
$key
));
if ($document->getAttribute('blocked')) {
throw new Exception(Exception::QUERY_BLOCKED);
}
});
App::error()
->groups(['timeout'])
->inject('error')
->inject('request')
->inject('dbForProject')
->action(function (Throwable $error, Request $request, Database $dbForProject) {
if (!$error instanceof TimeoutException) {
return;
}
$route = Request::getRoute();
$collectionId = $route->getParamValue('collectionId');
$databaseId = $route->getParamValue('databaseId');
$queries = $request->getParam('queries', []);
$key = md5(json_encode([
$request->getMethod(),
$request->getURI(),
$queries
]));
/** @var Document $document */
$document = Authorization::skip(fn () => $dbForProject->getDocument(
'slowQueries',
$key
));
if ($document->isEmpty()) {
$document = Authorization::skip(fn () => $dbForProject->createDocument('slowQueries', new Document([
'$id' => $key,
'blocked' => false,
'count' => 1,
'queries' => $queries,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
'path' => $request->getURI()
])));
} else {
$document->setAttribute('count', $document->getAttribute('count') + 1);
$max = intval(App::getEnv('_APP_SLOW_QUERIES_MAX_HITS', 5));
if ($document->getAttribute('count') >= $max) {
$document->setAttribute('blocked', true);
}
$document = Authorization::skip(fn () => $dbForProject->updateDocument(
'slowQueries',
$document->getId(),
$document
));
}
// Set the correct exception to pass to the next shutdown hook
if ($document->getAttribute('blocked')) {
App::setResource('error', fn () => new Exception(Exception::QUERY_BLOCKED));
} else {
App::setResource('error', fn () => new Exception(Exception::QUERY_TIMEOUT));
}
});
App::post('/v1/databases')
->desc('Create database')
->groups(['api', 'database'])
@@ -924,8 +1007,7 @@ App::get('/v1/databases/:databaseId/collections')
->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true)
->inject('response')
->inject('dbForProject')
->inject('mode')
->action(function (string $databaseId, array $queries, string $search, Response $response, Database $dbForProject, string $mode) {
->action(function (string $databaseId, array $queries, string $search, Response $response, Database $dbForProject) {
$database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId));
@@ -1217,8 +1299,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId')
->inject('dbForProject')
->inject('queueForDatabase')
->inject('queueForEvents')
->inject('mode')
->action(function (string $databaseId, string $collectionId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, string $mode) {
->action(function (string $databaseId, string $collectionId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) {
$database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId));
@@ -3370,7 +3451,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
App::get('/v1/databases/:databaseId/collections/:collectionId/documents')
->alias('/v1/database/collections/:collectionId/documents', ['databaseId' => 'default'])
->desc('List documents')
->groups(['api', 'database'])
->groups(['api', 'database', 'timeout'])
->label('scope', 'documents.read')
->label('resourceType', RESOURCE_TYPE_DATABASES)
->label('sdk', new Method(
@@ -3389,11 +3470,12 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents')
->param('databaseId', '', new UID(), 'Database ID.')
->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).')
->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
->inject('request')
->inject('response')
->inject('dbForProject')
->inject('mode')
->inject('queueForUsage')
->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) {
->action(function (string $databaseId, string $collectionId, array $queries, Request $request, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) {
$database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId));
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
@@ -3440,9 +3522,17 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents')
$cursor->setValue($cursorDocument);
}
$timeout = App::getEnv('_APP_SLOW_QUERIES') === 'enabled'
? App::getEnv('_APP_SLOW_QUERIES_TIMEOUT')
: null;
if (App::isDevelopment() && intval($request->getHeader('x-appwrite-timeout')) > 0) {
$timeout = intval($request->getHeader('x-appwrite-timeout'));
$dbForProject->setTimeout($timeout);
}
$documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries);
$total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT);
$operations = 0;
// Add $collectionId and $databaseId for all documents
@@ -4441,3 +4531,113 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
'documents' => $usage[$metrics[0]]['data'],
]), Response::MODEL_USAGE_COLLECTION);
});
App::get('/v1/databases/:databaseId/slow-queries')
->desc('List slow queries')
->groups(['api', 'database'])
->label('docs', false)
->label('scope', 'documents.read')
->label('sdk', new Method(
namespace: 'databases',
name: 'listSlowQueries',
description: '/docs/references/databases/list-slow-queries.md',
auth: [AuthType::ADMIN],
responses: [
new SDKResponse(
code: Response::STATUS_CODE_OK,
model: Response::MODEL_SLOW_QUERY_LIST,
)
]
))
->param('databaseId', '', new UID(), 'Database ID.')
->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
->inject('response')
->inject('dbForProject')
->action(function ($databaseId, array $queries, Response $response, Database $dbForProject) {
$queries = Query::parseQueries($queries);
$queries[] = Query::equal('databaseId', [$databaseId]);
$cursor = Query::getByType($queries, [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]);
$cursor = reset($cursor);
if ($cursor) {
$documentId = $cursor->getValue();
$cursorDocument = $dbForProject->getDocument('slowQueries', $documentId);
if ($cursorDocument->isEmpty()) {
throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Slow query '{$documentId}' for the 'cursor' value not found.");
}
$cursor->setValue($cursorDocument);
}
$response->dynamic(new Document([
'documents' => $dbForProject->find('slowQueries', $queries),
'total' => $dbForProject->count('slowQueries', Query::groupByType($queries)['filters'], APP_LIMIT_COUNT),
]), Response::MODEL_DOCUMENT_LIST);
});
App::get('/v1/databases/:databaseId/slow-queries/:slowQueryId')
->desc('Get slow query')
->groups(['api', 'database'])
->label('docs', false)
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk', new Method(
namespace: 'databases',
name: 'getSlowQuery',
description: '/docs/references/databases/get-slow-query.md',
auth: [AuthType::ADMIN],
responses: [
new SDKResponse(
code: Response::STATUS_CODE_OK,
model: Response::MODEL_SLOW_QUERY,
)
]
))
->param('databaseId', '', new UID(), 'Database ID.')
->param('slowQueryId', '', new UID(), 'Document ID.')
->inject('response')
->inject('dbForProject')
->action(function (string $databaseId, string $slowQueryId, Response $response, Database $dbForProject) {
$document = $dbForProject->getDocument('slowQueries', $slowQueryId);
if ($document->isEmpty() || $document->getAttribute('databaseId') !== $databaseId) {
throw new Exception(Exception::DOCUMENT_NOT_FOUND);
}
$response->dynamic($document, Response::MODEL_SLOW_QUERY);
});
App::delete('/v1/databases/:databaseId/slow-queries/:slowQueryId')
->desc('Delete slow query')
->desc('List Documents')
->groups(['api', 'database'])
->label('scope', 'documents.write')
->label('usage.metric', 'documents.{scope}.requests.delete')
->label('docs', false)
->label('sdk', new Method(
namespace: 'databases',
name: 'deleteSlowQuery',
description: '/docs/references/databases/delete-slow-query.md',
auth: [AuthType::ADMIN],
responses: [
new SDKResponse(
code: Response::STATUS_CODE_NOCONTENT,
model: Response::MODEL_NONE,
)
]
))
->param('databaseId', '', new UID(), 'Database ID.')
->param('slowQueryId', '', new UID(), 'Document ID.')
->inject('response')
->inject('dbForProject')
->action(function (string $databaseId, string $slowQueryId, Response $response, Database $dbForProject) {
$document = $dbForProject->getDocument('slowQueries', $slowQueryId);
if ($document->isEmpty() || $document->getAttribute('databaseId') !== $databaseId) {
throw new Exception(Exception::DOCUMENT_NOT_FOUND);
}
$dbForProject->deleteDocument('slowQueries', $slowQueryId);
$response->noContent();
});
+3
View File
@@ -163,6 +163,9 @@ $image = $this->getParam('image', '');
- _APP_MIGRATIONS_FIREBASE_CLIENT_ID
- _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET
- _APP_ASSISTANT_OPENAI_API_KEY
- _APP_SLOW_QUERIES_MAX_HITS
- _APP_SLOW_QUERIES_TIMEOUT
- _APP_SLOW_QUERIES
appwrite-console:
<<: *x-logging
+1 -1
View File
@@ -30,7 +30,7 @@
}
},
"require": {
"php": ">=8.0.0",
"php": ">=8.3",
"ext-curl": "*",
"ext-imagick": "*",
"ext-mbstring": "*",
Generated
+8 -8
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "e8d26e7e836db255ba42cf55c3798c97",
"content-hash": "1112973adf57691665674ee76d7dc3ec",
"packages": [
{
"name": "adhocore/jwt",
@@ -2453,16 +2453,16 @@
},
{
"name": "symfony/http-client",
"version": "v7.2.2",
"version": "v7.2.3",
"source": {
"type": "git",
"url": "https://github.com/symfony/http-client.git",
"reference": "339ba21476eb184290361542f732ad12c97591ec"
"reference": "7ce6078c79a4a7afff931c413d2959d3bffbfb8d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/http-client/zipball/339ba21476eb184290361542f732ad12c97591ec",
"reference": "339ba21476eb184290361542f732ad12c97591ec",
"url": "https://api.github.com/repos/symfony/http-client/zipball/7ce6078c79a4a7afff931c413d2959d3bffbfb8d",
"reference": "7ce6078c79a4a7afff931c413d2959d3bffbfb8d",
"shasum": ""
},
"require": {
@@ -2528,7 +2528,7 @@
"http"
],
"support": {
"source": "https://github.com/symfony/http-client/tree/v7.2.2"
"source": "https://github.com/symfony/http-client/tree/v7.2.3"
},
"funding": [
{
@@ -2544,7 +2544,7 @@
"type": "tidelift"
}
],
"time": "2024-12-30T18:35:15+00:00"
"time": "2025-01-28T15:51:35+00:00"
},
{
"name": "symfony/http-client-contracts",
@@ -8507,7 +8507,7 @@
"prefer-stable": false,
"prefer-lowest": false,
"platform": {
"php": ">=8.0.0",
"php": ">=8.3",
"ext-curl": "*",
"ext-imagick": "*",
"ext-mbstring": "*",
+3
View File
@@ -196,6 +196,9 @@ services:
- _APP_DATABASE_SHARED_TABLES_V1
- _APP_DATABASE_SHARED_NAMESPACE
- _APP_FUNCTIONS_CREATION_ABUSE_LIMIT
- _APP_SLOW_QUERIES_MAX_HITS
- _APP_SLOW_QUERIES_TIMEOUT
- _APP_SLOW_QUERIES
appwrite-console:
<<: *x-logging
@@ -0,0 +1 @@
Delete a slow query by its unique ID.
@@ -0,0 +1 @@
Get a slow query by its unique ID. This endpoint response returns a JSON object with the document data.
@@ -0,0 +1 @@
Get a list of all the slow queries documents in a given database. You can use the query params to filter your results.
+5
View File
@@ -261,6 +261,11 @@ class Exception extends \Exception
public const MIGRATION_IN_PROGRESS = 'migration_in_progress';
public const MIGRATION_PROVIDER_ERROR = 'migration_provider_error';
/** Timeout */
public const QUERY_BLOCKED = 'query_blocked';
public const QUERY_TIMEOUT = 'query_timeout';
/** Realtime */
public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid';
public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages';
+8 -2
View File
@@ -84,6 +84,7 @@ use Appwrite\Utopia\Response\Model\ProviderRepository;
use Appwrite\Utopia\Response\Model\Rule;
use Appwrite\Utopia\Response\Model\Runtime;
use Appwrite\Utopia\Response\Model\Session;
use Appwrite\Utopia\Response\Model\SlowQuery;
use Appwrite\Utopia\Response\Model\Specification;
use Appwrite\Utopia\Response\Model\Subscriber;
use Appwrite\Utopia\Response\Model\Target;
@@ -111,7 +112,6 @@ use Appwrite\Utopia\Response\Model\Webhook;
use Exception;
use JsonException;
use Swoole\Http\Response as SwooleHTTPResponse;
// Keep last
use Utopia\Database\Document;
use Utopia\Swoole\Response as SwooleResponse;
@@ -152,6 +152,10 @@ class Response extends SwooleResponse
public const MODEL_DOCUMENT = 'document';
public const MODEL_DOCUMENT_LIST = 'documentList';
public const MODEL_SLOW_QUERY = 'slowQuery';
public const MODEL_SLOW_QUERY_LIST = 'slowQueryList';
// Database Attributes
public const MODEL_ATTRIBUTE = 'attribute';
public const MODEL_ATTRIBUTE_LIST = 'attributeList';
@@ -384,6 +388,7 @@ class Response extends SwooleResponse
->setModel(new BaseList('Migrations Firebase Projects List', self::MODEL_MIGRATION_FIREBASE_PROJECT_LIST, 'projects', self::MODEL_MIGRATION_FIREBASE_PROJECT))
->setModel(new BaseList('Specifications List', self::MODEL_SPECIFICATION_LIST, 'specifications', self::MODEL_SPECIFICATION))
->setModel(new BaseList('VCS Content List', self::MODEL_VCS_CONTENT_LIST, 'contents', self::MODEL_VCS_CONTENT))
->setModel(new BaseList('Slow Query List', self::MODEL_SLOW_QUERY_LIST, 'slowQueries', self::MODEL_SLOW_QUERY))
// Entities
->setModel(new Database())
->setModel(new Collection())
@@ -482,6 +487,7 @@ class Response extends SwooleResponse
->setModel(new Migration())
->setModel(new MigrationReport())
->setModel(new MigrationFirebaseProject())
->setModel(new SlowQuery())
// Tests (keep last)
->setModel(new Mock());
@@ -773,7 +779,7 @@ class Response extends SwooleResponse
/**
* Function to add a response filter, the order of filters are first in - first out.
*
* @param $filter the response filter to set
* @param $filter Filter The response filter to set
*
* @return void
*/
@@ -0,0 +1,76 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class SlowQuery extends Model
{
public function __construct()
{
$this
->addRule('$id', [
'type' => self::TYPE_STRING,
'description' => 'Slow Query ID.',
'default' => '',
'example' => '5e5ea5c16897e',
])
->addRule('$createdAt', [
'type' => self::TYPE_DATETIME,
'description' => 'Slow query creation date in ISO 8601 format.',
'default' => '',
'example' => self::TYPE_DATETIME_EXAMPLE,
])
->addRule('$updatedAt', [
'type' => self::TYPE_DATETIME,
'description' => 'Slow query update date in ISO 8601 format.',
'default' => '',
'example' => self::TYPE_DATETIME_EXAMPLE,
])
->addRule('blocked', [
'type' => self::TYPE_BOOLEAN,
'description' => 'Is this slow query blocked from execution?',
'default' => false,
'example' => true,
])
->addRule('count', [
'type' => self::TYPE_INTEGER,
'description' => 'The number of times this slow query has executed.',
'default' => 0,
'example' => 5,
])
->addRule('queries', [
'type' => self::TYPE_STRING,
'description' => 'The slow query itself.',
'default' => '',
'example' => '["limit(10000)"]',
])
->addRule('path', [
'type' => self::TYPE_STRING,
'description' => 'The path of the endpoint that executed this slow query.',
'default' => '',
'example' => '/v1/databases/123/collections',
]);
}
/**
* Get Name
*
* @return string
*/
public function getName(): string
{
return 'Slow Queries';
}
/**
* Get Type
*
* @return string
*/
public function getType(): string
{
return Response::MODEL_SLOW_QUERY;
}
}
+16 -11
View File
@@ -12,7 +12,6 @@ use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\Database\Validator\Datetime as DatetimeValidator;
use Utopia\Validator\JSON;
trait DatabasesBase
{
@@ -1334,7 +1333,7 @@ trait DatabasesBase
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), []);
]));
$this->assertIsArray($movies['body']['indexes']);
$this->assertCount(8, $movies['body']['indexes']);
@@ -4787,16 +4786,22 @@ trait DatabasesBase
]);
}
$response = $this->client->call(Client::METHOD_GET, '/databases/' . $data['databaseId'] . '/collections/' . $data['$id'] . '/documents', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-timeout' => 1,
], $this->getHeaders()), [
'queries' => [
$docs = [];
for ($i = 0; $i <= 5; $i++) { // _APP_SLOW_QUERIES_MAX_HITS = 5
$docs[] = $this->client->call(Client::METHOD_GET, '/databases/' . $data['databaseId'] . '/collections/' . $data['$id'] . '/documents', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-timeout' => 1,
], $this->getHeaders()), [
Query::notEqual('longtext', 'appwrite')->toString(),
],
]);
]);
}
$this->assertEquals(408, $response['headers']['status-code']);
$this->assertEquals(408, $docs[0]['headers']['status-code']); // create
$this->assertEquals(408, $docs[1]['headers']['status-code']); // update
$this->assertEquals(408, $docs[2]['headers']['status-code']); // update
$this->assertEquals(408, $docs[3]['headers']['status-code']); // update
$this->assertEquals(403, $docs[4]['headers']['status-code']); // update
$this->assertEquals(403, $docs[5]['headers']['status-code']); // blocked
}
}
@@ -94,7 +94,19 @@ class DatabasesConsoleClientTest extends Scope
$this->assertEquals(201, $tvShows['headers']['status-code']);
$this->assertEquals($tvShows['body']['name'], 'TvShows');
return ['moviesId' => $movies['body']['$id'], 'databaseId' => $databaseId, 'tvShowsId' => $tvShows['body']['$id']];
$this->client->call(Client::METHOD_PUT, '/databases/' . $databaseId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Database Updated',
'enabled' => true,
]);
return [
'moviesId' => $movies['body']['$id'],
'databaseId' => $databaseId,
'tvShowsId' => $tvShows['body']['$id']
];
}
/**
@@ -331,4 +343,130 @@ class DatabasesConsoleClientTest extends Scope
$this->assertLessThanOrEqual(1, count($logs['body']['logs']));
$this->assertIsNumeric($logs['body']['total']);
}
public function testTimeouts(): array
{
$database = $this->client->call(Client::METHOD_POST, '/databases', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'databaseId' => ID::unique(),
'name' => 'slowQueriesDatabase',
]);
$databaseId = $database['body']['$id'];
$collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'collectionId' => ID::unique(),
'name' => 'Slow Queries',
'documentSecurity' => true,
'permissions' => [
Permission::create(Role::user($this->getUser()['$id'])),
],
]);
$collectionId = $collection['body']['$id'];
$this->assertEquals(201, $collection['headers']['status-code']);
$longtext = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'key' => 'longtext',
'size' => 100000000,
'required' => false,
'default' => null,
]);
$this->assertEquals($longtext['headers']['status-code'], 202);
for ($i = 0; $i <= 1; $i++) {
$this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'documentId' => ID::unique(),
'data' => [
'longtext' => file_get_contents(__DIR__ . '/../../../resources/longtext.txt'),
],
'permissions' => [
Permission::read(Role::user($this->getUser()['$id'])),
Permission::update(Role::user($this->getUser()['$id'])),
Permission::delete(Role::user($this->getUser()['$id'])),
]
]);
}
$docs = [];
for ($i = 0; $i <= 5; $i++) { // _APP_SLOW_QUERIES_MAX_HITS = 5
$docs[] = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-timeout' => 1,
], $this->getHeaders()), [
'queries' => [
Query::notEqual('longtext', 'appwrite')->toString(),
],
]);
}
$this->assertEquals(408, $docs[0]['headers']['status-code']); // insert
$this->assertEquals(408, $docs[1]['headers']['status-code']); // update
$this->assertEquals(408, $docs[2]['headers']['status-code']); // update
$this->assertEquals(408, $docs[3]['headers']['status-code']); // update
$this->assertEquals(403, $docs[4]['headers']['status-code']); // update
$this->assertEquals(403, $docs[5]['headers']['status-code']); // blocked
return [
'databaseId' => $databaseId,
'collectionId' => $collectionId
];
}
/**
* @depends testTimeouts
*/
public function testConsoleTimeouts($data): void
{
$documents = $this->client->call(Client::METHOD_GET, '/databases/' . $data['databaseId'] . '/slow-queries', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => [
Query::equal('collectionId', $data['$id'])->toString(),
Query::equal('blocked', [true])->toString(),
Query::orderAsc('$updatedAt')->toString(),
]
]);
$this->assertEquals(200, $documents['headers']['status-code']);
$this->assertEquals(1, $documents['body']['total']);
$this->assertEquals(true, $documents['body']['documents'][0]['blocked']);
$this->assertEquals(5, $documents['body']['documents'][0]['count']);
$doc = $this->client->call(Client::METHOD_GET, '/databases/' . $data['databaseId'] . '/slow-queries/' . $documents['body']['documents'][0]['$id'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $doc['headers']['status-code']);
$this->assertEquals($doc['body']['queries'][0], 'notEqual("longtext", "appwrite")');
$response = $this->client->call(Client::METHOD_DELETE, '/databases/' . $data['databaseId'] . '/slow-queries/' . $documents['body']['documents'][0]['$id'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(204, $response['headers']['status-code']);
$doc = $this->client->call(Client::METHOD_GET, '/databases/' . $data['databaseId'] . '/slow-queries/' . $documents['body']['documents'][0]['$id'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(404, $doc['headers']['status-code']);
}
}
File diff suppressed because it is too large Load Diff