From f07f246f8db2dd4ae3525ed31a1f09de8b879f53 Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Fri, 22 Sep 2023 22:53:41 +0530 Subject: [PATCH 01/67] fix conflicts --- app/config/errors.php | 9 +++++++++ app/console | 2 +- app/controllers/api/account.php | 8 ++++++++ src/Appwrite/Extend/Exception.php | 2 ++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/app/config/errors.php b/app/config/errors.php index 575a4588ba..560741b9d3 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -239,6 +239,15 @@ return [ 'name' => Exception::USER_OAUTH2_PROVIDER_ERROR, 'description' => 'OAuth2 provider returned some error.', 'code' => 424, + Exception::USER_EMAIL_ALREADY_VERIFIED => [ + 'name' => Exception::USER_EMAIL_ALREADY_VERIFIED, + 'description' => 'User email is already verified', + 'code' => 400, + ], + Exception::USER_PHONE_ALREADY_VERIFIED => [ + 'name' => Exception::USER_PHONE_ALREADY_VERIFIED, + 'description' => 'User phone is already verified', + 'code' => 400 ], /** Teams */ diff --git a/app/console b/app/console index 9b4bcb8140..88b6d59051 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 9b4bcb8140484669421685b4ba89fa1c4d331360 +Subproject commit 88b6d59051992ed86183ee83d77bf678d1cb73bf diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 87dcd95f03..575a343391 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2669,6 +2669,10 @@ App::post('/v1/account/verification') if (empty(App::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP Disabled'); } + + if($user->getAttribute('emailVerification') == true){ + throw new Exception(Exception::USER_EMAIL_ALREADY_VERIFIED); + } $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); @@ -2894,6 +2898,10 @@ App::post('/v1/account/verification/phone') if (empty($user->getAttribute('phone'))) { throw new Exception(Exception::USER_PHONE_NOT_FOUND); } + + if($user->getAttribute('phoneVerification') == true){ + throw new Exception(Exception::USER_PHONE_ALREADY_VERIFIED); + } $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 77dc03e310..5f779ff841 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -84,6 +84,8 @@ class Exception extends \Exception public const USER_OAUTH2_BAD_REQUEST = 'user_oauth2_bad_request'; public const USER_OAUTH2_UNAUTHORIZED = 'user_oauth2_unauthorized'; public const USER_OAUTH2_PROVIDER_ERROR = 'user_oauth2_provider_error'; + public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_alread_verified'; + public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified'; /** Teams */ public const TEAM_NOT_FOUND = 'team_not_found'; From 6b4799912069ff6fed0442120a2147f4b01383c9 Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Fri, 22 Sep 2023 22:56:07 +0530 Subject: [PATCH 02/67] fix conflicts --- app/config/errors.php | 4 ++-- app/controllers/api/account.php | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index 560741b9d3..192b03e2b0 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -242,12 +242,12 @@ return [ Exception::USER_EMAIL_ALREADY_VERIFIED => [ 'name' => Exception::USER_EMAIL_ALREADY_VERIFIED, 'description' => 'User email is already verified', - 'code' => 400, + 'code' => 409, ], Exception::USER_PHONE_ALREADY_VERIFIED => [ 'name' => Exception::USER_PHONE_ALREADY_VERIFIED, 'description' => 'User phone is already verified', - 'code' => 400 + 'code' => 409 ], /** Teams */ diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 575a343391..9b21ef7d0e 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2669,8 +2669,8 @@ App::post('/v1/account/verification') if (empty(App::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP Disabled'); } - - if($user->getAttribute('emailVerification') == true){ + + if ($user->getAttribute('emailVerification')) { throw new Exception(Exception::USER_EMAIL_ALREADY_VERIFIED); } @@ -2898,8 +2898,8 @@ App::post('/v1/account/verification/phone') if (empty($user->getAttribute('phone'))) { throw new Exception(Exception::USER_PHONE_NOT_FOUND); } - - if($user->getAttribute('phoneVerification') == true){ + + if ($user->getAttribute('phoneVerification')) { throw new Exception(Exception::USER_PHONE_ALREADY_VERIFIED); } From 56a3b3df9927b8e5fa9823023fe22c76c7c6dee1 Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Sun, 7 May 2023 03:16:34 +0000 Subject: [PATCH 03/67] Add test cases --- tests/e2e/Services/Account/AccountBase.php | 53 +++++++++++++++++++ .../Account/AccountCustomClientTest.php | 24 +++++++++ 2 files changed, 77 insertions(+) diff --git a/tests/e2e/Services/Account/AccountBase.php b/tests/e2e/Services/Account/AccountBase.php index e64cf2c4ca..57d6d8de83 100644 --- a/tests/e2e/Services/Account/AccountBase.php +++ b/tests/e2e/Services/Account/AccountBase.php @@ -891,6 +891,7 @@ trait AccountBase return $data; } + /** * @depends testCreateAccountVerification */ @@ -945,6 +946,58 @@ trait AccountBase return $data; } + /** + * @depends testUpdateAccountVerification + */ + public function testCreateAccountVerificationForVerifiedEmail($data): array + { + $email = $data['email'] ?? ''; + $name = $data['name'] ?? ''; + $session = $data['session'] ?? ''; + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + + ]), [ + 'url' => 'http://localhost/verification', + ]); + + $this->assertEquals(409, $response['headers']['status-code']); + + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'url' => 'localhost/verification', + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'url' => 'http://remotehost/verification', + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + return $data; + } + /** * @depends testUpdateAccountVerification */ diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 1441ab7f98..ee8978d73a 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1068,4 +1068,28 @@ class AccountCustomClientTest extends Scope return $data; } + + /** + * @depends testPhoneVerification + */ + #[Retry(count: 1)] + public function testPhoneVerificationForVerifiedPhone(array $data): array + { + $session = $data['session'] ?? ''; + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_POST, '/account/verification/phone', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + + ])); + + $this->assertEquals(409, $response['headers']['status-code']); + + return \array_merge($data); + } } From 5771ad35ed4bf9c9160b2df5e5a75b9d56ae13c7 Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Wed, 21 Jun 2023 15:55:39 +0000 Subject: [PATCH 04/67] fix test cases --- tests/e2e/Services/Account/AccountBase.php | 25 ------------------- .../Account/AccountCustomClientTest.php | 2 +- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/tests/e2e/Services/Account/AccountBase.php b/tests/e2e/Services/Account/AccountBase.php index 57d6d8de83..e2f56d54a4 100644 --- a/tests/e2e/Services/Account/AccountBase.php +++ b/tests/e2e/Services/Account/AccountBase.php @@ -970,31 +970,6 @@ trait AccountBase $this->assertEquals(409, $response['headers']['status-code']); - /** - * Test for FAILURE - */ - $response = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ - 'origin' => 'http://localhost', - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ]), [ - 'url' => 'localhost/verification', - ]); - - $this->assertEquals(400, $response['headers']['status-code']); - - $response = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ - 'origin' => 'http://localhost', - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ]), [ - 'url' => 'http://remotehost/verification', - ]); - - $this->assertEquals(400, $response['headers']['status-code']); - return $data; } diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index ee8978d73a..1719d4e049 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1090,6 +1090,6 @@ class AccountCustomClientTest extends Scope $this->assertEquals(409, $response['headers']['status-code']); - return \array_merge($data); + return $data; } } From b420c570bc2e76753eaaf540d345b86eafd3c7ef Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Sat, 14 Oct 2023 11:51:27 +0530 Subject: [PATCH 05/67] Reverted to last commit before merging main --- app/console | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/console b/app/console index 88b6d59051..9b4bcb8140 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 88b6d59051992ed86183ee83d77bf678d1cb73bf +Subproject commit 9b4bcb8140484669421685b4ba89fa1c4d331360 From 1489f77499727711195ba0404122efbaafa61b90 Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Sat, 14 Oct 2023 12:45:29 +0530 Subject: [PATCH 06/67] fix changes suggested by steven and also fixed some bugs came after merging with 1.4.x --- app/config/errors.php | 1 + tests/e2e/Services/Account/AccountBase.php | 2 - .../Account/AccountCustomClientTest.php | 1 - .../Webhooks/WebhooksCustomClientTest.php | 228 +++++++++--------- 4 files changed, 115 insertions(+), 117 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index 192b03e2b0..e287b846b1 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -239,6 +239,7 @@ return [ 'name' => Exception::USER_OAUTH2_PROVIDER_ERROR, 'description' => 'OAuth2 provider returned some error.', 'code' => 424, + ], Exception::USER_EMAIL_ALREADY_VERIFIED => [ 'name' => Exception::USER_EMAIL_ALREADY_VERIFIED, 'description' => 'User email is already verified', diff --git a/tests/e2e/Services/Account/AccountBase.php b/tests/e2e/Services/Account/AccountBase.php index e2f56d54a4..953b89ced7 100644 --- a/tests/e2e/Services/Account/AccountBase.php +++ b/tests/e2e/Services/Account/AccountBase.php @@ -891,7 +891,6 @@ trait AccountBase return $data; } - /** * @depends testCreateAccountVerification */ @@ -963,7 +962,6 @@ trait AccountBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ]), [ 'url' => 'http://localhost/verification', ]); diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 1719d4e049..04322292d8 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1085,7 +1085,6 @@ class AccountCustomClientTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ])); $this->assertEquals(409, $response['headers']['status-code']); diff --git a/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php b/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php index 67c2dec36b..21c270d199 100644 --- a/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php +++ b/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php @@ -636,6 +636,120 @@ class WebhooksCustomClientTest extends Scope return $data; } + /** + * @depends testUpdateAccountPrefs + */ + public function testCreateAccountVerification($data): array + { + $id = $data['id'] ?? ''; + $email = $data['email'] ?? ''; + $session = $data['session'] ?? ''; + + $verification = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'url' => 'http://localhost/verification', + ]); + + $verificationId = $verification['body']['$id']; + + $this->assertEquals(201, $verification['headers']['status-code']); + $this->assertIsArray($verification['body']); + + $webhook = $this->getLastRequest(); + $signatureKey = $this->getProject()['signatureKey']; + $payload = json_encode($webhook['data']); + $url = $webhook['url']; + $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); + + $this->assertEquals($webhook['method'], 'POST'); + $this->assertEquals($webhook['headers']['Content-Type'], 'application/json'); + $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); + $this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('users.*.verification.*.create', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.*.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.*.create", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); + $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); + $this->assertNotEmpty($webhook['data']['$id']); + $this->assertNotEmpty($webhook['data']['userId']); + $this->assertNotEmpty($webhook['data']['secret']); + $this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire'])); + + $data['secret'] = $webhook['data']['secret']; + + return $data; + } + + /** + * @depends testCreateAccountVerification + */ + public function testUpdateAccountVerification($data): array + { + $id = $data['id'] ?? ''; + $email = $data['email'] ?? ''; + $session = $data['session'] ?? ''; + $secret = $data['secret'] ?? ''; + + $verification = $this->client->call(Client::METHOD_PUT, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'userId' => $id, + 'secret' => $secret, + ]); + + $verificationId = $verification['body']['$id']; + + $this->assertEquals(200, $verification['headers']['status-code']); + $this->assertIsArray($verification['body']); + + $webhook = $this->getLastRequest(); + $signatureKey = $this->getProject()['signatureKey']; + $payload = json_encode($webhook['data']); + $url = $webhook['url']; + $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); + + $this->assertEquals($webhook['method'], 'POST'); + $this->assertEquals($webhook['headers']['Content-Type'], 'application/json'); + $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); + $this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('users.*.verification.*.update', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.*.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.*.update", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); + $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); + $this->assertNotEmpty($webhook['data']['$id']); + $this->assertNotEmpty($webhook['data']['userId']); + $this->assertNotEmpty($webhook['data']['secret']); + $this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire'])); + + $data['secret'] = $webhook['data']['secret']; + + return $data; + } + /** * @depends testUpdateAccountPrefs */ @@ -751,120 +865,6 @@ class WebhooksCustomClientTest extends Scope return $data; } - /** - * @depends testUpdateAccountPrefs - */ - public function testCreateAccountVerification($data): array - { - $id = $data['id'] ?? ''; - $email = $data['email'] ?? ''; - $session = $data['session'] ?? ''; - - $verification = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ - 'origin' => 'http://localhost', - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ]), [ - 'url' => 'http://localhost/verification', - ]); - - $verificationId = $verification['body']['$id']; - - $this->assertEquals(201, $verification['headers']['status-code']); - $this->assertIsArray($verification['body']); - - $webhook = $this->getLastRequest(); - $signatureKey = $this->getProject()['signatureKey']; - $payload = json_encode($webhook['data']); - $url = $webhook['url']; - $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); - - $this->assertEquals($webhook['method'], 'POST'); - $this->assertEquals($webhook['headers']['Content-Type'], 'application/json'); - $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); - $this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('users.*.verification.*.create', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.*.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.*.create", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); - $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); - $this->assertNotEmpty($webhook['data']['$id']); - $this->assertNotEmpty($webhook['data']['userId']); - $this->assertNotEmpty($webhook['data']['secret']); - $this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire'])); - - $data['secret'] = $webhook['data']['secret']; - - return $data; - } - - /** - * @depends testCreateAccountVerification - */ - public function testUpdateAccountVerification($data): array - { - $id = $data['id'] ?? ''; - $email = $data['email'] ?? ''; - $session = $data['session'] ?? ''; - $secret = $data['secret'] ?? ''; - - $verification = $this->client->call(Client::METHOD_PUT, '/account/verification', array_merge([ - 'origin' => 'http://localhost', - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ]), [ - 'userId' => $id, - 'secret' => $secret, - ]); - - $verificationId = $verification['body']['$id']; - - $this->assertEquals(200, $verification['headers']['status-code']); - $this->assertIsArray($verification['body']); - - $webhook = $this->getLastRequest(); - $signatureKey = $this->getProject()['signatureKey']; - $payload = json_encode($webhook['data']); - $url = $webhook['url']; - $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); - - $this->assertEquals($webhook['method'], 'POST'); - $this->assertEquals($webhook['headers']['Content-Type'], 'application/json'); - $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); - $this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('users.*.verification.*.update', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.*.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.*.update", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); - $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); - $this->assertNotEmpty($webhook['data']['$id']); - $this->assertNotEmpty($webhook['data']['userId']); - $this->assertNotEmpty($webhook['data']['secret']); - $this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire'])); - - $data['secret'] = $webhook['data']['secret']; - - return $data; - } - /** * @depends testCreateTeamMembership */ From 68486d4795ff3a620fb5bdbb8460b25b7677f3c7 Mon Sep 17 00:00:00 2001 From: KRISH SONI <67964054+krishvsoni@users.noreply.github.com> Date: Sun, 29 Oct 2023 23:17:27 +0530 Subject: [PATCH 07/67] Update README-CN.md --- README-CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-CN.md b/README-CN.md index 1df799f7ef..18915b283c 100644 --- a/README-CN.md +++ b/README-CN.md @@ -55,7 +55,7 @@ Appwrite 可以提供给开发者用户验证,外部授权,用户数据读 ## 安装 -Appwrite 的容器化服务器只需要一行指令就可以运行。您可以使用 docker-compose 在本地主机上运行 Appwrite,也可以在任何其他容器化工具(如 Kubernetes、Docker Swarm 或 Rancher)上运行 Appwrite。 +Appwrite 的容器化服务器只需要一行指令就可以运行。您可以使用 docker-compose 在本地主机上运行 Appwrite,也可以在任何其他容器化工具(如 [Kubernetes](https://kubernetes.io/docs/home/)、[Docker Swarm](https://docs.docker.com/engine/swarm/) 或 [Rancher](https://rancher.com/docs/))上运行 Appwrite。 启动 Appwrite 服务器的最简单方法是运行我们的 docker-compose 文件。在运行安装命令之前,请确保您的机器上安装了 [Docker](https://dockerdocs.cn/get-docker/index.html): From 0ecdeed3298e50590438f0e8a96dd14c38096020 Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Sat, 4 Nov 2023 13:01:05 +0530 Subject: [PATCH 08/67] Checkout to latest tag of console submodule --- app/console | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/console b/app/console index 9b4bcb8140..9810ce8581 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 9b4bcb8140484669421685b4ba89fa1c4d331360 +Subproject commit 9810ce85812ca26c95b7d35196848c92e8ba813d From 0f3c43897621e892e1b1dceed6378cf385112803 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 8 Nov 2023 12:05:51 +0200 Subject: [PATCH 09/67] added inf metric --- .../Platform/Tasks/DeleteOrphanedProjects.php | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2824f4e286..aeaab4b248 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -13,6 +13,8 @@ use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Pools\Group; use Utopia\Registry\Registry; +use Utopia\Validator\Boolean; +use Utopia\Validator\Hostname; class DeleteOrphanedProjects extends Action { @@ -26,25 +28,23 @@ class DeleteOrphanedProjects extends Action $this ->desc('Get stats for projects') + ->param('commit', false, new boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') ->inject('dbForConsole') ->inject('register') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($pools, $cache, $dbForConsole, $register); + ->callback(function (bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($commit, $pools, $cache, $dbForConsole, $register); }); } - public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + public function action(bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void { Console::title('Delete orphaned projects V1'); Console::success(APP_NAME . ' Delete orphaned projects started'); - /** @var array $collections */ - $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; - /* Initialise new Utopia app */ $app = new App('UTC'); $console = $app->getResource('console'); @@ -54,7 +54,7 @@ class DeleteOrphanedProjects extends Action $totalProjects = $dbForConsole->count('projects'); Console::success("Found a total of: {$totalProjects} projects"); - $orphans = 0; + $orphans = 1; $count = 0; $limit = 30; $sum = 30; @@ -80,17 +80,22 @@ class DeleteOrphanedProjects extends Action $dbForProject->setDefaultDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); $collectionsCreated = $dbForProject->count(Database::METADATA); - $message = ' (' . $collectionsCreated . ') collections where found on project (' . $project->getId() . '))'; - if ($collectionsCreated < (count($collectionsConfig) + 2)) { - Console::error($message); + if ($collectionsCreated === 0) { + if ($commit === true) { + Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); + $this->deleteProject($dbForConsole, $project->getId()); + } else { + Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); + } $orphans++; - } else { - Console::log($message); } } catch (\Throwable $th) { - //$dbForConsole->deleteDocument('projects', $project->getId()); - //Console::success('Deleting project (' . $project->getId() . ')'); - Console::error(' (0) collections where found for project (' . $project->getId() . ')'); + if ($commit === true) { + Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); + $this->deleteProject($dbForConsole, $project->getId()); + } else { + Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); + } $orphans++; } finally { $pools @@ -110,6 +115,15 @@ class DeleteOrphanedProjects extends Action $count = $count + $sum; } - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans'); + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); + } + + private function deleteProject(Database $dbForConsole, $projectId): void + { + try { + $dbForConsole->deleteDocument('projects', $projectId); + } catch (\Throwable $th) { + Console::error('Error when trying to delete project (' . $projectId . ') ' . $th->getMessage()); + } } } From a029287e5a1cb958e59dde4f06268c304f1ab270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 13 Nov 2023 14:07:11 +0100 Subject: [PATCH 10/67] Implement health tresholds --- app/controllers/api/health.php | 131 +++++++++++++++--- .../Health/HealthCustomServerTest.php | 93 +++++++++++++ 2 files changed, 204 insertions(+), 20 deletions(-) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index e0e26781cf..5d64e71526 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -14,6 +14,7 @@ use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; +use Utopia\Validator\Integer; use Utopia\Validator\Text; App::get('/v1/health') @@ -344,11 +345,20 @@ App::get('/v1/health/queue/webhooks') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/logs') @@ -362,11 +372,20 @@ App::get('/v1/health/queue/logs') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::AUDITS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/certificates') @@ -380,11 +399,20 @@ App::get('/v1/health/queue/certificates') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/builds') @@ -398,11 +426,20 @@ App::get('/v1/health/queue/builds') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::BUILDS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/databases') @@ -417,11 +454,20 @@ App::get('/v1/health/queue/databases') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) ->param('name', 'database_db_main', new Text(256), 'Queue name for which to check the queue size', true) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (string $name, Connection $queue, Response $response) { + ->action(function (string $name, int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client($name, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/deletes') @@ -435,11 +481,20 @@ App::get('/v1/health/queue/deletes') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::DELETE_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/mails') @@ -453,11 +508,20 @@ App::get('/v1/health/queue/mails') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::MAILS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/messaging') @@ -471,11 +535,20 @@ App::get('/v1/health/queue/messaging') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::MESSAGING_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/migrations') @@ -489,11 +562,20 @@ App::get('/v1/health/queue/migrations') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/functions') @@ -507,11 +589,20 @@ App::get('/v1/health/queue/functions') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/storage/local') diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index eafc961e8b..9a178ef1fa 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -138,6 +138,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/webhooks?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -155,6 +164,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/logs?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -172,6 +190,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/certificates?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -189,6 +216,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/functions?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -206,6 +242,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/builds?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -225,6 +270,18 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/databases', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'database_db_main', + 'treshold' => '0' + ]); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -242,6 +299,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/deletes?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -259,6 +325,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/mails?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -276,6 +351,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/messaging?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -293,6 +377,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/migrations?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } From 967854d34af107b4b3c97d6360bf367150f61dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 13 Nov 2023 14:39:33 +0100 Subject: [PATCH 11/67] Fix grammar --- app/controllers/api/health.php | 100 +++++++++--------- .../Health/HealthCustomServerTest.php | 20 ++-- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 5d64e71526..90e080d5fa 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -345,17 +345,17 @@ App::get('/v1/health/queue/webhooks') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -372,17 +372,17 @@ App::get('/v1/health/queue/logs') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::AUDITS_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -399,17 +399,17 @@ App::get('/v1/health/queue/certificates') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -426,17 +426,17 @@ App::get('/v1/health/queue/builds') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::BUILDS_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -454,17 +454,17 @@ App::get('/v1/health/queue/databases') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) ->param('name', 'database_db_main', new Text(256), 'Queue name for which to check the queue size', true) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (string $name, int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (string $name, int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client($name, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -481,17 +481,17 @@ App::get('/v1/health/queue/deletes') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::DELETE_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -508,17 +508,17 @@ App::get('/v1/health/queue/mails') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::MAILS_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -535,17 +535,17 @@ App::get('/v1/health/queue/messaging') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::MESSAGING_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -562,17 +562,17 @@ App::get('/v1/health/queue/migrations') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -589,17 +589,17 @@ App::get('/v1/health/queue/functions') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 9a178ef1fa..8fa9faadd2 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -141,7 +141,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/webhooks?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/webhooks?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -167,7 +167,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/logs?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/logs?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -193,7 +193,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/certificates?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/certificates?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -219,7 +219,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/functions?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/functions?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -245,7 +245,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/builds?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/builds?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -278,7 +278,7 @@ class HealthCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'name' => 'database_db_main', - 'treshold' => '0' + 'threshold' => '0' ]); $this->assertEquals(500, $response['headers']['status-code']); @@ -302,7 +302,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/deletes?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/deletes?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -328,7 +328,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/mails?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/mails?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -354,7 +354,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/messaging?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/messaging?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -380,7 +380,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/migrations?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/migrations?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); From b9007a59564f38b54cf90ba0d88aa62060d62fff Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 15:41:10 +0200 Subject: [PATCH 12/67] wrapping create stats query with Authorization::skip --- src/Appwrite/Usage/Calculators/TimeSeries.php | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index e0a12b443f..9ead48e93f 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -8,6 +8,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use InfluxDB\Database as InfluxDatabase; use DateTime; +use Utopia\Database\Validator\Authorization; use Utopia\Registry\Registry; class TimeSeries extends Calculator @@ -422,14 +423,16 @@ class TimeSeries extends Calculator */ private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void { + $id = \md5("{$time}_{$period}_{$metric}"); $project = $this->database->getDocument('projects', $projectId); $database = call_user_func($this->getProjectDB, $project); - try { - $document = $database->getDocument('stats', $id); - if ($document->isEmpty()) { - $database->createDocument('stats', new Document([ + Authorization::skip(function () use ($database, $id, $period, $time, $metric, $value, $type, $projectId) { + try { + $document = $database->getDocument('stats', $id); + if ($document->isEmpty()) { + $database->createDocument('stats', new Document([ '$id' => $id, 'period' => $period, 'time' => $time, @@ -437,21 +440,22 @@ class TimeSeries extends Calculator 'value' => $value, 'type' => $type, 'region' => $this->region, - ])); - } else { - $database->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); + ])); + } else { + $database->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $value) + ); + } + } catch (\Exception $e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); + } else { + throw $e; + } } - } catch (\Exception $e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); - } else { - throw $e; - } - } + }); $this->register->get('pools')->reclaim(); } From dd0bf66212450041ab7518a0f958c8fdbb9c8166 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 19:24:55 +0200 Subject: [PATCH 13/67] delete orphaned projects task --- .../Platform/Tasks/DeleteOrphanedProjects.php | 67 +++++++++++-------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index aeaab4b248..2af9c54a6e 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -2,11 +2,9 @@ namespace Appwrite\Platform\Tasks; -use PHPMailer\PHPMailer\PHPMailer; use Utopia\App; use Utopia\Config\Config; use Utopia\Database\Query; -use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -14,7 +12,6 @@ use Utopia\Database\Database; use Utopia\Pools\Group; use Utopia\Registry\Registry; use Utopia\Validator\Boolean; -use Utopia\Validator\Hostname; class DeleteOrphanedProjects extends Action { @@ -27,7 +24,7 @@ class DeleteOrphanedProjects extends Action { $this - ->desc('Get stats for projects') + ->desc('Delete orphaned projects') ->param('commit', false, new boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') @@ -45,6 +42,9 @@ class DeleteOrphanedProjects extends Action Console::title('Delete orphaned projects V1'); Console::success(APP_NAME . ' Delete orphaned projects started'); + /** @var array $collections */ + $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; + /* Initialise new Utopia app */ $app = new App('UTC'); $console = $app->getResource('console'); @@ -55,6 +55,7 @@ class DeleteOrphanedProjects extends Action Console::success("Found a total of: {$totalProjects} projects"); $orphans = 1; + $cnt = 0; $count = 0; $limit = 30; $sum = 30; @@ -79,24 +80,43 @@ class DeleteOrphanedProjects extends Action $dbForProject = new Database($adapter, $cache); $dbForProject->setDefaultDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); - $collectionsCreated = $dbForProject->count(Database::METADATA); - if ($collectionsCreated === 0) { - if ($commit === true) { - Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); - $this->deleteProject($dbForConsole, $project->getId()); - } else { - Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); + $collectionsCreated = 0; + $cnt++; + if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { + $collectionsCreated = $dbForProject->count(Database::METADATA); + } + + $msg = '(' . $cnt . ') ignoring found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; + /** + * +2 == audit+abuse + */ + if ($collectionsCreated === (count($collectionsConfig) + 2)) { + Console::log($msg . ' ignoring....'); + continue; + } + + Console::log($msg); + + if ($collectionsCreated > 0) { + $collections = $dbForProject->find(Database::METADATA, []); + foreach ($collections as $collection) { + if ($commit) { + $dbForProject->deleteCollection($collection->getId()); + $dbForConsole->deleteCachedCollection($collection->getId()); + } + Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')'); } + } + if ($commit) { + $dbForConsole->deleteDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); + } + + Console::info('--Deleting project no (' . $project->getInternalId() . ')'); + $orphans++; - } } catch (\Throwable $th) { - if ($commit === true) { - Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); - $this->deleteProject($dbForConsole, $project->getId()); - } else { - Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); - } - $orphans++; + Console::error('Error: ' . $th->getMessage()); } finally { $pools ->get($db) @@ -117,13 +137,4 @@ class DeleteOrphanedProjects extends Action Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); } - - private function deleteProject(Database $dbForConsole, $projectId): void - { - try { - $dbForConsole->deleteDocument('projects', $projectId); - } catch (\Throwable $th) { - Console::error('Error when trying to delete project (' . $projectId . ') ' . $th->getMessage()); - } - } } From 60c0f4c97393d8d665da5ab7d7b697fafa69abbc Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 19:27:20 +0200 Subject: [PATCH 14/67] delete orphaned projects task --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2af9c54a6e..2da3c91384 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -86,9 +86,9 @@ class DeleteOrphanedProjects extends Action $collectionsCreated = $dbForProject->count(Database::METADATA); } - $msg = '(' . $cnt . ') ignoring found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; + $msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; /** - * +2 == audit+abuse + * +2 = audit+abuse */ if ($collectionsCreated === (count($collectionsConfig) + 2)) { Console::log($msg . ' ignoring....'); From 3d9ee8bc521367d8242de2e4035970d44a9050da Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 19:28:48 +0200 Subject: [PATCH 15/67] delete orphaned projects task --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2da3c91384..ee9931bde4 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -25,7 +25,7 @@ class DeleteOrphanedProjects extends Action $this ->desc('Delete orphaned projects') - ->param('commit', false, new boolean(true), 'Commit project deletion', true) + ->param('commit', false, new Boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') ->inject('dbForConsole') From be85b173e774c0f85ea95bc651b0829648c5859e Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 14 Nov 2023 16:49:16 +0530 Subject: [PATCH 16/67] Only delete repositories linked to the particular project --- src/Appwrite/Platform/Workers/Deletes.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 6bb8636695..2246816436 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -730,6 +730,7 @@ class Deletes extends Action */ Console::info("Deleting VCS repositories and comments linked to function " . $functionId); $this->deleteByGroup('repositories', [ + Query::equal('projectInternalId', [$projectInternalId]), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('resourceType', ['function']), ], $dbForConsole, function (Document $document) use ($dbForConsole) { From 78eb5105e7ffc8ea4a204bf25c08e9c7e87c1884 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:08:09 +0530 Subject: [PATCH 17/67] Update projectInternalId var --- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 2246816436..4b888b9fd6 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -730,7 +730,7 @@ class Deletes extends Action */ Console::info("Deleting VCS repositories and comments linked to function " . $functionId); $this->deleteByGroup('repositories', [ - Query::equal('projectInternalId', [$projectInternalId]), + Query::equal('projectInternalId', [$project->getInternalId()]), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('resourceType', ['function']), ], $dbForConsole, function (Document $document) use ($dbForConsole) { From 4f2f76db2239510fe833828daefc9e628ddc2a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 14:11:54 +0100 Subject: [PATCH 18/67] Improve deletion relation with IDs --- src/Appwrite/Platform/Workers/Deletes.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 4b888b9fd6..b95a13a12e 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -735,10 +735,10 @@ class Deletes extends Action Query::equal('resourceType', ['function']), ], $dbForConsole, function (Document $document) use ($dbForConsole) { $providerRepositoryId = $document->getAttribute('providerRepositoryId', ''); - $projectId = $document->getAttribute('projectId', ''); + $projectInternalId = $document->getAttribute('projectInternalId', ''); $this->deleteByGroup('vcsComments', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), - Query::equal('projectId', [$projectId]), + Query::equal('projectInternalId', [$projectInternalId]), ], $dbForConsole); }); From fb6455b783c554536aac864753301b840708e432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 14:43:33 +0100 Subject: [PATCH 19/67] Add script to patch missing repos documents --- Dockerfile | 1 + bin/patch-recreate-repositories-documents | 3 + .../patchRecreateRepositoriesDocuments.php | 141 ++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 bin/patch-recreate-repositories-documents create mode 100644 src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php diff --git a/Dockerfile b/Dockerfile index 33b1434659..059c499bd9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -100,6 +100,7 @@ RUN chmod +x /usr/local/bin/doctor && \ RUN chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ + chmod +x /usr/local/bin/patch-recreate-repositories-documents && \ chmod +x /usr/local/bin/patch-delete-project-collections && \ chmod +x /usr/local/bin/delete-orphaned-projects && \ chmod +x /usr/local/bin/clear-card-cache && \ diff --git a/bin/patch-recreate-repositories-documents b/bin/patch-recreate-repositories-documents new file mode 100644 index 0000000000..8c6c4157f4 --- /dev/null +++ b/bin/patch-recreate-repositories-documents @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php patch-recreate-repositories-documents $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php new file mode 100644 index 0000000000..5749380d0a --- /dev/null +++ b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php @@ -0,0 +1,141 @@ +desc('Recreate missing repositories in consoleDB from projectDBs. They can be missing if you used Appwrite 1.4.10 or 1.4.11, and deleted a function.') + ->param('after', '', new Text(36), 'After cursor', true) + ->param('projectId', '', new Text(36), 'Select project to validate', true) + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn ($after, $projectId, $dbForConsole, $getProjectDB) => $this->action($after, $projectId, $dbForConsole, $getProjectDB)); + } + + public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void + { + Console::info("Starting the patch"); + + $startTime = microtime(true); + + if(!empty($projectId)) { + $project = $dbForConsole->getDocument('projects', $projectId); + $dbForProject = call_user_func($getProjectDB, $project); + $this->recreateRepositories($dbForConsole, $dbForProject, $project); + } else { + $queries = []; + if(!empty($after)) { + Console::info("Iterating remaining projects after project with ID {$after}"); + $project = $dbForConsole->getDocument('projects', $after); + $queries = [Query::cursorAfter($project)]; + } else { + Console::info("Iterating all projects"); + } + $this->foreachDocument($dbForConsole, 'projects', $queries, function(Document $project) use($getProjectDB, $dbForConsole){ + $dbForProject = call_user_func($getProjectDB, $project); + $this->recreateRepositories($dbForConsole, $dbForProject, $project); + }); + } + + $endTime = microtime(true); + $timeTaken = $endTime - $startTime; + + $hours = (int)($timeTaken / 3600); + $timeTaken -= $hours * 3600; + $minutes = (int)($timeTaken / 60); + $timeTaken -= $minutes * 60; + $seconds = (int)$timeTaken; + $milliseconds = ($timeTaken - $seconds) * 1000; + Console::info("Recreate patch completed in $hours h, $minutes m, $seconds s, $milliseconds mis ( total $timeTaken milliseconds)"); + } + + protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void + { + $limit = 1000; + $results = []; + $sum = $limit; + $latestDocument = null; + + while ($sum === $limit) { + $newQueries = $queries; + + if ($latestDocument != null) { + array_unshift($newQueries, Query::cursorAfter($latestDocument)); + } + $newQueries[] = Query::limit($limit); + $results = $database->find($collection, $newQueries); + + if (empty($results)) { + return; + } + + $sum = count($results); + + foreach ($results as $document) { + if (is_callable($callback)) { + $callback($document); + } + } + $latestDocument = $results[array_key_last($results)]; + } + } + + public function recreateRepositories(Database $dbForConsole, Database $dbForProject, Document $project): void + { + $projectId = $project->getId(); + Console::log("Running patch for project {$projectId}"); + + $this->foreachDocument($dbForProject, 'functions', [], function(Document $function) use ($dbForConsole, $project) { + $isConnected = !empty($function->getAttribute('providerRepositoryId', '')); + + if($isConnected) { + $repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', '')); + + if($repository->isEmpty()) { + $projectId = $project->getId(); + $functionId = $function->getId(); + Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}"); + + $repository = $dbForConsole->createDocument('repositories', new Document([ + '$id' => ID::unique(), + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'installationId' => $function->getAttribute('installationId', ''), + 'installationInternalId' => $function->getAttribute('installationInternalId', ''), + 'projectId' => $project->getId(), + 'projectInternalId' => $project->getInternalId(), + 'providerRepositoryId' => $function->getAttribute('providerRepositoryId', ''), + 'resourceId' => $function->getId(), + 'resourceInternalId' => $function->getInternalId(), + 'resourceType' => 'function', + 'providerPullRequestIds' => [] + ])); + } + } + }); + } +} From c133bccfa93b52cca8dd9e0210f4d2ee8fc2d921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 15:06:58 +0100 Subject: [PATCH 20/67] Finish recreate repos docs script --- .env | 38 ++++++++++++++++--- src/Appwrite/Platform/Services/Tasks.php | 2 + .../patchRecreateRepositoriesDocuments.php | 32 ++++++++++------ 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/.env b/.env index ad551e705a..759ddcc051 100644 --- a/.env +++ b/.env @@ -90,12 +90,38 @@ _APP_GRAPHQL_MAX_COMPLEXITY=250 _APP_GRAPHQL_MAX_DEPTH=3 _APP_DOCKER_HUB_USERNAME= _APP_DOCKER_HUB_PASSWORD= -_APP_VCS_GITHUB_APP_NAME= -_APP_VCS_GITHUB_PRIVATE_KEY=disabled -_APP_VCS_GITHUB_APP_ID= -_APP_VCS_GITHUB_CLIENT_ID= -_APP_VCS_GITHUB_CLIENT_SECRET= -_APP_VCS_GITHUB_WEBHOOK_SECRET= +_APP_VCS_GITHUB_APP_NAME=appwrite-generated-on-22-5-2023 +_APP_VCS_GITHUB_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEApUP3k7ratjenwOA1AtnmvZbNIm9Mhu181ryrnb7M1z1/OLUo +jZ1iS8u1K/GQtoqCNI8RR38rBXW1hzfC34qAL3XrJr4caK+AKzV7B4y9e2Zn+VlF +ZZ2rknNPlRLCwll6V60M7suqLONUADs6bvEs8GIR926gwNi8SmomtIj8k8UZFW00 +9L0OM0pvajJKASPynZ7jXfxic5Yu060BQq+IAtOkOaxyBF1xiGQRBuAt2movTI2X +4PwGcSCy7YD6v2DRgHIoYehtdWx0DrbRgCXsmwMin5BR/7ZhBSJE7YVH9vSqWThe +V8VmLKPiZZC5ZW2Vf28kM/FlkCgjfd0tNXs8lQIDAQABAoIBAQCWaFYhODSnE93z +ttnoH2JVd7J4PW0be3ZbhNh3t1d8KPbpKE6hG/SC4QGg3bgDuekoZnCmbkE8NdWh +G4maotVo3FvIJct7JwZxzLmMtHUaoqfMEogLJEUrAxERrkJcWMz0kIUtq2PUeIxR +rZXPtGVe3RJW63MYL+ilnRhexDGDVa5I50jFCTT5GXXHl6tw6tei1rDc2kI7pb3k +BGAxsRGXUK6Di9EmR2Z8xFHvM3LeuSWR2WalR8+abn661P3J+InMqh0s+TBB86kH +6NUxoWKSCXjPfMWj15M0Wwm4CBlzgc6GS106hmHPi3YAQLCV4x0l4qmq+bdsQOnN +VAOnPAohAoGBANidEYsMm5jaTklFnIExBkm5OHcKRuMgplMoEsVE/dglBPCJB3Uf +JUmhMRVazztDGD0ciImypt4j9klR4zqSQv4Aa8OdwW/jcuNPm3qQrdSm86iP2Lbe +V9DNKK0vQ/3srwIYxl9qabOLaQrKAeiPxfUuL91iyDIYtsPPIgfvVGfdAoGBAMNQ +v5wruODDgf0mMm1nA9LNHlkMi8uaVvxFAhjWtmPOH4FHXgzMDGC6syvL+d1XIdtF +tA/j/f/A1zFsYzeZRVBqVmpd8rvRzFTaRrBgLjI/vxmbJ7syr9rT7iZYvFYUv3c+ +mr8m5AIGULiGmMYnSWttIi2prlA17FC5Qp/lq/gZAoGAVrBlePSOwNl9Qy2suLda +AN8zjdB7FiLW7ai3+mLmBD6sf2cXqPPSBGmSLy2sidcMOEjXC+SHi5dw1V8ERUiL +rwOUHTFhXNn1/Kq7Wo3UQ6qdEPSgkm7hThsNEGI+H709POWVXlJEAyrj2wGFSgFg +BAN7/GmwHPxvCGY5BFvvt7ECgYBIWqOA4RmN+h8vfnTz3lOmReJWLrWi6TwMHCxY +s0HB21wEckG/D+AN/Vvef6PCgULDjiDUOiugEPonDvX6ZMcusRXuNXt0ZJYDYREK +ybaTWtYaUEX5rR9EO3pfrkOmx+zd6c09vtR8g4ZntUTnMyqZp0YgEFnI0REIHnk1 +7sk0EQKBgGTzNU9Ir6cEZh4j+Qf5rA38bejkD8aRAYg5ozQcbNRJnrC7QnpmZPeD +X/E9MZ6wj1BVXEn2oNC63n3QB+B8OhrIDAYDbnaCLzVDl/BTuom3uTCYk0beKncz +AurSDpc15RFYjqn0DPBSii/DTaQIz0Rg+seZrOp5Ii2LrSlsnDPf +-----END RSA PRIVATE KEY-----" +_APP_VCS_GITHUB_APP_ID=337303 +_APP_VCS_GITHUB_CLIENT_ID=Iv1.306ee38582d3f948 +_APP_VCS_GITHUB_CLIENT_SECRET=eafc638eaeebe95c0db0fdf59a0a99b9e41832eb +_APP_VCS_GITHUB_WEBHOOK_SECRET=gzbrfuenqodiefbrg39u _APP_MIGRATIONS_FIREBASE_CLIENT_ID= _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET= _APP_ASSISTANT_OPENAI_API_KEY= \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index e725ff5f3e..28d7046dd1 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -19,6 +19,7 @@ use Appwrite\Platform\Tasks\VolumeSync; use Appwrite\Platform\Tasks\CalcTierStats; use Appwrite\Platform\Tasks\Upgrade; use Appwrite\Platform\Tasks\DeleteOrphanedProjects; +use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments; class Tasks extends Service { @@ -42,6 +43,7 @@ class Tasks extends Service ->addAction(Specs::getName(), new Specs()) ->addAction(CalcTierStats::getName(), new CalcTierStats()) ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects()) + ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments()) ; } diff --git a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php index 5749380d0a..4d04802f50 100644 --- a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php +++ b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php @@ -3,8 +3,6 @@ namespace Appwrite\Platform\Tasks; use Utopia\Platform\Action; -use Appwrite\Event\Certificate; -use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document; @@ -12,10 +10,9 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; -use Utopia\Validator\Hostname; use Utopia\Validator\Text; -class patchRecreateRepositoriesDocuments extends Action +class PatchRecreateRepositoriesDocuments extends Action { public static function getName(): string { @@ -34,25 +31,25 @@ class patchRecreateRepositoriesDocuments extends Action } public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void - { + { Console::info("Starting the patch"); $startTime = microtime(true); - if(!empty($projectId)) { + if (!empty($projectId)) { $project = $dbForConsole->getDocument('projects', $projectId); $dbForProject = call_user_func($getProjectDB, $project); $this->recreateRepositories($dbForConsole, $dbForProject, $project); } else { $queries = []; - if(!empty($after)) { + if (!empty($after)) { Console::info("Iterating remaining projects after project with ID {$after}"); $project = $dbForConsole->getDocument('projects', $after); $queries = [Query::cursorAfter($project)]; } else { Console::info("Iterating all projects"); } - $this->foreachDocument($dbForConsole, 'projects', $queries, function(Document $project) use($getProjectDB, $dbForConsole){ + $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole) { $dbForProject = call_user_func($getProjectDB, $project); $this->recreateRepositories($dbForConsole, $dbForProject, $project); }); @@ -106,13 +103,13 @@ class patchRecreateRepositoriesDocuments extends Action $projectId = $project->getId(); Console::log("Running patch for project {$projectId}"); - $this->foreachDocument($dbForProject, 'functions', [], function(Document $function) use ($dbForConsole, $project) { + $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForConsole, $project) { $isConnected = !empty($function->getAttribute('providerRepositoryId', '')); - if($isConnected) { + if ($isConnected) { $repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', '')); - if($repository->isEmpty()) { + if ($repository->isEmpty()) { $projectId = $project->getId(); $functionId = $function->getId(); Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}"); @@ -134,6 +131,19 @@ class patchRecreateRepositoriesDocuments extends Action 'resourceType' => 'function', 'providerPullRequestIds' => [] ])); + + $function = $dbForProject->updateDocument('functions', $function->getId(), $function + ->setAttribute('repositoryId', $repository->getId()) + ->setAttribute('repositoryInternalId', $repository->getInternalId())); + + $this->foreachDocument($dbForProject, 'deployments', [ + Query::equal('resourceInternalId', [$function->getInternalId()]), + Query::equal('resourceType', ['functions']) + ], function (Document $deployment) use ($dbForProject, $repository) { + $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment + ->setAttribute('repositoryId', $repository->getId()) + ->setAttribute('repositoryInternalId', $repository->getInternalId())); + }); } } }); From 2572cb43c2007829bb4f91dd58fc0a680cc3f7d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 15:11:17 +0100 Subject: [PATCH 21/67] Code cleanup --- .env | 38 ++++++-------------------------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/.env b/.env index 759ddcc051..ad551e705a 100644 --- a/.env +++ b/.env @@ -90,38 +90,12 @@ _APP_GRAPHQL_MAX_COMPLEXITY=250 _APP_GRAPHQL_MAX_DEPTH=3 _APP_DOCKER_HUB_USERNAME= _APP_DOCKER_HUB_PASSWORD= -_APP_VCS_GITHUB_APP_NAME=appwrite-generated-on-22-5-2023 -_APP_VCS_GITHUB_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEApUP3k7ratjenwOA1AtnmvZbNIm9Mhu181ryrnb7M1z1/OLUo -jZ1iS8u1K/GQtoqCNI8RR38rBXW1hzfC34qAL3XrJr4caK+AKzV7B4y9e2Zn+VlF -ZZ2rknNPlRLCwll6V60M7suqLONUADs6bvEs8GIR926gwNi8SmomtIj8k8UZFW00 -9L0OM0pvajJKASPynZ7jXfxic5Yu060BQq+IAtOkOaxyBF1xiGQRBuAt2movTI2X -4PwGcSCy7YD6v2DRgHIoYehtdWx0DrbRgCXsmwMin5BR/7ZhBSJE7YVH9vSqWThe -V8VmLKPiZZC5ZW2Vf28kM/FlkCgjfd0tNXs8lQIDAQABAoIBAQCWaFYhODSnE93z -ttnoH2JVd7J4PW0be3ZbhNh3t1d8KPbpKE6hG/SC4QGg3bgDuekoZnCmbkE8NdWh -G4maotVo3FvIJct7JwZxzLmMtHUaoqfMEogLJEUrAxERrkJcWMz0kIUtq2PUeIxR -rZXPtGVe3RJW63MYL+ilnRhexDGDVa5I50jFCTT5GXXHl6tw6tei1rDc2kI7pb3k -BGAxsRGXUK6Di9EmR2Z8xFHvM3LeuSWR2WalR8+abn661P3J+InMqh0s+TBB86kH -6NUxoWKSCXjPfMWj15M0Wwm4CBlzgc6GS106hmHPi3YAQLCV4x0l4qmq+bdsQOnN -VAOnPAohAoGBANidEYsMm5jaTklFnIExBkm5OHcKRuMgplMoEsVE/dglBPCJB3Uf -JUmhMRVazztDGD0ciImypt4j9klR4zqSQv4Aa8OdwW/jcuNPm3qQrdSm86iP2Lbe -V9DNKK0vQ/3srwIYxl9qabOLaQrKAeiPxfUuL91iyDIYtsPPIgfvVGfdAoGBAMNQ -v5wruODDgf0mMm1nA9LNHlkMi8uaVvxFAhjWtmPOH4FHXgzMDGC6syvL+d1XIdtF -tA/j/f/A1zFsYzeZRVBqVmpd8rvRzFTaRrBgLjI/vxmbJ7syr9rT7iZYvFYUv3c+ -mr8m5AIGULiGmMYnSWttIi2prlA17FC5Qp/lq/gZAoGAVrBlePSOwNl9Qy2suLda -AN8zjdB7FiLW7ai3+mLmBD6sf2cXqPPSBGmSLy2sidcMOEjXC+SHi5dw1V8ERUiL -rwOUHTFhXNn1/Kq7Wo3UQ6qdEPSgkm7hThsNEGI+H709POWVXlJEAyrj2wGFSgFg -BAN7/GmwHPxvCGY5BFvvt7ECgYBIWqOA4RmN+h8vfnTz3lOmReJWLrWi6TwMHCxY -s0HB21wEckG/D+AN/Vvef6PCgULDjiDUOiugEPonDvX6ZMcusRXuNXt0ZJYDYREK -ybaTWtYaUEX5rR9EO3pfrkOmx+zd6c09vtR8g4ZntUTnMyqZp0YgEFnI0REIHnk1 -7sk0EQKBgGTzNU9Ir6cEZh4j+Qf5rA38bejkD8aRAYg5ozQcbNRJnrC7QnpmZPeD -X/E9MZ6wj1BVXEn2oNC63n3QB+B8OhrIDAYDbnaCLzVDl/BTuom3uTCYk0beKncz -AurSDpc15RFYjqn0DPBSii/DTaQIz0Rg+seZrOp5Ii2LrSlsnDPf ------END RSA PRIVATE KEY-----" -_APP_VCS_GITHUB_APP_ID=337303 -_APP_VCS_GITHUB_CLIENT_ID=Iv1.306ee38582d3f948 -_APP_VCS_GITHUB_CLIENT_SECRET=eafc638eaeebe95c0db0fdf59a0a99b9e41832eb -_APP_VCS_GITHUB_WEBHOOK_SECRET=gzbrfuenqodiefbrg39u +_APP_VCS_GITHUB_APP_NAME= +_APP_VCS_GITHUB_PRIVATE_KEY=disabled +_APP_VCS_GITHUB_APP_ID= +_APP_VCS_GITHUB_CLIENT_ID= +_APP_VCS_GITHUB_CLIENT_SECRET= +_APP_VCS_GITHUB_WEBHOOK_SECRET= _APP_MIGRATIONS_FIREBASE_CLIENT_ID= _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET= _APP_ASSISTANT_OPENAI_API_KEY= \ No newline at end of file From 141b864a56c509f4adb561b98244a8edfdda7fc0 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:49:23 +0530 Subject: [PATCH 22/67] Update permissions for create repository document --- app/controllers/api/functions.php | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 268acc0692..9720a68889 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -761,11 +761,9 @@ App::put('/v1/functions/:functionId') $repository = $dbForConsole->createDocument('repositories', new Document([ '$id' => ID::unique(), '$permissions' => [ - Permission::read(Role::team(ID::custom($teamId))), - Permission::update(Role::team(ID::custom($teamId), 'owner')), - Permission::update(Role::team(ID::custom($teamId), 'developer')), - Permission::delete(Role::team(ID::custom($teamId), 'owner')), - Permission::delete(Role::team(ID::custom($teamId), 'developer')), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), ], 'installationId' => $installation->getId(), 'installationInternalId' => $installation->getInternalId(), From c2e5849c49738b59001d61606dc0a642b9702e0b Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 14 Nov 2023 16:49:45 +0200 Subject: [PATCH 23/67] removing blank line --- src/Appwrite/Usage/Calculators/TimeSeries.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 9ead48e93f..38574b9b03 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -423,7 +423,6 @@ class TimeSeries extends Calculator */ private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void { - $id = \md5("{$time}_{$period}_{$metric}"); $project = $this->database->getDocument('projects', $projectId); $database = call_user_func($this->getProjectDB, $project); From 7e1b618769448fd222d51551f64080d49443c1fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 16:50:59 +0100 Subject: [PATCH 24/67] Fix permission issues with repositories collection --- app/controllers/api/functions.php | 18 ++++++++++++------ app/controllers/api/vcs.php | 24 ++++++++++++------------ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 9720a68889..cbdbd3a1cb 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -242,12 +242,16 @@ App::post('/v1/functions') // Git connect logic if (!empty($providerRepositoryId)) { + $teamId = $project->getAttribute('teamId', ''); + $repository = $dbForConsole->createDocument('repositories', new Document([ '$id' => ID::unique(), '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), + Permission::read(Role::team(ID::custom($teamId))), + Permission::update(Role::team(ID::custom($teamId), 'owner')), + Permission::update(Role::team(ID::custom($teamId), 'developer')), + Permission::delete(Role::team(ID::custom($teamId), 'owner')), + Permission::delete(Role::team(ID::custom($teamId), 'developer')), ], 'installationId' => $installation->getId(), 'installationInternalId' => $installation->getInternalId(), @@ -761,9 +765,11 @@ App::put('/v1/functions/:functionId') $repository = $dbForConsole->createDocument('repositories', new Document([ '$id' => ID::unique(), '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), + Permission::read(Role::team(ID::custom($teamId))), + Permission::update(Role::team(ID::custom($teamId), 'owner')), + Permission::update(Role::team(ID::custom($teamId), 'developer')), + Permission::delete(Role::team(ID::custom($teamId), 'owner')), + Permission::delete(Role::team(ID::custom($teamId), 'developer')), ], 'installationId' => $installation->getId(), 'installationInternalId' => $installation->getInternalId(), diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 8b61580b76..68a84d0a1d 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -857,10 +857,10 @@ App::post('/v1/vcs/github/events') $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); //find functionId from functions table - $repositories = $dbForConsole->find('repositories', [ + $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::limit(100), - ]); + ])); // create new deployment only on push and not when branch is created if (!$providerBranchCreated) { @@ -877,13 +877,13 @@ App::post('/v1/vcs/github/events') ]); foreach ($installations as $installation) { - $repositories = $dbForConsole->find('repositories', [ + $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [ Query::equal('installationInternalId', [$installation->getInternalId()]), Query::limit(1000) - ]); + ])); foreach ($repositories as $repository) { - $dbForConsole->deleteDocument('repositories', $repository->getId()); + Authorization::skip(fn () => $dbForConsole->deleteDocument('repositories', $repository->getId())); } $dbForConsole->deleteDocument('installations', $installation->getId()); @@ -915,10 +915,10 @@ App::post('/v1/vcs/github/events') $providerCommitAuthor = $commitDetails["commitAuthor"] ?? ''; $providerCommitMessage = $commitDetails["commitMessage"] ?? ''; - $repositories = $dbForConsole->find('repositories', [ + $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::orderDesc('$createdAt') - ]); + ])); $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $external, $dbForConsole, $queueForBuilds, $getProjectDB, $request); } elseif ($parsedPayload["action"] == "closed") { @@ -929,10 +929,10 @@ App::post('/v1/vcs/github/events') $external = $parsedPayload["external"] ?? true; if ($external) { - $repositories = $dbForConsole->find('repositories', [ + $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::orderDesc('$createdAt') - ]); + ])); foreach ($repositories as $repository) { $providerPullRequestIds = $repository->getAttribute('providerPullRequestIds', []); @@ -1092,9 +1092,9 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor throw new Exception(Exception::INSTALLATION_NOT_FOUND); } - $repository = $dbForConsole->getDocument('repositories', $repositoryId, [ + $repository = Authorization::skip(fn () => $dbForConsole->getDocument('repositories', $repositoryId, [ Query::equal('projectInternalId', [$project->getInternalId()]) - ]); + ])); if ($repository->isEmpty()) { throw new Exception(Exception::REPOSITORY_NOT_FOUND); @@ -1109,7 +1109,7 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor // TODO: Delete from array when PR is closed - $repository = $dbForConsole->updateDocument('repositories', $repository->getId(), $repository); + $repository = Authorization::skip(fn () => $dbForConsole->updateDocument('repositories', $repository->getId(), $repository)); $privateKey = App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'); $githubAppId = App::getEnv('_APP_VCS_GITHUB_APP_ID'); From 1c4fea0fc3e77b80cecad66a556ba9856d9d9076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 17:45:02 +0100 Subject: [PATCH 25/67] Fix patch script, make errors silent --- .../patchRecreateRepositoriesDocuments.php | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php index 4d04802f50..93e6c527bb 100644 --- a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php +++ b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php @@ -37,9 +37,17 @@ class PatchRecreateRepositoriesDocuments extends Action $startTime = microtime(true); if (!empty($projectId)) { - $project = $dbForConsole->getDocument('projects', $projectId); - $dbForProject = call_user_func($getProjectDB, $project); - $this->recreateRepositories($dbForConsole, $dbForProject, $project); + try { + $project = $dbForConsole->getDocument('projects', $projectId); + $dbForProject = call_user_func($getProjectDB, $project); + $this->recreateRepositories($dbForConsole, $dbForProject, $project); + } catch (\Throwable $th) { + Console::error("Unexpected error occured with Project ID {$projectId}"); + Console::error('[Error] Type: ' . get_class($th)); + Console::error('[Error] Message: ' . $th->getMessage()); + Console::error('[Error] File: ' . $th->getFile()); + Console::error('[Error] Line: ' . $th->getLine()); + } } else { $queries = []; if (!empty($after)) { @@ -50,8 +58,18 @@ class PatchRecreateRepositoriesDocuments extends Action Console::info("Iterating all projects"); } $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole) { - $dbForProject = call_user_func($getProjectDB, $project); - $this->recreateRepositories($dbForConsole, $dbForProject, $project); + $projectId = $project->getId(); + + try { + $dbForProject = call_user_func($getProjectDB, $project); + $this->recreateRepositories($dbForConsole, $dbForProject, $project); + } catch (\Throwable $th) { + Console::error("Unexpected error occured with Project ID {$projectId}"); + Console::error('[Error] Type: ' . get_class($th)); + Console::error('[Error] Message: ' . $th->getMessage()); + Console::error('[Error] File: ' . $th->getFile()); + Console::error('[Error] Line: ' . $th->getLine()); + } }); } From c3015340ce7ae0d3686ce8aa5a11b36b4226c5f7 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 15 Nov 2023 16:46:43 +1300 Subject: [PATCH 26/67] Update database --- composer.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/composer.lock b/composer.lock index 25f0c3964d..d362f3aa9c 100644 --- a/composer.lock +++ b/composer.lock @@ -1906,16 +1906,16 @@ }, { "name": "utopia-php/database", - "version": "0.45.1", + "version": "0.45.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "0e76f996439b80794ab73c2fffdb51ebd6676e4b" + "reference": "dc789f2c1fd8b5ee07ff883e11c9ad7970824788" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/0e76f996439b80794ab73c2fffdb51ebd6676e4b", - "reference": "0e76f996439b80794ab73c2fffdb51ebd6676e4b", + "url": "https://api.github.com/repos/utopia-php/database/zipball/dc789f2c1fd8b5ee07ff883e11c9ad7970824788", + "reference": "dc789f2c1fd8b5ee07ff883e11c9ad7970824788", "shasum": "" }, "require": { @@ -1956,9 +1956,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.45.1" + "source": "https://github.com/utopia-php/database/tree/0.45.2" }, - "time": "2023-11-01T08:30:19+00:00" + "time": "2023-11-15T03:38:47+00:00" }, { "name": "utopia-php/domains", From 94931ff4ce5d8bfb1057c4743db488d2566874a3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 15 Nov 2023 16:47:49 +1300 Subject: [PATCH 27/67] Fix non-PSR-compliant file name --- ...toriesDocuments.php => PatchRecreateRepositoriesDocuments.php} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/Appwrite/Platform/Tasks/{patchRecreateRepositoriesDocuments.php => PatchRecreateRepositoriesDocuments.php} (100%) diff --git a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php similarity index 100% rename from src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php rename to src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php From 2e70ed59af5efcccbf3bf55e14176310e240e5e0 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Wed, 15 Nov 2023 11:41:42 +0530 Subject: [PATCH 28/67] Fix git installation deletion --- app/controllers/api/vcs.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 68a84d0a1d..1b0c993e11 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -1046,8 +1046,8 @@ App::delete('/v1/vcs/installations/:installationId') ->inject('response') ->inject('project') ->inject('dbForConsole') - ->inject('deletes') - ->action(function (string $installationId, Response $response, Document $project, Database $dbForConsole, Delete $deletes) { + ->inject('queueForDeletes') + ->action(function (string $installationId, Response $response, Document $project, Database $dbForConsole, Delete $queueForDeletes) { $installation = $dbForConsole->getDocument('installations', $installationId); if ($installation->isEmpty()) { @@ -1058,7 +1058,7 @@ App::delete('/v1/vcs/installations/:installationId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove installation from DB'); } - $deletes + $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($installation); From df38a8efd13b0050705fffcd6d19a6346eb49594 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 15 Nov 2023 13:44:37 +0100 Subject: [PATCH 29/67] ci: validate composer file and lock --- .github/workflows/linter.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 095449de2f..eeb1707e61 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -18,6 +18,10 @@ jobs: - run: git checkout HEAD^2 + - name: Validate composer.json and composer.lock + run: | + docker run --rm -v $PWD:/app composer sh -c \ + "composer install --profile --ignore-platform-reqs && composer validate" - name: Run Linter run: | docker run --rm -v $PWD:/app composer sh -c \ From 42a92933a8e6b018423afc15ec68fdb0c7aee578 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 15 Nov 2023 15:42:44 +0100 Subject: [PATCH 30/67] ci: remove install before composer validate --- .github/workflows/linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index eeb1707e61..899c27a135 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -21,7 +21,7 @@ jobs: - name: Validate composer.json and composer.lock run: | docker run --rm -v $PWD:/app composer sh -c \ - "composer install --profile --ignore-platform-reqs && composer validate" + "composer validate" - name: Run Linter run: | docker run --rm -v $PWD:/app composer sh -c \ From 7efecbca8b0637d0a16c7d78ded922d43a44a5d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Wed, 15 Nov 2023 17:20:50 +0000 Subject: [PATCH 31/67] chore: fixed indentation --- src/Appwrite/Usage/Calculators/TimeSeries.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 38574b9b03..6dbf49c44c 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -432,13 +432,13 @@ class TimeSeries extends Calculator $document = $database->getDocument('stats', $id); if ($document->isEmpty()) { $database->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => $type, - 'region' => $this->region, + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => $type, + 'region' => $this->region, ])); } else { $database->updateDocument( From 31d5b75034e95c196666e9516f9e2eb01ac97d9a Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 15 Nov 2023 17:23:35 +0000 Subject: [PATCH 32/67] Bump console to version 3.2.7 --- .gitmodules | 2 +- app/console | 2 +- app/init.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index ff2e0a6aab..af12124355 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = 3.2.6 + branch = 3.2.7 diff --git a/app/console b/app/console index f7c34a1b37..49d039ed07 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit f7c34a1b37d53dd5f28c83b4e12a4e68fcd9b484 +Subproject commit 49d039ed07628155e7f56e2c997fcef90ecde267 diff --git a/app/init.php b/app/init.php index be6b440498..2572d01151 100644 --- a/app/init.php +++ b/app/init.php @@ -109,7 +109,7 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 516; +const APP_CACHE_BUSTER = 327; const APP_VERSION_STABLE = '1.4.11'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; From 6b5e734dfaedfb6fb291d7dab6a7a883dcc5a555 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 15 Nov 2023 20:09:50 +0200 Subject: [PATCH 33/67] indentation small fix --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index ee9931bde4..5f0ecbe1db 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -112,9 +112,9 @@ class DeleteOrphanedProjects extends Action $dbForConsole->deleteCachedDocument('projects', $project->getId()); } - Console::info('--Deleting project no (' . $project->getInternalId() . ')'); + Console::info('--Deleting project no (' . $project->getInternalId() . ')'); - $orphans++; + $orphans++; } catch (\Throwable $th) { Console::error('Error: ' . $th->getMessage()); } finally { From 37e1e251705f11f7f192bbc2002b343a226508aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Wed, 15 Nov 2023 18:31:33 +0000 Subject: [PATCH 34/67] chore: update versions and changelog --- CHANGES.md | 4 ++++ README-CN.md | 6 +++--- README.md | 6 +++--- app/init.php | 2 +- src/Appwrite/Migration/Migration.php | 1 + 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e8f0c08c1f..513ee2ec90 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# Version 1.4.12 + +## Bug fixes + # Version 1.4.11 ## Miscellaneous diff --git a/README-CN.md b/README-CN.md index 4e45c423c4..1bbc8eb234 100644 --- a/README-CN.md +++ b/README-CN.md @@ -66,7 +66,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.12 ``` ### Windows @@ -78,7 +78,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.12 ``` #### PowerShell @@ -88,7 +88,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.12 ``` 运行后,可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后,服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index 88615a355a..3a1d2dbe9f 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.12 ``` ### Windows @@ -88,7 +88,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.12 ``` #### PowerShell @@ -98,7 +98,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.12 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/init.php b/app/init.php index 2572d01151..2c0219eec2 100644 --- a/app/init.php +++ b/app/init.php @@ -110,7 +110,7 @@ const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 327; -const APP_VERSION_STABLE = '1.4.11'; +const APP_VERSION_STABLE = '1.4.12'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 406acae7df..8f68e31be4 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -76,6 +76,7 @@ abstract class Migration '1.4.9' => 'V19', '1.4.10' => 'V19', '1.4.11' => 'V19', + '1.4.12' => 'V19' ]; /** From e145ad4cb4ed9c0bc6e6c8613bf51c17e5b5ba5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Wed, 15 Nov 2023 18:41:57 +0000 Subject: [PATCH 35/67] chore: update versions and changelog --- CHANGES.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 513ee2ec90..889f65e1e7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,16 @@ # Version 1.4.12 +## Miscellaneous +* Bump console to version 3.2.7 [#7148](https://github.com/appwrite/appwrite/pull/7148) +* Chore update database to 0.45.2 [#7138](https://github.com/appwrite/appwrite/pull/7138) +* Implement queue thresholds for the health API [#7123](https://github.com/appwrite/appwrite/pull/7123) +* Add Authorization::skip to the usage worker [#7124](https://github.com/appwrite/appwrite/pull/7124) + ## Bug fixes +* fix: use queueForDeletes in git installation delete endpoint [#7140](https://github.com/appwrite/appwrite/pull/7140) +* fix: patch script, make errors silent [#7134](https://github.com/appwrite/appwrite/pull/7134) +* fix: repositories recreation script [#7133](https://github.com/appwrite/appwrite/pull/7133) +* fix: Only delete repositories linked to the particular project [#7131](https://github.com/appwrite/appwrite/pull/7131) # Version 1.4.11 From fccbe405db1b39e4b580b9a9759c58c3761656d8 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 16 Nov 2023 23:04:38 +0530 Subject: [PATCH 36/67] Add chunkId to abuse key to prevent rate limit for SDKs --- app/controllers/api/storage.php | 2 +- app/controllers/shared/api.php | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f22249e5d2..1fae48dae0 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -339,7 +339,7 @@ App::post('/v1/storage/buckets/:bucketId/files') ->label('audits.resource', 'file/{response.$id}') ->label('usage.metric', 'files.{scope}.requests.create') ->label('usage.params', ['bucketId:{request.bucketId}']) - ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId},chunkId:{chunkId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index b37d76a816..4ea122e3c5 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -121,13 +121,16 @@ App::init() $abuseKeyLabel = (!is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; foreach ($abuseKeyLabel as $abuseKey) { + $start = $request->getContentRangeStart(); + $end = $request->getContentRangeEnd(); $timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject); $timeLimit ->setParam('{userId}', $user->getId()) ->setParam('{userAgent}', $request->getUserAgent('')) ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getHostname() . $route->getPath()) - ->setParam('{method}', $request->getMethod()); + ->setParam('{method}', $request->getMethod()) + ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start))); $timeLimitArray[] = $timeLimit; } From c34dc4dae0c9a7d3fa1c3123080fe748573b8812 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 16 Nov 2023 23:51:09 +0530 Subject: [PATCH 37/67] Add flag to send logs to logger --- app/config/errors.php | 9 +++++++++ app/controllers/api/health.php | 20 ++++++++++---------- app/controllers/general.php | 7 ++++++- src/Appwrite/Extend/Exception.php | 21 ++++++++++++++++++--- 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index d18c7c2dc4..961cc3a197 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -209,6 +209,7 @@ return [ 'name' => Exception::USER_AUTH_METHOD_UNSUPPORTED, 'description' => 'The requested authentication method is either disabled or unsupported. Please check the supported authentication methods in the Appwrite console.', 'code' => 501, + 'log' => false, ], Exception::USER_PHONE_ALREADY_EXISTS => [ 'name' => Exception::USER_PHONE_ALREADY_EXISTS, @@ -764,4 +765,12 @@ return [ 'description' => 'An error occurred on the provider\'s side. Please try again later.', 'code' => 400, ], + + /** Health */ + Exception::QUEUE_SIZE_EXCEEDS => [ + 'name' => Exception::QUEUE_SIZE_EXCEEDS, + 'description' => 'Queue size threshold hit.', + 'code' => 503, + 'log' => false + ], ]; diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 90e080d5fa..4db87befc2 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -355,7 +355,7 @@ App::get('/v1/health/queue/webhooks') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -382,7 +382,7 @@ App::get('/v1/health/queue/logs') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -409,7 +409,7 @@ App::get('/v1/health/queue/certificates') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -436,7 +436,7 @@ App::get('/v1/health/queue/builds') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -464,7 +464,7 @@ App::get('/v1/health/queue/databases') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -491,7 +491,7 @@ App::get('/v1/health/queue/deletes') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -518,7 +518,7 @@ App::get('/v1/health/queue/mails') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -545,7 +545,7 @@ App::get('/v1/health/queue/messaging') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -572,7 +572,7 @@ App::get('/v1/health/queue/migrations') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -599,7 +599,7 @@ App::get('/v1/health/queue/functions') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); diff --git a/app/controllers/general.php b/app/controllers/general.php index cf383b6710..a7f3223031 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -608,9 +608,14 @@ App::error() $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); + $publishLog = true; + + if ($error instanceof AppwriteException) { + $publishLog = $error->getLog(); + } if ($logger) { - if ($error->getCode() >= 500 || $error->getCode() === 0) { + if (($error->getCode() >= 500 || $error->getCode() === 0) && $publishLog) { try { /** @var Utopia\Database\Document $user */ $user = $utopia->getResource('user'); diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 5727a0ed11..a01705f15e 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -229,12 +229,16 @@ class Exception extends \Exception public const MIGRATION_PROVIDER_ERROR = 'migration_provider_error'; /** Realtime */ - public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid'; - public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages'; - public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation'; + public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid'; + public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages'; + public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation'; + + /** Health */ + public const QUEUE_SIZE_EXCEEDS = 'queue_size_exceeds'; protected string $type = ''; protected array $errors = []; + protected bool $log = true; public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null) { @@ -244,6 +248,7 @@ class Exception extends \Exception if (isset($this->errors[$type])) { $this->code = $this->errors[$type]['code']; $this->message = $this->errors[$type]['description']; + $this->log = $this->errors[$type]['log'] ?? true; } $this->message = $message ?? $this->message; @@ -273,4 +278,14 @@ class Exception extends \Exception { $this->type = $type; } + + /** + * Get the log of the exception. + * + * @return bool + */ + public function getLog(): bool + { + return $this->log; + } } From dceb1858b0698ffc3ae7291012ad615257ffbbc7 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 16 Nov 2023 20:31:53 +0200 Subject: [PATCH 38/67] comparison fix --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 5f0ecbe1db..26ada31416 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -90,7 +90,7 @@ class DeleteOrphanedProjects extends Action /** * +2 = audit+abuse */ - if ($collectionsCreated === (count($collectionsConfig) + 2)) { + if ($collectionsCreated >= (count($collectionsConfig) + 2)) { Console::log($msg . ' ignoring....'); continue; } From 8c9a662e1427d5beedd0ff63338c87bcdef3a4f4 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 16 Nov 2023 20:33:37 +0200 Subject: [PATCH 39/67] comparison fix --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 26ada31416..753240b66a 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -54,7 +54,7 @@ class DeleteOrphanedProjects extends Action $totalProjects = $dbForConsole->count('projects'); Console::success("Found a total of: {$totalProjects} projects"); - $orphans = 1; + $orphans = 0; $cnt = 0; $count = 0; $limit = 30; @@ -135,6 +135,6 @@ class DeleteOrphanedProjects extends Action $count = $count + $sum; } - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans'); } } From 881b92eeb810b0d54e223b7cfcf9085e4f486645 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 17 Nov 2023 00:19:43 +0530 Subject: [PATCH 40/67] Updated error codes in health tests --- .../Health/HealthCustomServerTest.php | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 8fa9faadd2..3ea1b884a7 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -145,7 +145,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -171,7 +171,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -197,7 +197,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -223,7 +223,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -249,7 +249,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -280,7 +280,7 @@ class HealthCustomServerTest extends Scope 'name' => 'database_db_main', 'threshold' => '0' ]); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -306,7 +306,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -332,7 +332,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -358,7 +358,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -384,7 +384,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } From 174449ce49b5557aaeecd11156110fa7cf185070 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 16 Nov 2023 23:14:26 +0200 Subject: [PATCH 41/67] comparison fix --- .../Platform/Tasks/DeleteOrphanedProjects.php | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 753240b66a..648a91dc57 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Tasks; use Utopia\App; use Utopia\Config\Config; +use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\Cache\Cache; @@ -45,6 +46,17 @@ class DeleteOrphanedProjects extends Action /** @var array $collections */ $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; + $collectionsConfig = array_merge([ + 'audit' => [ + '$id' => ID::custom('audit'), + '$collection' => Database::METADATA + ], + 'abuse' => [ + '$id' => ID::custom('abuse'), + '$collection' => Database::METADATA + ] + ], $collectionsConfig); + /* Initialise new Utopia app */ $app = new App('UTC'); $console = $app->getResource('console'); @@ -54,7 +66,7 @@ class DeleteOrphanedProjects extends Action $totalProjects = $dbForConsole->count('projects'); Console::success("Found a total of: {$totalProjects} projects"); - $orphans = 0; + $orphans = 1; $cnt = 0; $count = 0; $limit = 30; @@ -80,6 +92,7 @@ class DeleteOrphanedProjects extends Action $dbForProject = new Database($adapter, $cache); $dbForProject->setDefaultDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); + $collectionsCreated = 0; $cnt++; if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { @@ -87,10 +100,8 @@ class DeleteOrphanedProjects extends Action } $msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; - /** - * +2 = audit+abuse - */ - if ($collectionsCreated >= (count($collectionsConfig) + 2)) { + + if ($collectionsCreated >= count($collectionsConfig)) { Console::log($msg . ' ignoring....'); continue; } @@ -107,6 +118,7 @@ class DeleteOrphanedProjects extends Action Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')'); } } + if ($commit) { $dbForConsole->deleteDocument('projects', $project->getId()); $dbForConsole->deleteCachedDocument('projects', $project->getId()); @@ -116,7 +128,7 @@ class DeleteOrphanedProjects extends Action $orphans++; } catch (\Throwable $th) { - Console::error('Error: ' . $th->getMessage()); + Console::error('Error: ' . $th->getMessage() . ' ' . $th->getTraceAsString()); } finally { $pools ->get($db) @@ -135,6 +147,6 @@ class DeleteOrphanedProjects extends Action $count = $count + $sum; } - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans'); + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); } -} +} \ No newline at end of file From e079d4a8b5cedb054b939206a0171edf9a40a14c Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 16 Nov 2023 23:42:03 +0200 Subject: [PATCH 42/67] comparison fix --- .../Platform/Tasks/DeleteOrphanedProjects.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 648a91dc57..757b29c1b6 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -122,6 +122,15 @@ class DeleteOrphanedProjects extends Action if ($commit) { $dbForConsole->deleteDocument('projects', $project->getId()); $dbForConsole->deleteCachedDocument('projects', $project->getId()); + + if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { + try { + $dbForProject->deleteCollection(Database::METADATA); + $dbForProject->deleteCachedCollection(Database::METADATA); + } catch (\Throwable $th) { + Console::warning('Metadata collection does not exist'); + } + } } Console::info('--Deleting project no (' . $project->getInternalId() . ')'); @@ -149,4 +158,4 @@ class DeleteOrphanedProjects extends Action Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); } -} \ No newline at end of file +} From 40ec98569e74288c72ab5e31b1d91b0c97432f2d Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 17 Nov 2023 18:35:26 +0530 Subject: [PATCH 43/67] Update app/controllers/general.php Co-authored-by: Christy Jacob --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index a7f3223031..f086ccd26f 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -614,7 +614,7 @@ App::error() $publishLog = $error->getLog(); } - if ($logger) { + if ($logger && publish) { if (($error->getCode() >= 500 || $error->getCode() === 0) && $publishLog) { try { /** @var Utopia\Database\Document $user */ From 93292879821e52b4aa25c8687fc1832fcc62958d Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 17 Nov 2023 18:35:51 +0530 Subject: [PATCH 44/67] Update app/controllers/general.php Co-authored-by: Christy Jacob --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index f086ccd26f..ae6a003a17 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -608,7 +608,7 @@ App::error() $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); - $publishLog = true; + $publish = true; if ($error instanceof AppwriteException) { $publishLog = $error->getLog(); From cf1c914edb901454f4eeb0336baa967b7cffb136 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 17 Nov 2023 18:36:03 +0530 Subject: [PATCH 45/67] Update app/controllers/general.php Co-authored-by: Christy Jacob --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index ae6a003a17..bda54eda92 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -615,7 +615,7 @@ App::error() } if ($logger && publish) { - if (($error->getCode() >= 500 || $error->getCode() === 0) && $publishLog) { + if ($error->getCode() >= 500 || $error->getCode() === 0) { try { /** @var Utopia\Database\Document $user */ $user = $utopia->getResource('user'); From 827ad84f532b2cda4d87b7f0271bd6583741b08c Mon Sep 17 00:00:00 2001 From: Prateek Banga Date: Fri, 17 Nov 2023 19:09:31 +0530 Subject: [PATCH 46/67] fixes adding longer length enum --- app/controllers/api/databases.php | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index f47e3f8265..6a8901a509 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -1240,16 +1240,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') ->inject('queueForDatabase') ->inject('queueForEvents') ->action(function (string $databaseId, string $collectionId, string $key, array $elements, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { - // use length of longest string as attribute size - $size = 0; - foreach ($elements as $element) { - $length = \strlen($element); - if ($length === 0) { - throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Each enum element must not be empty'); - } - $size = ($length > $size) ? $length : $size; - } - if (!is_null($default) && !in_array($default, $elements)) { throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Default value not found in elements'); } @@ -1257,7 +1247,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') $attribute = createAttribute($databaseId, $collectionId, new Document([ 'key' => $key, 'type' => Database::VAR_STRING, - 'size' => $size, + 'size' => Database::LENGTH_KEY, 'required' => $required, 'default' => $default, 'array' => $array, From 7902defe69383d04a6724b3fd76b2e70de8a90d1 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 17 Nov 2023 19:29:57 +0530 Subject: [PATCH 47/67] Addressed PR comments --- app/config/errors.php | 8 ++++---- app/controllers/api/health.php | 20 ++++++++++---------- app/controllers/general.php | 4 ++-- src/Appwrite/Extend/Exception.php | 12 ++++++------ 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index 961cc3a197..bbedf0f24b 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -209,7 +209,7 @@ return [ 'name' => Exception::USER_AUTH_METHOD_UNSUPPORTED, 'description' => 'The requested authentication method is either disabled or unsupported. Please check the supported authentication methods in the Appwrite console.', 'code' => 501, - 'log' => false, + 'publishLog' => false, ], Exception::USER_PHONE_ALREADY_EXISTS => [ 'name' => Exception::USER_PHONE_ALREADY_EXISTS, @@ -767,10 +767,10 @@ return [ ], /** Health */ - Exception::QUEUE_SIZE_EXCEEDS => [ - 'name' => Exception::QUEUE_SIZE_EXCEEDS, + Exception::QUEUE_SIZE_EXCEEDED => [ + 'name' => Exception::QUEUE_SIZE_EXCEEDED, 'description' => 'Queue size threshold hit.', 'code' => 503, - 'log' => false + 'publishLog' => false ], ]; diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 4db87befc2..4f459dd862 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -355,7 +355,7 @@ App::get('/v1/health/queue/webhooks') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -382,7 +382,7 @@ App::get('/v1/health/queue/logs') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -409,7 +409,7 @@ App::get('/v1/health/queue/certificates') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -436,7 +436,7 @@ App::get('/v1/health/queue/builds') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -464,7 +464,7 @@ App::get('/v1/health/queue/databases') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -491,7 +491,7 @@ App::get('/v1/health/queue/deletes') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -518,7 +518,7 @@ App::get('/v1/health/queue/mails') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -545,7 +545,7 @@ App::get('/v1/health/queue/messaging') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -572,7 +572,7 @@ App::get('/v1/health/queue/migrations') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -599,7 +599,7 @@ App::get('/v1/health/queue/functions') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::QUEUE_SIZE_EXCEEDS, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); diff --git a/app/controllers/general.php b/app/controllers/general.php index bda54eda92..72138acea0 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -611,10 +611,10 @@ App::error() $publish = true; if ($error instanceof AppwriteException) { - $publishLog = $error->getLog(); + $publish = $error->getPublishLog(); } - if ($logger && publish) { + if ($logger && $publish) { if ($error->getCode() >= 500 || $error->getCode() === 0) { try { /** @var Utopia\Database\Document $user */ diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index a01705f15e..c63555262a 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -234,11 +234,11 @@ class Exception extends \Exception public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation'; /** Health */ - public const QUEUE_SIZE_EXCEEDS = 'queue_size_exceeds'; + public const QUEUE_SIZE_EXCEEDED = 'queue_size_exceeded'; protected string $type = ''; protected array $errors = []; - protected bool $log = true; + protected bool $publishLog = true; public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null) { @@ -248,7 +248,7 @@ class Exception extends \Exception if (isset($this->errors[$type])) { $this->code = $this->errors[$type]['code']; $this->message = $this->errors[$type]['description']; - $this->log = $this->errors[$type]['log'] ?? true; + $this->publishLog = $this->errors[$type]['publishLog'] ?? true; } $this->message = $message ?? $this->message; @@ -280,12 +280,12 @@ class Exception extends \Exception } /** - * Get the log of the exception. + * Get the value of publishLog for the exception. * * @return bool */ - public function getLog(): bool + public function getPublishLog(): bool { - return $this->log; + return $this->publishLog; } } From 228b813994aadcdd8261791d90ccaa5f8b5b99b9 Mon Sep 17 00:00:00 2001 From: Prateek Banga Date: Fri, 17 Nov 2023 20:06:28 +0530 Subject: [PATCH 48/67] fixes test case --- app/controllers/api/databases.php | 2 +- tests/e2e/Services/Databases/DatabasesBase.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 6a8901a509..f669527e21 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -1231,7 +1231,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', '', new Key(), 'Attribute Key.') - ->param('elements', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE, min: 0), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.') + ->param('elements', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE, min: 1), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.') ->param('required', null, new Boolean(), 'Is attribute required?') ->param('default', null, new Text(0), 'Default value for attribute when not provided. Cannot be set when attribute is required.', true) ->param('array', false, new Boolean(), 'Is attribute an array?', true) diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 34a433353c..d21fa28051 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -978,7 +978,7 @@ trait DatabasesBase ]); $this->assertEquals(400, $badEnum['headers']['status-code']); - $this->assertEquals('Each enum element must not be empty', $badEnum['body']['message']); + $this->assertEquals('Invalid `elements` param: Value must a valid array and Value must be a valid string and at least 1 chars and no longer than 4096 chars', $badEnum['body']['message']); return $data; } From ab35b75c37db4e30cc625bd094385c8f2cb7b428 Mon Sep 17 00:00:00 2001 From: Prateek Banga Date: Fri, 17 Nov 2023 20:14:55 +0530 Subject: [PATCH 49/67] fix test message --- app/controllers/api/databases.php | 2 +- tests/e2e/Services/Databases/DatabasesBase.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index f669527e21..c29af01f8a 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -1231,7 +1231,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', '', new Key(), 'Attribute Key.') - ->param('elements', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE, min: 1), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.') + ->param('elements', [], new ArrayList(new Text(DATABASE::LENGTH_KEY, min: 1), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.') ->param('required', null, new Boolean(), 'Is attribute required?') ->param('default', null, new Text(0), 'Default value for attribute when not provided. Cannot be set when attribute is required.', true) ->param('array', false, new Boolean(), 'Is attribute an array?', true) diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index d21fa28051..f86b817777 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -978,7 +978,7 @@ trait DatabasesBase ]); $this->assertEquals(400, $badEnum['headers']['status-code']); - $this->assertEquals('Invalid `elements` param: Value must a valid array and Value must be a valid string and at least 1 chars and no longer than 4096 chars', $badEnum['body']['message']); + $this->assertEquals('Invalid `elements` param: Value must a valid array and Value must be a valid string and at least 1 chars and no longer than 255 chars', $badEnum['body']['message']); return $data; } From 9183f2a875cdbd9753f9a2a1b5f4d8861a4e24e2 Mon Sep 17 00:00:00 2001 From: Prateek Banga Date: Fri, 17 Nov 2023 20:56:18 +0530 Subject: [PATCH 50/67] changes enum size validation in update controller --- app/controllers/api/databases.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index c29af01f8a..8df9854c87 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -1231,7 +1231,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', '', new Key(), 'Attribute Key.') - ->param('elements', [], new ArrayList(new Text(DATABASE::LENGTH_KEY, min: 1), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.') + ->param('elements', [], new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.') ->param('required', null, new Boolean(), 'Is attribute required?') ->param('default', null, new Text(0), 'Default value for attribute when not provided. Cannot be set when attribute is required.', true) ->param('array', false, new Boolean(), 'Is attribute an array?', true) @@ -1920,7 +1920,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/ ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', '', new Key(), 'Attribute Key.') - ->param('elements', null, new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.') + ->param('elements', null, new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.') ->param('required', null, new Boolean(), 'Is attribute required?') ->param('default', null, new Nullable(new Text(0)), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->inject('response') From b68be491b9318efb8a39036f467e72e5142cbe91 Mon Sep 17 00:00:00 2001 From: Prateek Banga Date: Fri, 17 Nov 2023 20:59:09 +0530 Subject: [PATCH 51/67] updated description --- app/controllers/api/databases.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 8df9854c87..0d2e2f22d1 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -1231,7 +1231,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', '', new Key(), 'Attribute Key.') - ->param('elements', [], new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.') + ->param('elements', [], new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . DATABASE::LENGTH_KEY . ' characters long.') ->param('required', null, new Boolean(), 'Is attribute required?') ->param('default', null, new Text(0), 'Default value for attribute when not provided. Cannot be set when attribute is required.', true) ->param('array', false, new Boolean(), 'Is attribute an array?', true) @@ -1920,7 +1920,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/ ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', '', new Key(), 'Attribute Key.') - ->param('elements', null, new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.') + ->param('elements', null, new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . DATABASE::LENGTH_KEY . ' characters long.') ->param('required', null, new Boolean(), 'Is attribute required?') ->param('default', null, new Nullable(new Text(0)), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->inject('response') From 695d6f4065f412719910b1a5c9b657f99a75011c Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Sat, 18 Nov 2023 00:04:57 +0530 Subject: [PATCH 52/67] Renamed variables --- app/config/errors.php | 4 ++-- app/controllers/general.php | 2 +- src/Appwrite/Extend/Exception.php | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index bbedf0f24b..c0628920d9 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -209,7 +209,7 @@ return [ 'name' => Exception::USER_AUTH_METHOD_UNSUPPORTED, 'description' => 'The requested authentication method is either disabled or unsupported. Please check the supported authentication methods in the Appwrite console.', 'code' => 501, - 'publishLog' => false, + 'publish' => false, ], Exception::USER_PHONE_ALREADY_EXISTS => [ 'name' => Exception::USER_PHONE_ALREADY_EXISTS, @@ -771,6 +771,6 @@ return [ 'name' => Exception::QUEUE_SIZE_EXCEEDED, 'description' => 'Queue size threshold hit.', 'code' => 503, - 'publishLog' => false + 'publish' => false ], ]; diff --git a/app/controllers/general.php b/app/controllers/general.php index 72138acea0..e443b96fc9 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -611,7 +611,7 @@ App::error() $publish = true; if ($error instanceof AppwriteException) { - $publish = $error->getPublishLog(); + $publish = $error->isPublishable(); } if ($logger && $publish) { diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index c63555262a..6449ffd93a 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -238,7 +238,7 @@ class Exception extends \Exception protected string $type = ''; protected array $errors = []; - protected bool $publishLog = true; + protected bool $publish = true; public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null) { @@ -248,7 +248,7 @@ class Exception extends \Exception if (isset($this->errors[$type])) { $this->code = $this->errors[$type]['code']; $this->message = $this->errors[$type]['description']; - $this->publishLog = $this->errors[$type]['publishLog'] ?? true; + $this->publish = $this->errors[$type]['publish'] ?? true; } $this->message = $message ?? $this->message; @@ -280,12 +280,12 @@ class Exception extends \Exception } /** - * Get the value of publishLog for the exception. + * Check whether the log is publishable for the exception. * * @return bool */ - public function getPublishLog(): bool + public function isPublishable(): bool { - return $this->publishLog; + return $this->publish; } } From 926e1f3fc8eebef2ea3f8fb61b816f7b14e0ff66 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Fri, 17 Nov 2023 19:34:33 +0000 Subject: [PATCH 53/67] Bump console to version 3.2.8 --- .gitmodules | 2 +- app/console | 2 +- app/init.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index af12124355..c939b9ee2c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = 3.2.7 + branch = 3.2.8 diff --git a/app/console b/app/console index 49d039ed07..6bf696aeb3 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 49d039ed07628155e7f56e2c997fcef90ecde267 +Subproject commit 6bf696aeb329d23e4ebe9d1e71edbd144cf2d58e diff --git a/app/init.php b/app/init.php index 2c0219eec2..f57e3640b2 100644 --- a/app/init.php +++ b/app/init.php @@ -109,7 +109,7 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 327; +const APP_CACHE_BUSTER = 328; const APP_VERSION_STABLE = '1.4.12'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; From a96e571258cdbadcc5be4d1c8512ebb171b9a7b6 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Fri, 17 Nov 2023 19:51:25 +0000 Subject: [PATCH 54/67] Bump appwrite version to 1.4.13 --- README-CN.md | 6 +++--- README.md | 6 +++--- app/init.php | 2 +- src/Appwrite/Migration/Migration.php | 3 ++- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/README-CN.md b/README-CN.md index 8e4695e608..0ce27a0038 100644 --- a/README-CN.md +++ b/README-CN.md @@ -66,7 +66,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.4.12 + appwrite/appwrite:1.4.13 ``` ### Windows @@ -78,7 +78,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.4.12 + appwrite/appwrite:1.4.13 ``` #### PowerShell @@ -88,7 +88,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.4.12 + appwrite/appwrite:1.4.13 ``` 运行后,可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后,服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index 3a1d2dbe9f..62b084d320 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.4.12 + appwrite/appwrite:1.4.13 ``` ### Windows @@ -88,7 +88,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.4.12 + appwrite/appwrite:1.4.13 ``` #### PowerShell @@ -98,7 +98,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.4.12 + appwrite/appwrite:1.4.13 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/init.php b/app/init.php index f57e3640b2..4e8e8103c9 100644 --- a/app/init.php +++ b/app/init.php @@ -110,7 +110,7 @@ const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 328; -const APP_VERSION_STABLE = '1.4.12'; +const APP_VERSION_STABLE = '1.4.13'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 8f68e31be4..0a2dd2f186 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -76,7 +76,8 @@ abstract class Migration '1.4.9' => 'V19', '1.4.10' => 'V19', '1.4.11' => 'V19', - '1.4.12' => 'V19' + '1.4.12' => 'V19', + '1.4.13' => 'V19' ]; /** From 37e1e3adafffa5024b6c1ab1bd89207f29827178 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Fri, 17 Nov 2023 19:59:58 +0000 Subject: [PATCH 55/67] Add 1.4.13 to CHANGES.md --- CHANGES.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 889f65e1e7..83e09896c8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,21 @@ +# Version 1.4.13 + +## Notable changes + +* Change enum size validation in update controller [#7164](https://github.com/appwrite/appwrite/pull/7164) +* Bump console to version 3.2.8 in [#7167](https://github.com/appwrite/appwrite/pull/7167) + +## Bug fixes + +* Fix error after adding bigger enum [#7162](https://github.com/appwrite/appwrite/pull/7162) +* Add chunkId to abuse key to prevent rate limit for SDKs [#7154](https://github.com/appwrite/appwrite/pull/7154) + +## Miscellaneous + +* Fix enum test case [#7163](https://github.com/appwrite/appwrite/pull/7163) +* Add flag to send logs to logger [#7155](https://github.com/appwrite/appwrite/pull/7155) +* Add a CI task to validate composer file and lock [#7142](https://github.com/appwrite/appwrite/pull/7142) + # Version 1.4.12 ## Miscellaneous From b2381798ab2d8452ac6798a5cbfe853eabe42866 Mon Sep 17 00:00:00 2001 From: prateek banga Date: Mon, 20 Nov 2023 22:01:20 +0530 Subject: [PATCH 56/67] migrates enum attribute size to 255 --- app/init.php | 19 +++++++++++++++++++ src/Appwrite/Migration/Version/V19.php | 21 +++++++++++++++++++++ src/Appwrite/Platform/Tasks/Migrate.php | 8 ++++++-- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index 4e8e8103c9..4cbf08cf2f 100644 --- a/app/init.php +++ b/app/init.php @@ -767,6 +767,25 @@ $register->set('pools', function () { return $group; }); +$register->set('db', function () { + // This is usually for our workers or CLI commands scope + $dbHost = App::getEnv('_APP_DB_HOST', ''); + $dbPort = App::getEnv('_APP_DB_PORT', ''); + $dbUser = App::getEnv('_APP_DB_USER', ''); + $dbPass = App::getEnv('_APP_DB_PASS', ''); + $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); + + $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true, + )); + + return $pdo; +}); $register->set('influxdb', function () { // Register DB connection diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index b9ccf3c302..10a64d4b03 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -10,6 +10,7 @@ use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception; +use Utopia\Database\Query; class V19 extends Migration { @@ -41,6 +42,11 @@ class V19 extends Migration Console::info('Migrating Buckets'); $this->migrateBuckets(); + if ($this->project->getId() !== 'console') { + Console::info('Migrating Enum Attribute Size'); + $this->migrateEnumAttributeSize(); + } + Console::info('Migrating Documents'); $this->forEachDocument([$this, 'fixDocument']); @@ -640,6 +646,21 @@ class V19 extends Migration return $commands; } + private function migrateEnumAttributeSize(): void + { + \Co\run(function () { + foreach ($this->documentsIterator('attributes') as $attribute) { + go(function ($attribute) { + $attribute->setAttribute('size', Database::LENGTH_KEY); + $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); + $databaseInternalId = $attribute->getAttribute('databaseInternalId'); + $collectionInternalId = $attribute->getAttribute('collectionInternalId'); + $this->changeAttributeInternalType('database_' . $databaseInternalId . '_collection_' . $collectionInternalId, $attribute->getAttribute('key'), 'varchar(255)'); + }, $attribute); + } + }); + } + /** * Fix run on each document * diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 38e5c8aa46..682b9b5559 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -11,6 +11,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +use Utopia\Registry\Registry; use Utopia\Validator\Text; class Migrate extends Action @@ -29,7 +30,8 @@ class Migrate extends Action ->inject('cache') ->inject('dbForConsole') ->inject('getProjectDB') - ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB)); + ->inject('register') + ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $cache, $dbForConsole, $getProjectDB, $register)); } private function clearProjectsCache(Cache $cache, Document $project) @@ -41,7 +43,7 @@ class Migrate extends Action } } - public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB) + public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register) { Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { @@ -89,9 +91,11 @@ class Migrate extends Action try { // TODO: Iterate through all project DBs + /** @var Database $projectDB */ $projectDB = $getProjectDB($project); $migration ->setProject($project, $projectDB, $dbForConsole) + ->setPDO($register->get('db', true)) ->execute(); } catch (\Throwable $th) { Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); From da3798e52f80edbeec13f4add596087e7511c1ed Mon Sep 17 00:00:00 2001 From: prateek banga Date: Mon, 20 Nov 2023 23:31:34 +0530 Subject: [PATCH 57/67] review changes --- src/Appwrite/Migration/Version/V19.php | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 10a64d4b03..6f2cbfac49 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -648,17 +648,17 @@ class V19 extends Migration private function migrateEnumAttributeSize(): void { - \Co\run(function () { - foreach ($this->documentsIterator('attributes') as $attribute) { - go(function ($attribute) { - $attribute->setAttribute('size', Database::LENGTH_KEY); - $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); - $databaseInternalId = $attribute->getAttribute('databaseInternalId'); - $collectionInternalId = $attribute->getAttribute('collectionInternalId'); - $this->changeAttributeInternalType('database_' . $databaseInternalId . '_collection_' . $collectionInternalId, $attribute->getAttribute('key'), 'varchar(255)'); - }, $attribute); + foreach ($this->documentsIterator('attributes') as $attribute) { + if ($attribute->getAttribute('format') !== 'enum' || $attribute->getAttribute('size') >= Database::LENGTH_KEY) { + return; } - }); + + $attribute->setAttribute('size', Database::LENGTH_KEY); + $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); + $databaseInternalId = $attribute->getAttribute('databaseInternalId'); + $collectionInternalId = $attribute->getAttribute('collectionInternalId'); + $this->projectDB->updateAttribute('database_' . $databaseInternalId . '_collection_' . $collectionInternalId, $attribute->getAttribute('key'), size: 255); + } } /** From 9a12ab8f8a322fd91162ac8eda4c5c0a302c8c06 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 20 Nov 2023 19:36:52 -0500 Subject: [PATCH 58/67] feat: add migration stats task --- Dockerfile | 3 +- bin/get-migration-stats | 3 + src/Appwrite/Platform/Services/Tasks.php | 2 + src/Appwrite/Platform/Tasks/CalcTierStats.php | 2 +- .../Platform/Tasks/GetMigrationStats.php | 184 ++++++++++++++++++ 5 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 bin/get-migration-stats create mode 100644 src/Appwrite/Platform/Tasks/GetMigrationStats.php diff --git a/Dockerfile b/Dockerfile index 059c499bd9..8721301269 100755 --- a/Dockerfile +++ b/Dockerfile @@ -105,7 +105,8 @@ RUN chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/delete-orphaned-projects && \ chmod +x /usr/local/bin/clear-card-cache && \ chmod +x /usr/local/bin/calc-users-stats && \ - chmod +x /usr/local/bin/calc-tier-stats + chmod +x /usr/local/bin/calc-tier-stats && \ + chmod +x /usr/local/bin/get-migration-stats # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/bin/get-migration-stats b/bin/get-migration-stats new file mode 100644 index 0000000000..efc1934e15 --- /dev/null +++ b/bin/get-migration-stats @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php get-migration-stats $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 28d7046dd1..dc6ddc1a5b 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -19,6 +19,7 @@ use Appwrite\Platform\Tasks\VolumeSync; use Appwrite\Platform\Tasks\CalcTierStats; use Appwrite\Platform\Tasks\Upgrade; use Appwrite\Platform\Tasks\DeleteOrphanedProjects; +use Appwrite\Platform\Tasks\GetMigrationStats; use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments; class Tasks extends Service @@ -44,6 +45,7 @@ class Tasks extends Service ->addAction(CalcTierStats::getName(), new CalcTierStats()) ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects()) ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments()) + ->addAction(GetMigrationStats::getName(), new GetMigrationStats()) ; } diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index 2a2bc20af9..c579dd2627 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -200,7 +200,7 @@ class CalcTierStats extends Action try { /** Get Domains */ - $stats['Domains'] = $dbForConsole->count('domains', [ + $stats['Domains'] = $dbForConsole->count('rules', [ Query::equal('projectInternalId', [$project->getInternalId()]), ]); } catch (\Throwable) { diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php new file mode 100644 index 0000000000..bd7885c7ce --- /dev/null +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -0,0 +1,184 @@ +desc('Get stats for projects') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->inject('register') + ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($pools, $cache, $dbForConsole, $register); + }); + } + + /** + * @throws \Utopia\Exception + * @throws CannotInsertRecord + */ + public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + { + //docker compose exec -t appwrite get-migration-stats + + Console::title('Migration stats calculation V1'); + Console::success(APP_NAME . ' Migration stats calculation has started'); + + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + + /** CSV stuff */ + $this->date = date('Y-m-d'); + $this->path = "{$this->directory}/migration_stats_{$this->date}.csv"; + $csv = Writer::createFromPath($this->path, 'w'); + $csv->insertOne($this->columns); + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects'); + Console::success("Found a total of: {$totalProjects} projects"); + + $projects = [$console]; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + Console::info("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + /** Get Project ID */ + $stats['Project ID'] = $project->getId(); + + /** Get Migration details */ + $migrations = $dbForProject->find('migrations', [ + Query::limit(500) + ]); + + $migrations = array_map(function ($migration) use ($project) { + $result['Project ID'] = $project->getId(); + $result['$id'] = $migration->getAttribute('$id'); + $result['$createdAt'] = $migration->getAttribute('$createdAt'); + $result['status'] = $migration->getAttribute('status'); + $result['stage'] = $migration->getAttribute('stage'); + $result['source'] = $migration->getAttribute('source'); + return array_values($result); + }, $migrations); + + $csv->insertAll($migrations); + } catch (\Throwable $th) { + Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + } + + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); + + $pools + ->get('console') + ->reclaim(); + + /** @var PHPMailer $mail */ + $mail = $register->get('smtp'); + + $mail->clearAddresses(); + $mail->clearAllRecipients(); + $mail->clearReplyTos(); + $mail->clearAttachments(); + $mail->clearBCCs(); + $mail->clearCCs(); + + try { + /** Addresses */ + $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); + $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); + + foreach ($recipients as $recipient) { + $mail->addAddress($recipient); + } + + /** Attachments */ + $mail->addAttachment($this->path); + + /** Content */ + $mail->Subject = "Migration Report for {$this->date}"; + $mail->Body = "Please find the migration report atttached"; + $mail->send(); + Console::success('Email has been sent!'); + } catch (Exception $e) { + Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); + } + } +} From 28c61acec68a4960c553e27611e87da0a7abc2b1 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 20 Nov 2023 19:39:07 -0500 Subject: [PATCH 59/67] feat: update migration task --- src/Appwrite/Platform/Tasks/CalcTierStats.php | 4 ++-- src/Appwrite/Platform/Tasks/GetMigrationStats.php | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index c579dd2627..ef900aab34 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -99,8 +99,8 @@ class CalcTierStats extends Action $projects = [$console]; $count = 0; - $limit = 30; - $sum = 30; + $limit = 100; + $sum = 100; $offset = 0; while (!empty($projects)) { foreach ($projects as $project) { diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php index bd7885c7ce..3ef4f03d25 100644 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -79,8 +79,8 @@ class GetMigrationStats extends Action $projects = [$console]; $count = 0; - $limit = 30; - $sum = 30; + $limit = 100; + $sum = 100; $offset = 0; while (!empty($projects)) { foreach ($projects as $project) { @@ -123,7 +123,9 @@ class GetMigrationStats extends Action return array_values($result); }, $migrations); - $csv->insertAll($migrations); + if(!empty($migrations)) { + $csv->insertAll($migrations); + } } catch (\Throwable $th) { Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); } finally { From f81d6756097eea3fbf3c982dbdcdfca46b570d23 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 20 Nov 2023 20:10:24 -0500 Subject: [PATCH 60/67] feat: run linter --- src/Appwrite/Platform/Tasks/GetMigrationStats.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php index 3ef4f03d25..6fddf2d7e0 100644 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -123,7 +123,7 @@ class GetMigrationStats extends Action return array_values($result); }, $migrations); - if(!empty($migrations)) { + if (!empty($migrations)) { $csv->insertAll($migrations); } } catch (\Throwable $th) { From 9d31d8d950c972a86c797f43ae2261dbd0512902 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 20 Nov 2023 20:10:56 -0500 Subject: [PATCH 61/67] Update src/Appwrite/Platform/Tasks/GetMigrationStats.php Co-authored-by: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> --- src/Appwrite/Platform/Tasks/GetMigrationStats.php | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php index 6fddf2d7e0..b76e0428d7 100644 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -114,13 +114,14 @@ class GetMigrationStats extends Action ]); $migrations = array_map(function ($migration) use ($project) { - $result['Project ID'] = $project->getId(); - $result['$id'] = $migration->getAttribute('$id'); - $result['$createdAt'] = $migration->getAttribute('$createdAt'); - $result['status'] = $migration->getAttribute('status'); - $result['stage'] = $migration->getAttribute('stage'); - $result['source'] = $migration->getAttribute('source'); - return array_values($result); + return [ + $project->getId(), + $migration->getAttribute('$id'), + $migration->getAttribute('$createdAt'), + $migration->getAttribute('status'), + $migration->getAttribute('stage'), + $migration->getAttribute('source'), + ]; }, $migrations); if (!empty($migrations)) { From 8eb8351d5ff74cfa71bdb9d021a2ed953f6e8496 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 20 Nov 2023 20:26:38 -0500 Subject: [PATCH 62/67] feat: add migration counts to hamster --- src/Appwrite/Platform/Tasks/CalcTierStats.php | 8 ++++++++ src/Appwrite/Platform/Tasks/Hamster.php | 3 +++ 2 files changed, 11 insertions(+) diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index ef900aab34..e6559a05d7 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -41,6 +41,7 @@ class CalcTierStats extends Action 'Functions', 'Deployments', 'Executions', + 'Migrations', ]; protected string $directory = '/usr/local'; @@ -290,6 +291,13 @@ class CalcTierStats extends Action $stats['Executions'] = 0; } + /** Get Total Migrations */ + try { + $stats['Migrations'] = $dbForProject->count('migrations', []); + } catch (\Throwable) { + $stats['Migrations'] = 0; + } + $csv->insertOne(array_values($stats)); } catch (\Throwable $th) { Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 1d5d3b0b26..2e2c2d466e 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -149,6 +149,9 @@ class Hamster extends Action /** Get Total Teams */ $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); + /** Get Total Migrations */ + $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); + /** Get Total Members */ $teamInternalId = $project->getAttribute('teamInternalId', null); if ($teamInternalId) { From 7e625ba653c0b40fb637f3015af58979aca256ee Mon Sep 17 00:00:00 2001 From: prateek banga Date: Tue, 21 Nov 2023 11:48:27 +0530 Subject: [PATCH 63/67] adds queries support in documentsIterator --- src/Appwrite/Migration/Migration.php | 12 +++++++++--- src/Appwrite/Migration/Version/V19.php | 6 +----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 0a2dd2f186..7f2a362e7d 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -195,17 +195,23 @@ abstract class Migration * @return iterable * @throws \Exception */ - public function documentsIterator(string $collectionId): iterable + public function documentsIterator(string $collectionId, $queries = []): iterable { $sum = 0; $nextDocument = null; $collectionCount = $this->projectDB->count($collectionId); + $queries[] = Query::limit($this->limit); do { - $queries = [Query::limit($this->limit)]; if ($nextDocument !== null) { - $queries[] = Query::cursorAfter($nextDocument); + $lastQuery = $queries[\count($queries) - 1]; + if ($lastQuery->getMethod() === 'cursorAfter') { + $queries[\count($queries) - 1] = Query::cursorAfter($nextDocument); + } else { + $queries[] = Query::cursorAfter($nextDocument); + } } + $documents = $this->projectDB->find($collectionId, $queries); $count = count($documents); $sum += $count; diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 6f2cbfac49..b8c2c54650 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -648,11 +648,7 @@ class V19 extends Migration private function migrateEnumAttributeSize(): void { - foreach ($this->documentsIterator('attributes') as $attribute) { - if ($attribute->getAttribute('format') !== 'enum' || $attribute->getAttribute('size') >= Database::LENGTH_KEY) { - return; - } - + foreach ($this->documentsIterator('attributes', [Query::equal('format', ['enum']), Query::lessThan('size', Database::LENGTH_KEY)]) as $attribute) { $attribute->setAttribute('size', Database::LENGTH_KEY); $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); $databaseInternalId = $attribute->getAttribute('databaseInternalId'); From 022f0a663edd0b870cf677faf2a10041adc6a5e5 Mon Sep 17 00:00:00 2001 From: prateek banga Date: Tue, 21 Nov 2023 12:23:36 +0530 Subject: [PATCH 64/67] changes array search for cursorAfter query --- src/Appwrite/Migration/Migration.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 7f2a362e7d..9ee4d9236a 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -204,9 +204,10 @@ abstract class Migration do { if ($nextDocument !== null) { - $lastQuery = $queries[\count($queries) - 1]; - if ($lastQuery->getMethod() === 'cursorAfter') { - $queries[\count($queries) - 1] = Query::cursorAfter($nextDocument); + $cursorQueryIndex = \array_search('cursorAfter', \array_map(fn (Query $query) => $query->getMethod(), $queries)); + + if ($cursorQueryIndex !== false) { + $queries[$cursorQueryIndex] = Query::cursorAfter($nextDocument); } else { $queries[] = Query::cursorAfter($nextDocument); } From aa6a1a2e31feff9a7702f5456c1869a4e8689ad5 Mon Sep 17 00:00:00 2001 From: Prateek Banga <30731059+fanatic75@users.noreply.github.com> Date: Tue, 21 Nov 2023 12:24:08 +0530 Subject: [PATCH 65/67] Update src/Appwrite/Migration/Version/V19.php Co-authored-by: Jake Barnby --- src/Appwrite/Migration/Version/V19.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index b8c2c54650..2131af4ff8 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -648,7 +648,10 @@ class V19 extends Migration private function migrateEnumAttributeSize(): void { - foreach ($this->documentsIterator('attributes', [Query::equal('format', ['enum']), Query::lessThan('size', Database::LENGTH_KEY)]) as $attribute) { + foreach ($this->documentsIterator('attributes', [ + Query::equal('format', ['enum']), + Query::lessThan('size', Database::LENGTH_KEY) + ]) as $attribute) { $attribute->setAttribute('size', Database::LENGTH_KEY); $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); $databaseInternalId = $attribute->getAttribute('databaseInternalId'); From 1720acb2b40d593ab2fb3642257605c39ba059fd Mon Sep 17 00:00:00 2001 From: prateek banga Date: Tue, 21 Nov 2023 12:32:30 +0530 Subject: [PATCH 66/67] lint fix --- src/Appwrite/Migration/Version/V19.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index 2131af4ff8..46a0a073a6 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -648,10 +648,12 @@ class V19 extends Migration private function migrateEnumAttributeSize(): void { - foreach ($this->documentsIterator('attributes', [ - Query::equal('format', ['enum']), + foreach ( + $this->documentsIterator('attributes', [ + Query::equal('format', ['enum']), Query::lessThan('size', Database::LENGTH_KEY) - ]) as $attribute) { + ]) as $attribute + ) { $attribute->setAttribute('size', Database::LENGTH_KEY); $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); $databaseInternalId = $attribute->getAttribute('databaseInternalId'); From 5179c4d08a5b70e82172c1e9cd34b8f072ef08ca Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 21 Nov 2023 12:08:26 +0100 Subject: [PATCH 67/67] chore: upgrade console to 3.2.9 --- .gitmodules | 2 +- app/console | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index c939b9ee2c..dad7c9ed3e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = 3.2.8 + branch = 3.2.9 diff --git a/app/console b/app/console index 6bf696aeb3..212b742992 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 6bf696aeb329d23e4ebe9d1e71edbd144cf2d58e +Subproject commit 212b7429926d097a31ed71d2410e39c600c56f3b