diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 095449de2f..899c27a135 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -18,6 +18,10 @@ jobs: - run: git checkout HEAD^2 + - name: Validate composer.json and composer.lock + run: | + docker run --rm -v $PWD:/app composer sh -c \ + "composer validate" - name: Run Linter run: | docker run --rm -v $PWD:/app composer sh -c \ diff --git a/.gitmodules b/.gitmodules index ff2e0a6aab..c939b9ee2c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = 3.2.6 + branch = 3.2.8 diff --git a/CHANGES.md b/CHANGES.md index e8f0c08c1f..83e09896c8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,35 @@ +# Version 1.4.13 + +## Notable changes + +* Change enum size validation in update controller [#7164](https://github.com/appwrite/appwrite/pull/7164) +* Bump console to version 3.2.8 in [#7167](https://github.com/appwrite/appwrite/pull/7167) + +## Bug fixes + +* Fix error after adding bigger enum [#7162](https://github.com/appwrite/appwrite/pull/7162) +* Add chunkId to abuse key to prevent rate limit for SDKs [#7154](https://github.com/appwrite/appwrite/pull/7154) + +## Miscellaneous + +* Fix enum test case [#7163](https://github.com/appwrite/appwrite/pull/7163) +* Add flag to send logs to logger [#7155](https://github.com/appwrite/appwrite/pull/7155) +* Add a CI task to validate composer file and lock [#7142](https://github.com/appwrite/appwrite/pull/7142) + +# Version 1.4.12 + +## Miscellaneous +* Bump console to version 3.2.7 [#7148](https://github.com/appwrite/appwrite/pull/7148) +* Chore update database to 0.45.2 [#7138](https://github.com/appwrite/appwrite/pull/7138) +* Implement queue thresholds for the health API [#7123](https://github.com/appwrite/appwrite/pull/7123) +* Add Authorization::skip to the usage worker [#7124](https://github.com/appwrite/appwrite/pull/7124) + +## Bug fixes +* fix: use queueForDeletes in git installation delete endpoint [#7140](https://github.com/appwrite/appwrite/pull/7140) +* fix: patch script, make errors silent [#7134](https://github.com/appwrite/appwrite/pull/7134) +* fix: repositories recreation script [#7133](https://github.com/appwrite/appwrite/pull/7133) +* fix: Only delete repositories linked to the particular project [#7131](https://github.com/appwrite/appwrite/pull/7131) + # Version 1.4.11 ## Miscellaneous diff --git a/Dockerfile b/Dockerfile index 059c499bd9..8721301269 100755 --- a/Dockerfile +++ b/Dockerfile @@ -105,7 +105,8 @@ RUN chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/delete-orphaned-projects && \ chmod +x /usr/local/bin/clear-card-cache && \ chmod +x /usr/local/bin/calc-users-stats && \ - chmod +x /usr/local/bin/calc-tier-stats + chmod +x /usr/local/bin/calc-tier-stats && \ + chmod +x /usr/local/bin/get-migration-stats # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/README-CN.md b/README-CN.md index 4e45c423c4..0ce27a0038 100644 --- a/README-CN.md +++ b/README-CN.md @@ -55,7 +55,7 @@ Appwrite 可以提供给开发者用户验证,外部授权,用户数据读 ## 安装 -Appwrite 的容器化服务器只需要一行指令就可以运行。您可以使用 docker-compose 在本地主机上运行 Appwrite,也可以在任何其他容器化工具(如 Kubernetes、Docker Swarm 或 Rancher)上运行 Appwrite。 +Appwrite 的容器化服务器只需要一行指令就可以运行。您可以使用 docker-compose 在本地主机上运行 Appwrite,也可以在任何其他容器化工具(如 [Kubernetes](https://kubernetes.io/docs/home/)、[Docker Swarm](https://docs.docker.com/engine/swarm/) 或 [Rancher](https://rancher.com/docs/))上运行 Appwrite。 启动 Appwrite 服务器的最简单方法是运行我们的 docker-compose 文件。在运行安装命令之前,请确保您的机器上安装了 [Docker](https://dockerdocs.cn/get-docker/index.html): @@ -66,7 +66,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.13 ``` ### Windows @@ -78,7 +78,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.13 ``` #### PowerShell @@ -88,7 +88,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.13 ``` 运行后,可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后,服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index 88615a355a..62b084d320 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.13 ``` ### Windows @@ -88,7 +88,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.13 ``` #### PowerShell @@ -98,7 +98,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.13 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/config/errors.php b/app/config/errors.php index 2a61b74270..c0628920d9 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -209,6 +209,7 @@ return [ 'name' => Exception::USER_AUTH_METHOD_UNSUPPORTED, 'description' => 'The requested authentication method is either disabled or unsupported. Please check the supported authentication methods in the Appwrite console.', 'code' => 501, + 'publish' => false, ], Exception::USER_PHONE_ALREADY_EXISTS => [ 'name' => Exception::USER_PHONE_ALREADY_EXISTS, @@ -240,6 +241,16 @@ return [ 'description' => 'OAuth2 provider returned some error.', 'code' => 424, ], + Exception::USER_EMAIL_ALREADY_VERIFIED => [ + 'name' => Exception::USER_EMAIL_ALREADY_VERIFIED, + 'description' => 'User email is already verified', + 'code' => 409, + ], + Exception::USER_PHONE_ALREADY_VERIFIED => [ + 'name' => Exception::USER_PHONE_ALREADY_VERIFIED, + 'description' => 'User phone is already verified', + 'code' => 409 + ], /** Teams */ Exception::TEAM_NOT_FOUND => [ @@ -754,4 +765,12 @@ return [ 'description' => 'An error occurred on the provider\'s side. Please try again later.', 'code' => 400, ], + + /** Health */ + Exception::QUEUE_SIZE_EXCEEDED => [ + 'name' => Exception::QUEUE_SIZE_EXCEEDED, + 'description' => 'Queue size threshold hit.', + 'code' => 503, + 'publish' => false + ], ]; diff --git a/app/console b/app/console index f7c34a1b37..6bf696aeb3 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit f7c34a1b37d53dd5f28c83b4e12a4e68fcd9b484 +Subproject commit 6bf696aeb329d23e4ebe9d1e71edbd144cf2d58e diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 0a067cc8fd..c210b19f4f 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2662,6 +2662,10 @@ App::post('/v1/account/verification') throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP Disabled'); } + if ($user->getAttribute('emailVerification')) { + throw new Exception(Exception::USER_EMAIL_ALREADY_VERIFIED); + } + $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); $isAppUser = Auth::isAppUser($roles); @@ -2883,6 +2887,10 @@ App::post('/v1/account/verification/phone') throw new Exception(Exception::USER_PHONE_NOT_FOUND); } + if ($user->getAttribute('phoneVerification')) { + throw new Exception(Exception::USER_PHONE_ALREADY_VERIFIED); + } + $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); $isAppUser = Auth::isAppUser($roles); diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index f47e3f8265..0d2e2f22d1 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -1231,7 +1231,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', '', new Key(), 'Attribute Key.') - ->param('elements', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE, min: 0), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.') + ->param('elements', [], new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . DATABASE::LENGTH_KEY . ' characters long.') ->param('required', null, new Boolean(), 'Is attribute required?') ->param('default', null, new Text(0), 'Default value for attribute when not provided. Cannot be set when attribute is required.', true) ->param('array', false, new Boolean(), 'Is attribute an array?', true) @@ -1240,16 +1240,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') ->inject('queueForDatabase') ->inject('queueForEvents') ->action(function (string $databaseId, string $collectionId, string $key, array $elements, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { - // use length of longest string as attribute size - $size = 0; - foreach ($elements as $element) { - $length = \strlen($element); - if ($length === 0) { - throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Each enum element must not be empty'); - } - $size = ($length > $size) ? $length : $size; - } - if (!is_null($default) && !in_array($default, $elements)) { throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Default value not found in elements'); } @@ -1257,7 +1247,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') $attribute = createAttribute($databaseId, $collectionId, new Document([ 'key' => $key, 'type' => Database::VAR_STRING, - 'size' => $size, + 'size' => Database::LENGTH_KEY, 'required' => $required, 'default' => $default, 'array' => $array, @@ -1930,7 +1920,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/ ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', '', new Key(), 'Attribute Key.') - ->param('elements', null, new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.') + ->param('elements', null, new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . DATABASE::LENGTH_KEY . ' characters long.') ->param('required', null, new Boolean(), 'Is attribute required?') ->param('default', null, new Nullable(new Text(0)), 'Default value for attribute when not provided. Cannot be set when attribute is required.') ->inject('response') diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 90e080d5fa..4f459dd862 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -355,7 +355,7 @@ App::get('/v1/health/queue/webhooks') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -382,7 +382,7 @@ App::get('/v1/health/queue/logs') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -409,7 +409,7 @@ App::get('/v1/health/queue/certificates') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -436,7 +436,7 @@ App::get('/v1/health/queue/builds') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -464,7 +464,7 @@ App::get('/v1/health/queue/databases') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -491,7 +491,7 @@ App::get('/v1/health/queue/deletes') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -518,7 +518,7 @@ App::get('/v1/health/queue/mails') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -545,7 +545,7 @@ App::get('/v1/health/queue/messaging') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -572,7 +572,7 @@ App::get('/v1/health/queue/migrations') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -599,7 +599,7 @@ App::get('/v1/health/queue/functions') $size = $client->getQueueSize(); if ($size >= $threshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); + throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f22249e5d2..1fae48dae0 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -339,7 +339,7 @@ App::post('/v1/storage/buckets/:bucketId/files') ->label('audits.resource', 'file/{response.$id}') ->label('usage.metric', 'files.{scope}.requests.create') ->label('usage.params', ['bucketId:{request.bucketId}']) - ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId},chunkId:{chunkId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) diff --git a/app/controllers/general.php b/app/controllers/general.php index cf383b6710..e443b96fc9 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -608,8 +608,13 @@ App::error() $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); + $publish = true; - if ($logger) { + if ($error instanceof AppwriteException) { + $publish = $error->isPublishable(); + } + + if ($logger && $publish) { if ($error->getCode() >= 500 || $error->getCode() === 0) { try { /** @var Utopia\Database\Document $user */ diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index b37d76a816..4ea122e3c5 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -121,13 +121,16 @@ App::init() $abuseKeyLabel = (!is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; foreach ($abuseKeyLabel as $abuseKey) { + $start = $request->getContentRangeStart(); + $end = $request->getContentRangeEnd(); $timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject); $timeLimit ->setParam('{userId}', $user->getId()) ->setParam('{userAgent}', $request->getUserAgent('')) ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getHostname() . $route->getPath()) - ->setParam('{method}', $request->getMethod()); + ->setParam('{method}', $request->getMethod()) + ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start))); $timeLimitArray[] = $timeLimit; } diff --git a/app/init.php b/app/init.php index fd6e5d5cec..1d2b899356 100644 --- a/app/init.php +++ b/app/init.php @@ -103,8 +103,8 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 516; -const APP_VERSION_STABLE = '1.4.11'; +const APP_CACHE_BUSTER = 328; +const APP_VERSION_STABLE = '1.4.13'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; @@ -761,6 +761,25 @@ $register->set('pools', function () { return $group; }); +$register->set('db', function () { + // This is usually for our workers or CLI commands scope + $dbHost = App::getEnv('_APP_DB_HOST', ''); + $dbPort = App::getEnv('_APP_DB_PORT', ''); + $dbUser = App::getEnv('_APP_DB_USER', ''); + $dbPass = App::getEnv('_APP_DB_PASS', ''); + $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); + + $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true, + )); + + return $pdo; +}); $register->set('influxdb', function () { // Register DB connection diff --git a/bin/get-migration-stats b/bin/get-migration-stats new file mode 100644 index 0000000000..efc1934e15 --- /dev/null +++ b/bin/get-migration-stats @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php get-migration-stats $@ \ No newline at end of file diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 285b8ea239..6449ffd93a 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -84,6 +84,8 @@ class Exception extends \Exception public const USER_OAUTH2_BAD_REQUEST = 'user_oauth2_bad_request'; public const USER_OAUTH2_UNAUTHORIZED = 'user_oauth2_unauthorized'; public const USER_OAUTH2_PROVIDER_ERROR = 'user_oauth2_provider_error'; + public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_alread_verified'; + public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified'; /** Teams */ public const TEAM_NOT_FOUND = 'team_not_found'; @@ -227,12 +229,16 @@ class Exception extends \Exception public const MIGRATION_PROVIDER_ERROR = 'migration_provider_error'; /** Realtime */ - public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid'; - public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages'; - public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation'; + public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid'; + public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages'; + public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation'; + + /** Health */ + public const QUEUE_SIZE_EXCEEDED = 'queue_size_exceeded'; protected string $type = ''; protected array $errors = []; + protected bool $publish = true; public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null) { @@ -242,6 +248,7 @@ class Exception extends \Exception if (isset($this->errors[$type])) { $this->code = $this->errors[$type]['code']; $this->message = $this->errors[$type]['description']; + $this->publish = $this->errors[$type]['publish'] ?? true; } $this->message = $message ?? $this->message; @@ -271,4 +278,14 @@ class Exception extends \Exception { $this->type = $type; } + + /** + * Check whether the log is publishable for the exception. + * + * @return bool + */ + public function isPublishable(): bool + { + return $this->publish; + } } diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 406acae7df..9ee4d9236a 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -76,6 +76,8 @@ abstract class Migration '1.4.9' => 'V19', '1.4.10' => 'V19', '1.4.11' => 'V19', + '1.4.12' => 'V19', + '1.4.13' => 'V19' ]; /** @@ -193,17 +195,24 @@ abstract class Migration * @return iterable * @throws \Exception */ - public function documentsIterator(string $collectionId): iterable + public function documentsIterator(string $collectionId, $queries = []): iterable { $sum = 0; $nextDocument = null; $collectionCount = $this->projectDB->count($collectionId); + $queries[] = Query::limit($this->limit); do { - $queries = [Query::limit($this->limit)]; if ($nextDocument !== null) { - $queries[] = Query::cursorAfter($nextDocument); + $cursorQueryIndex = \array_search('cursorAfter', \array_map(fn (Query $query) => $query->getMethod(), $queries)); + + if ($cursorQueryIndex !== false) { + $queries[$cursorQueryIndex] = Query::cursorAfter($nextDocument); + } else { + $queries[] = Query::cursorAfter($nextDocument); + } } + $documents = $this->projectDB->find($collectionId, $queries); $count = count($documents); $sum += $count; diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index b9ccf3c302..46a0a073a6 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -10,6 +10,7 @@ use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception; +use Utopia\Database\Query; class V19 extends Migration { @@ -41,6 +42,11 @@ class V19 extends Migration Console::info('Migrating Buckets'); $this->migrateBuckets(); + if ($this->project->getId() !== 'console') { + Console::info('Migrating Enum Attribute Size'); + $this->migrateEnumAttributeSize(); + } + Console::info('Migrating Documents'); $this->forEachDocument([$this, 'fixDocument']); @@ -640,6 +646,22 @@ class V19 extends Migration return $commands; } + private function migrateEnumAttributeSize(): void + { + foreach ( + $this->documentsIterator('attributes', [ + Query::equal('format', ['enum']), + Query::lessThan('size', Database::LENGTH_KEY) + ]) as $attribute + ) { + $attribute->setAttribute('size', Database::LENGTH_KEY); + $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); + $databaseInternalId = $attribute->getAttribute('databaseInternalId'); + $collectionInternalId = $attribute->getAttribute('collectionInternalId'); + $this->projectDB->updateAttribute('database_' . $databaseInternalId . '_collection_' . $collectionInternalId, $attribute->getAttribute('key'), size: 255); + } + } + /** * Fix run on each document * diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 28d7046dd1..dc6ddc1a5b 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -19,6 +19,7 @@ use Appwrite\Platform\Tasks\VolumeSync; use Appwrite\Platform\Tasks\CalcTierStats; use Appwrite\Platform\Tasks\Upgrade; use Appwrite\Platform\Tasks\DeleteOrphanedProjects; +use Appwrite\Platform\Tasks\GetMigrationStats; use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments; class Tasks extends Service @@ -44,6 +45,7 @@ class Tasks extends Service ->addAction(CalcTierStats::getName(), new CalcTierStats()) ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects()) ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments()) + ->addAction(GetMigrationStats::getName(), new GetMigrationStats()) ; } diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index 2a2bc20af9..e6559a05d7 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -41,6 +41,7 @@ class CalcTierStats extends Action 'Functions', 'Deployments', 'Executions', + 'Migrations', ]; protected string $directory = '/usr/local'; @@ -99,8 +100,8 @@ class CalcTierStats extends Action $projects = [$console]; $count = 0; - $limit = 30; - $sum = 30; + $limit = 100; + $sum = 100; $offset = 0; while (!empty($projects)) { foreach ($projects as $project) { @@ -200,7 +201,7 @@ class CalcTierStats extends Action try { /** Get Domains */ - $stats['Domains'] = $dbForConsole->count('domains', [ + $stats['Domains'] = $dbForConsole->count('rules', [ Query::equal('projectInternalId', [$project->getInternalId()]), ]); } catch (\Throwable) { @@ -290,6 +291,13 @@ class CalcTierStats extends Action $stats['Executions'] = 0; } + /** Get Total Migrations */ + try { + $stats['Migrations'] = $dbForProject->count('migrations', []); + } catch (\Throwable) { + $stats['Migrations'] = 0; + } + $csv->insertOne(array_values($stats)); } catch (\Throwable $th) { Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2824f4e286..757b29c1b6 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -2,17 +2,17 @@ namespace Appwrite\Platform\Tasks; -use PHPMailer\PHPMailer\PHPMailer; use Utopia\App; use Utopia\Config\Config; +use Utopia\Database\Helpers\ID; use Utopia\Database\Query; -use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Pools\Group; use Utopia\Registry\Registry; +use Utopia\Validator\Boolean; class DeleteOrphanedProjects extends Action { @@ -25,18 +25,19 @@ class DeleteOrphanedProjects extends Action { $this - ->desc('Get stats for projects') + ->desc('Delete orphaned projects') + ->param('commit', false, new Boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') ->inject('dbForConsole') ->inject('register') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($pools, $cache, $dbForConsole, $register); + ->callback(function (bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($commit, $pools, $cache, $dbForConsole, $register); }); } - public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + public function action(bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void { Console::title('Delete orphaned projects V1'); @@ -45,6 +46,17 @@ class DeleteOrphanedProjects extends Action /** @var array $collections */ $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; + $collectionsConfig = array_merge([ + 'audit' => [ + '$id' => ID::custom('audit'), + '$collection' => Database::METADATA + ], + 'abuse' => [ + '$id' => ID::custom('abuse'), + '$collection' => Database::METADATA + ] + ], $collectionsConfig); + /* Initialise new Utopia app */ $app = new App('UTC'); $console = $app->getResource('console'); @@ -54,7 +66,8 @@ class DeleteOrphanedProjects extends Action $totalProjects = $dbForConsole->count('projects'); Console::success("Found a total of: {$totalProjects} projects"); - $orphans = 0; + $orphans = 1; + $cnt = 0; $count = 0; $limit = 30; $sum = 30; @@ -79,19 +92,52 @@ class DeleteOrphanedProjects extends Action $dbForProject = new Database($adapter, $cache); $dbForProject->setDefaultDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); - $collectionsCreated = $dbForProject->count(Database::METADATA); - $message = ' (' . $collectionsCreated . ') collections where found on project (' . $project->getId() . '))'; - if ($collectionsCreated < (count($collectionsConfig) + 2)) { - Console::error($message); - $orphans++; - } else { - Console::log($message); + + $collectionsCreated = 0; + $cnt++; + if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { + $collectionsCreated = $dbForProject->count(Database::METADATA); } - } catch (\Throwable $th) { - //$dbForConsole->deleteDocument('projects', $project->getId()); - //Console::success('Deleting project (' . $project->getId() . ')'); - Console::error(' (0) collections where found for project (' . $project->getId() . ')'); + + $msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; + + if ($collectionsCreated >= count($collectionsConfig)) { + Console::log($msg . ' ignoring....'); + continue; + } + + Console::log($msg); + + if ($collectionsCreated > 0) { + $collections = $dbForProject->find(Database::METADATA, []); + foreach ($collections as $collection) { + if ($commit) { + $dbForProject->deleteCollection($collection->getId()); + $dbForConsole->deleteCachedCollection($collection->getId()); + } + Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')'); + } + } + + if ($commit) { + $dbForConsole->deleteDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); + + if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { + try { + $dbForProject->deleteCollection(Database::METADATA); + $dbForProject->deleteCachedCollection(Database::METADATA); + } catch (\Throwable $th) { + Console::warning('Metadata collection does not exist'); + } + } + } + + Console::info('--Deleting project no (' . $project->getInternalId() . ')'); + $orphans++; + } catch (\Throwable $th) { + Console::error('Error: ' . $th->getMessage() . ' ' . $th->getTraceAsString()); } finally { $pools ->get($db) @@ -110,6 +156,6 @@ class DeleteOrphanedProjects extends Action $count = $count + $sum; } - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans'); + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); } } diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php new file mode 100644 index 0000000000..b76e0428d7 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -0,0 +1,187 @@ +desc('Get stats for projects') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->inject('register') + ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($pools, $cache, $dbForConsole, $register); + }); + } + + /** + * @throws \Utopia\Exception + * @throws CannotInsertRecord + */ + public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + { + //docker compose exec -t appwrite get-migration-stats + + Console::title('Migration stats calculation V1'); + Console::success(APP_NAME . ' Migration stats calculation has started'); + + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + + /** CSV stuff */ + $this->date = date('Y-m-d'); + $this->path = "{$this->directory}/migration_stats_{$this->date}.csv"; + $csv = Writer::createFromPath($this->path, 'w'); + $csv->insertOne($this->columns); + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects'); + Console::success("Found a total of: {$totalProjects} projects"); + + $projects = [$console]; + $count = 0; + $limit = 100; + $sum = 100; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + Console::info("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + /** Get Project ID */ + $stats['Project ID'] = $project->getId(); + + /** Get Migration details */ + $migrations = $dbForProject->find('migrations', [ + Query::limit(500) + ]); + + $migrations = array_map(function ($migration) use ($project) { + return [ + $project->getId(), + $migration->getAttribute('$id'), + $migration->getAttribute('$createdAt'), + $migration->getAttribute('status'), + $migration->getAttribute('stage'), + $migration->getAttribute('source'), + ]; + }, $migrations); + + if (!empty($migrations)) { + $csv->insertAll($migrations); + } + } catch (\Throwable $th) { + Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + } + + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); + + $pools + ->get('console') + ->reclaim(); + + /** @var PHPMailer $mail */ + $mail = $register->get('smtp'); + + $mail->clearAddresses(); + $mail->clearAllRecipients(); + $mail->clearReplyTos(); + $mail->clearAttachments(); + $mail->clearBCCs(); + $mail->clearCCs(); + + try { + /** Addresses */ + $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); + $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); + + foreach ($recipients as $recipient) { + $mail->addAddress($recipient); + } + + /** Attachments */ + $mail->addAttachment($this->path); + + /** Content */ + $mail->Subject = "Migration Report for {$this->date}"; + $mail->Body = "Please find the migration report atttached"; + $mail->send(); + Console::success('Email has been sent!'); + } catch (Exception $e) { + Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); + } + } +} diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 1d5d3b0b26..2e2c2d466e 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -149,6 +149,9 @@ class Hamster extends Action /** Get Total Teams */ $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); + /** Get Total Migrations */ + $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); + /** Get Total Members */ $teamInternalId = $project->getAttribute('teamInternalId', null); if ($teamInternalId) { diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 38e5c8aa46..682b9b5559 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -11,6 +11,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +use Utopia\Registry\Registry; use Utopia\Validator\Text; class Migrate extends Action @@ -29,7 +30,8 @@ class Migrate extends Action ->inject('cache') ->inject('dbForConsole') ->inject('getProjectDB') - ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB)); + ->inject('register') + ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $cache, $dbForConsole, $getProjectDB, $register)); } private function clearProjectsCache(Cache $cache, Document $project) @@ -41,7 +43,7 @@ class Migrate extends Action } } - public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB) + public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register) { Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { @@ -89,9 +91,11 @@ class Migrate extends Action try { // TODO: Iterate through all project DBs + /** @var Database $projectDB */ $projectDB = $getProjectDB($project); $migration ->setProject($project, $projectDB, $dbForConsole) + ->setPDO($register->get('db', true)) ->execute(); } catch (\Throwable $th) { Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); diff --git a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php similarity index 100% rename from src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php rename to src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index e0a12b443f..6dbf49c44c 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -8,6 +8,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use InfluxDB\Database as InfluxDatabase; use DateTime; +use Utopia\Database\Validator\Authorization; use Utopia\Registry\Registry; class TimeSeries extends Calculator @@ -426,32 +427,34 @@ class TimeSeries extends Calculator $project = $this->database->getDocument('projects', $projectId); $database = call_user_func($this->getProjectDB, $project); - try { - $document = $database->getDocument('stats', $id); - if ($document->isEmpty()) { - $database->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => $type, - 'region' => $this->region, - ])); - } else { - $database->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); + Authorization::skip(function () use ($database, $id, $period, $time, $metric, $value, $type, $projectId) { + try { + $document = $database->getDocument('stats', $id); + if ($document->isEmpty()) { + $database->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => $type, + 'region' => $this->region, + ])); + } else { + $database->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $value) + ); + } + } catch (\Exception $e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); + } else { + throw $e; + } } - } catch (\Exception $e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); - } else { - throw $e; - } - } + }); $this->register->get('pools')->reclaim(); } diff --git a/tests/e2e/Services/Account/AccountBase.php b/tests/e2e/Services/Account/AccountBase.php index e64cf2c4ca..953b89ced7 100644 --- a/tests/e2e/Services/Account/AccountBase.php +++ b/tests/e2e/Services/Account/AccountBase.php @@ -945,6 +945,32 @@ trait AccountBase return $data; } + /** + * @depends testUpdateAccountVerification + */ + public function testCreateAccountVerificationForVerifiedEmail($data): array + { + $email = $data['email'] ?? ''; + $name = $data['name'] ?? ''; + $session = $data['session'] ?? ''; + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'url' => 'http://localhost/verification', + ]); + + $this->assertEquals(409, $response['headers']['status-code']); + + return $data; + } + /** * @depends testUpdateAccountVerification */ diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 0e0634850a..8e2117032f 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1068,4 +1068,27 @@ class AccountCustomClientTest extends Scope return $data; } + + /** + * @depends testPhoneVerification + */ + #[Retry(count: 1)] + public function testPhoneVerificationForVerifiedPhone(array $data): array + { + $session = $data['session'] ?? ''; + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_POST, '/account/verification/phone', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ])); + + $this->assertEquals(409, $response['headers']['status-code']); + + return $data; + } } diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 34a433353c..f86b817777 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -978,7 +978,7 @@ trait DatabasesBase ]); $this->assertEquals(400, $badEnum['headers']['status-code']); - $this->assertEquals('Each enum element must not be empty', $badEnum['body']['message']); + $this->assertEquals('Invalid `elements` param: Value must a valid array and Value must be a valid string and at least 1 chars and no longer than 255 chars', $badEnum['body']['message']); return $data; } diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 8fa9faadd2..3ea1b884a7 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -145,7 +145,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -171,7 +171,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -197,7 +197,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -223,7 +223,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -249,7 +249,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -280,7 +280,7 @@ class HealthCustomServerTest extends Scope 'name' => 'database_db_main', 'threshold' => '0' ]); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -306,7 +306,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -332,7 +332,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -358,7 +358,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } @@ -384,7 +384,7 @@ class HealthCustomServerTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); - $this->assertEquals(500, $response['headers']['status-code']); + $this->assertEquals(503, $response['headers']['status-code']); return []; } diff --git a/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php b/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php index 67c2dec36b..21c270d199 100644 --- a/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php +++ b/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php @@ -636,6 +636,120 @@ class WebhooksCustomClientTest extends Scope return $data; } + /** + * @depends testUpdateAccountPrefs + */ + public function testCreateAccountVerification($data): array + { + $id = $data['id'] ?? ''; + $email = $data['email'] ?? ''; + $session = $data['session'] ?? ''; + + $verification = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'url' => 'http://localhost/verification', + ]); + + $verificationId = $verification['body']['$id']; + + $this->assertEquals(201, $verification['headers']['status-code']); + $this->assertIsArray($verification['body']); + + $webhook = $this->getLastRequest(); + $signatureKey = $this->getProject()['signatureKey']; + $payload = json_encode($webhook['data']); + $url = $webhook['url']; + $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); + + $this->assertEquals($webhook['method'], 'POST'); + $this->assertEquals($webhook['headers']['Content-Type'], 'application/json'); + $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); + $this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('users.*.verification.*.create', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.*.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.*.create", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); + $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); + $this->assertNotEmpty($webhook['data']['$id']); + $this->assertNotEmpty($webhook['data']['userId']); + $this->assertNotEmpty($webhook['data']['secret']); + $this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire'])); + + $data['secret'] = $webhook['data']['secret']; + + return $data; + } + + /** + * @depends testCreateAccountVerification + */ + public function testUpdateAccountVerification($data): array + { + $id = $data['id'] ?? ''; + $email = $data['email'] ?? ''; + $session = $data['session'] ?? ''; + $secret = $data['secret'] ?? ''; + + $verification = $this->client->call(Client::METHOD_PUT, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'userId' => $id, + 'secret' => $secret, + ]); + + $verificationId = $verification['body']['$id']; + + $this->assertEquals(200, $verification['headers']['status-code']); + $this->assertIsArray($verification['body']); + + $webhook = $this->getLastRequest(); + $signatureKey = $this->getProject()['signatureKey']; + $payload = json_encode($webhook['data']); + $url = $webhook['url']; + $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); + + $this->assertEquals($webhook['method'], 'POST'); + $this->assertEquals($webhook['headers']['Content-Type'], 'application/json'); + $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); + $this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('users.*.verification.*.update', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.*.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.*.update", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); + $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); + $this->assertNotEmpty($webhook['data']['$id']); + $this->assertNotEmpty($webhook['data']['userId']); + $this->assertNotEmpty($webhook['data']['secret']); + $this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire'])); + + $data['secret'] = $webhook['data']['secret']; + + return $data; + } + /** * @depends testUpdateAccountPrefs */ @@ -751,120 +865,6 @@ class WebhooksCustomClientTest extends Scope return $data; } - /** - * @depends testUpdateAccountPrefs - */ - public function testCreateAccountVerification($data): array - { - $id = $data['id'] ?? ''; - $email = $data['email'] ?? ''; - $session = $data['session'] ?? ''; - - $verification = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ - 'origin' => 'http://localhost', - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ]), [ - 'url' => 'http://localhost/verification', - ]); - - $verificationId = $verification['body']['$id']; - - $this->assertEquals(201, $verification['headers']['status-code']); - $this->assertIsArray($verification['body']); - - $webhook = $this->getLastRequest(); - $signatureKey = $this->getProject()['signatureKey']; - $payload = json_encode($webhook['data']); - $url = $webhook['url']; - $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); - - $this->assertEquals($webhook['method'], 'POST'); - $this->assertEquals($webhook['headers']['Content-Type'], 'application/json'); - $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); - $this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('users.*.verification.*.create', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.*.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.*.create", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); - $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); - $this->assertNotEmpty($webhook['data']['$id']); - $this->assertNotEmpty($webhook['data']['userId']); - $this->assertNotEmpty($webhook['data']['secret']); - $this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire'])); - - $data['secret'] = $webhook['data']['secret']; - - return $data; - } - - /** - * @depends testCreateAccountVerification - */ - public function testUpdateAccountVerification($data): array - { - $id = $data['id'] ?? ''; - $email = $data['email'] ?? ''; - $session = $data['session'] ?? ''; - $secret = $data['secret'] ?? ''; - - $verification = $this->client->call(Client::METHOD_PUT, '/account/verification', array_merge([ - 'origin' => 'http://localhost', - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ]), [ - 'userId' => $id, - 'secret' => $secret, - ]); - - $verificationId = $verification['body']['$id']; - - $this->assertEquals(200, $verification['headers']['status-code']); - $this->assertIsArray($verification['body']); - - $webhook = $this->getLastRequest(); - $signatureKey = $this->getProject()['signatureKey']; - $payload = json_encode($webhook['data']); - $url = $webhook['url']; - $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); - - $this->assertEquals($webhook['method'], 'POST'); - $this->assertEquals($webhook['headers']['Content-Type'], 'application/json'); - $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); - $this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('users.*.verification.*.update', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.*.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.*.update", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); - $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); - $this->assertNotEmpty($webhook['data']['$id']); - $this->assertNotEmpty($webhook['data']['userId']); - $this->assertNotEmpty($webhook['data']['secret']); - $this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire'])); - - $data['secret'] = $webhook['data']['secret']; - - return $data; - } - /** * @depends testCreateTeamMembership */