From aa36d944dbbd1b57c5c7eb637e36749896b15632 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 1 Mar 2025 22:00:37 +0100 Subject: [PATCH 001/245] Add teamId to project array in e2e test --- tests/e2e/Scopes/ProjectCustom.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/Scopes/ProjectCustom.php b/tests/e2e/Scopes/ProjectCustom.php index f2617bd4be..d5c10481e6 100644 --- a/tests/e2e/Scopes/ProjectCustom.php +++ b/tests/e2e/Scopes/ProjectCustom.php @@ -146,6 +146,7 @@ trait ProjectCustom $project = [ '$id' => $project['body']['$id'], 'name' => $project['body']['name'], + 'teamId' => $team['body']['$id'], 'apiKey' => $key['body']['secret'], 'webhookId' => $webhook['body']['$id'], 'signatureKey' => $webhook['body']['signatureKey'], From cb05b87dda13e8deb6c311b76095ba80ad2d9ddd Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 10 Mar 2025 00:08:04 +0100 Subject: [PATCH 002/245] Upgraded users to use utopia/auth --- app/controllers/api/users.php | 112 ++- app/controllers/shared/api.php | 69 +- app/init.php | 19 + composer.json | 1 + composer.lock | 210 +++--- src/Appwrite/Platform/Workers/Builds.php | 829 +++++++++++++++++++++++ 6 files changed, 1122 insertions(+), 118 deletions(-) create mode 100644 src/Appwrite/Platform/Workers/Builds.php diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 962022927f..4a0c6013ed 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -28,6 +28,7 @@ use Appwrite\Utopia\Response; use MaxMind\Db\Reader; use Utopia\App; use Utopia\Audit\Audit; +use Utopia\Auth\Hash; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -53,12 +54,20 @@ use Utopia\Validator\Integer; use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; +use Utopia\Auth\Hashes\Argon2; +use Utopia\Auth\Hashes\Bcrypt; +use Utopia\Auth\Hashes\MD5; +use Utopia\Auth\Hashes\PHPass; +use Utopia\Auth\Hashes\Scrypt; +use Utopia\Auth\Hashes\ScryptModified; +use Utopia\Auth\Hashes\Sha; +use Utopia\Auth\Hashes\Plaintext; +use Utopia\Auth\Proofs\Password as ProofsPassword; /** TODO: Remove function when we move to using utopia/platform */ -function createUser(string $hash, mixed $hashOptions, string $userId, ?string $email, ?string $password, ?string $phone, string $name, Document $project, Database $dbForProject, Hooks $hooks): Document +function createUser(Hash $hash, string $userId, ?string $email, ?string $password, ?string $phone, string $name, Document $project, Database $dbForProject, Hooks $hooks): Document { $plaintextPassword = $password; - $hashOptionsObject = (\is_string($hashOptions)) ? \json_decode($hashOptions, true) : $hashOptions; // Cast to JSON array $passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0; if (!empty($email)) { @@ -92,7 +101,18 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e } } - $password = (!empty($password)) ? ($hash === 'plaintext' ? Auth::passwordHash($password, $hash, $hashOptionsObject) : $password) : null; + $hashedPassword = null; + + if (!empty($password)) { + if ($hash instanceof Plaintext) { // Password was never hashed, hash it with the default hash + $defaultHash = new ProofsPassword(); + $hashedPassword = $defaultHash->hash($password); + $hash = $defaultHash->getHash(); + } else { + $hashedPassword = $password; + } + } + $user = new Document([ '$id' => $userId, '$permissions' => [ @@ -106,11 +126,11 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e 'phoneVerification' => false, 'status' => true, 'labels' => [], - 'password' => $password, - 'passwordHistory' => is_null($password) || $passwordHistory === 0 ? [] : [$password], - 'passwordUpdate' => (!empty($password)) ? DateTime::now() : null, - 'hash' => $hash === 'plaintext' ? Auth::DEFAULT_ALGO : $hash, - 'hashOptions' => $hash === 'plaintext' ? Auth::DEFAULT_ALGO_OPTIONS : $hashOptionsObject + ['type' => $hash], + 'password' => $hashedPassword, + 'passwordHistory' => is_null($hashedPassword) || $passwordHistory === 0 ? [] : [$hashedPassword], + 'passwordUpdate' => (!empty($hashedPassword)) ? DateTime::now() : null, + 'hash' => $hash->getName(), + 'hashOptions' => $hash->getOptions(), 'registration' => DateTime::now(), 'reset' => false, 'name' => $name, @@ -121,7 +141,7 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e 'search' => implode(' ', [$userId, $email, $phone, $name]), ]); - if ($hash === 'plaintext') { + if ($hash instanceof Plaintext) { $hooks->trigger('passwordValidator', [$dbForProject, $project, $plaintextPassword, &$user, true]); } @@ -211,7 +231,9 @@ App::post('/v1/users') ->inject('dbForProject') ->inject('hooks') ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Hooks $hooks) { - $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $hooks); + $plaintext = new Plaintext(); + + $user = createUser($plaintext, $userId, $email, $password, $phone, $name, $project, $dbForProject, $hooks); $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($user, Response::MODEL_USER); @@ -244,7 +266,10 @@ App::post('/v1/users/bcrypt') ->inject('dbForProject') ->inject('hooks') ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Hooks $hooks) { - $user = createUser('bcrypt', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); + $bcrypt = new Bcrypt(); + $bcrypt->setCost(8); // Default cost + + $user = createUser($bcrypt, $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -278,7 +303,9 @@ App::post('/v1/users/md5') ->inject('dbForProject') ->inject('hooks') ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Hooks $hooks) { - $user = createUser('md5', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); + $md5 = new MD5(); + + $user = createUser($md5, $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -312,7 +339,13 @@ App::post('/v1/users/argon2') ->inject('dbForProject') ->inject('hooks') ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Hooks $hooks) { - $user = createUser('argon2', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); + $argon2 = new Argon2(); + $argon2 + ->setMemoryCost(2048) + ->setTimeCost(4) + ->setThreads(3); + + $user = createUser($argon2, $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -347,13 +380,12 @@ App::post('/v1/users/sha') ->inject('dbForProject') ->inject('hooks') ->action(function (string $userId, string $email, string $password, string $passwordVersion, string $name, Response $response, Document $project, Database $dbForProject, Hooks $hooks) { - $options = '{}'; - + $sha = new Sha(); if (!empty($passwordVersion)) { - $options = '{"version":"' . $passwordVersion . '"}'; + $sha->setVersion($passwordVersion); } - $user = createUser('sha', $options, $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); + $user = createUser($sha, $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -387,7 +419,9 @@ App::post('/v1/users/phpass') ->inject('dbForProject') ->inject('hooks') ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Hooks $hooks) { - $user = createUser('phpass', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); + $phpass = new PHPass(); + + $user = createUser($phpass, $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -426,15 +460,15 @@ App::post('/v1/users/scrypt') ->inject('dbForProject') ->inject('hooks') ->action(function (string $userId, string $email, string $password, string $passwordSalt, int $passwordCpu, int $passwordMemory, int $passwordParallel, int $passwordLength, string $name, Response $response, Document $project, Database $dbForProject, Hooks $hooks) { - $options = [ - 'salt' => $passwordSalt, - 'costCpu' => $passwordCpu, - 'costMemory' => $passwordMemory, - 'costParallel' => $passwordParallel, - 'length' => $passwordLength - ]; + $scrypt = new Scrypt(); + $scrypt + ->setSalt($passwordSalt) + ->setCpuCost($passwordCpu) + ->setMemoryCost($passwordMemory) + ->setParallelCost($passwordParallel) + ->setLength($passwordLength); - $user = createUser('scrypt', \json_encode($options), $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); + $user = createUser($scrypt, $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -471,7 +505,13 @@ App::post('/v1/users/scrypt-modified') ->inject('dbForProject') ->inject('hooks') ->action(function (string $userId, string $email, string $password, string $passwordSalt, string $passwordSaltSeparator, string $passwordSignerKey, string $name, Response $response, Document $project, Database $dbForProject, Hooks $hooks) { - $user = createUser('scryptMod', '{"signerKey":"' . $passwordSignerKey . '","saltSeparator":"' . $passwordSaltSeparator . '","salt":"' . $passwordSalt . '"}', $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); + $scryptModified = new ScryptModified(); + $scryptModified + ->setSalt($passwordSalt) + ->setSaltSeparator($passwordSaltSeparator) + ->setSignerKey($passwordSignerKey); + + $user = createUser($scryptModified, $userId, $email, $password, null, $name, $project, $dbForProject, $hooks); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -1012,7 +1052,6 @@ App::get('/v1/users/identities') } catch (QueryException $e) { throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - if (!empty($search)) { $queries[] = Query::search('search', $search); } @@ -1269,7 +1308,13 @@ App::patch('/v1/users/:userId/password') $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]); - $newPassword = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS); + // Create Argon2 hasher with default settings + $hasher = new Argon2(); + $hasher->setMemoryCost(65536); + $hasher->setTimeCost(4); + $hasher->setThreads(3); + + $newPassword = $hasher->hash($password); $historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0; $history = $user->getAttribute('passwordHistory', []); @@ -1287,8 +1332,12 @@ App::patch('/v1/users/:userId/password') ->setAttribute('password', $newPassword) ->setAttribute('passwordHistory', $history) ->setAttribute('passwordUpdate', DateTime::now()) - ->setAttribute('hash', Auth::DEFAULT_ALGO) - ->setAttribute('hashOptions', Auth::DEFAULT_ALGO_OPTIONS); + ->setAttribute('hash', 'argon2') + ->setAttribute('hashOptions', [ + 'memoryCost' => 65536, + 'timeCost' => 4, + 'threads' => 3 + ]); $user = $dbForProject->updateDocument('users', $user->getId(), $user); @@ -2466,3 +2515,4 @@ App::get('/v1/users/usage') 'sessions' => $usage[$metrics[1]]['data'], ]), Response::MODEL_USAGE_USERS); }); + diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index f8371ed8e6..f87ddf9730 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -200,33 +200,88 @@ App::init() ->action(function (App $utopia, Request $request, Database $dbForPlatform, Database $dbForProject, Audit $queueForAudits, Document $project, Document $user, ?Document $session, array $servers, string $mode, Document $team, ?Key $apiKey) { $route = $utopia->getRoute(); + /** + * Handle user authentication and session validation. + * + * This function follows a series of steps to determine the appropriate user session + * based on cookies, headers, and JWT tokens. + * + * Process: + * + * Project & Role Validation: + * 1. Check if the project is empty. If so, throw an exception. + * 2. Get the roles configuration. + * 3. Determine the role for the user based on the user document. + * 4. Get the scopes for the role. + * + * API Key Authentication: + * 5. If there is an API key: + * - Verify no user session exists simultaneously + * - Check if key is expired + * - Set role and scopes from API key + * - Handle special app role case + * - For standard keys, update last accessed time + * + * User Activity: + * 6. If the project is not the console and user is not admin: + * - Update user's last activity timestamp + * + * Access Control: + * 7. Get the method from the route + * 8. Validate namespace permissions + * 9. Validate scope permissions + * 10. Check if user is blocked + * + * Security Checks: + * 11. Verify password status (check if reset required) + * 12. Validate MFA requirements: + * - Check if MFA is enabled + * - Verify email status + * - Verify phone status + * - Verify authenticator status + * 13. Handle Multi-Factor Authentication: + * - Check remaining required factors + * - Validate factor completion + * - Throw exception if factors incomplete + */ + + // Step 1: Check if project is empty if ($project->isEmpty()) { throw new Exception(Exception::PROJECT_NOT_FOUND); } + // Step 2: Get roles configuration $roles = Config::getParam('roles', []); + // Step 3: Determine role for user + // TODO get scopes from the identity instead of the user roles config. The identity will containn the scopes the user authorized for the access token. + $role = $user->isEmpty() ? Role::guests()->toString() : Role::users()->toString(); + // Step 4: Get scopes for the role $scopes = $roles[$role]['scopes']; - // API Key authentication + // Step 5: API Key Authentication if (!empty($apiKey)) { + // Verify no user session exists simultaneously if (!$user->isEmpty()) { throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); } + // Check if key is expired if ($apiKey->isExpired()) { throw new Exception(Exception::PROJECT_KEY_EXPIRED); } + // Set role and scopes from API key $role = $apiKey->getRole(); $scopes = $apiKey->getScopes(); // Disable authorization checks for API keys Authorization::setDefaultStatus(false); + // Handle special app role case if ($apiKey->getRole() === Auth::USER_ROLE_APPS) { $user = new Document([ '$id' => '', @@ -240,6 +295,7 @@ App::init() $queueForAudits->setUser($user); } + // For standard keys, update last accessed time if ($apiKey->getType() === API_KEY_STANDARD) { $dbKey = $project->find( key: 'secret', @@ -307,7 +363,7 @@ App::init() Authorization::setRole($authRole); } - // Update project last activity + // Step 6: Update project and user last activity if (!$project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { @@ -316,7 +372,6 @@ App::init() } } - // Update user last activity if (!empty($user->getId())) { $accessedAt = $user->getAttribute('accessedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { @@ -330,6 +385,7 @@ App::init() } } + // Steps 7-9: Access Control - Method, Namespace and Scope Validation /** * @var ?Method $method */ @@ -351,21 +407,23 @@ App::init() } } - // Do now allow access if scope is not allowed + // Step 9: Validate scope permissions $scope = $route->getLabel('scope', 'none'); if (!\in_array($scope, $scopes)) { throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, $user->getAttribute('email', 'User') . ' (role: ' . \strtolower($roles[$role]['label']) . ') missing scope (' . $scope . ')'); } - // Do not allow access to blocked accounts + // Step 10: Check if user is blocked if (false === $user->getAttribute('status')) { // Account is blocked throw new Exception(Exception::USER_BLOCKED); } + // Step 11: Verify password status if ($user->getAttribute('reset')) { throw new Exception(Exception::USER_PASSWORD_RESET_REQUIRED); } + // Step 12: Validate MFA requirements $mfaEnabled = $user->getAttribute('mfa', false); $hasVerifiedEmail = $user->getAttribute('emailVerification', false); $hasVerifiedPhone = $user->getAttribute('phoneVerification', false); @@ -373,6 +431,7 @@ App::init() $hasMoreFactors = $hasVerifiedEmail || $hasVerifiedPhone || $hasVerifiedAuthenticator; $minimumFactors = ($mfaEnabled && $hasMoreFactors) ? 2 : 1; + // Step 13: Handle Multi-Factor Authentication if (!in_array('mfa', $route->getGroups())) { if ($session && \count($session->getAttribute('factors', [])) < $minimumFactors) { throw new Exception(Exception::USER_MORE_FACTORS_REQUIRED); diff --git a/app/init.php b/app/init.php index 4e36c91cb1..324c9cca9a 100644 --- a/app/init.php +++ b/app/init.php @@ -1289,6 +1289,25 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons /** @var Utopia\Database\Database $dbForPlatform */ /** @var string $mode */ + /** + * Handles user authentication and session validation. + * + * This function follows a series of steps to determine the appropriate user session + * based on cookies, headers, and JWT tokens. + * + * Process: + * 1. Checks the cookie based on mode: + * - If in admin mode, redirects to the console. + * - Otherwise, retrieves the project ID from the cookie. + * 2. If no cookie is found, attempts to retrieve the fallback header `x-fallback-cookies`. + * - If this method is used, returns the header: `X-Debug-Fallback: true`. + * 3. Fetches the user document from the appropriate database based on the mode. + * 4. If the user document is empty or the session key cannot be verified, sets an empty user document. + * 5. Regardless of the results from steps 1-4, attempts to fetch the JWT token. + * 6. If the JWT user has a valid session ID, updates the user variable with the user from `projectDB`, + * overwriting the previous value. + */ + Authorization::setDefaultStatus(true); Auth::setCookieName('a_session_' . $project->getId()); diff --git a/composer.json b/composer.json index 5f40f4d593..f7510c21ce 100644 --- a/composer.json +++ b/composer.json @@ -47,6 +47,7 @@ "appwrite/php-runtimes": "0.17.*", "appwrite/php-clamav": "2.0.*", "utopia-php/abuse": "0.50.*", + "utopia-php/auth": "dev-dev", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.51.*", "utopia-php/cache": "0.11.*", diff --git a/composer.lock b/composer.lock index 3690e25ed0..992fad6dbf 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6883b3e81cfb0c5355997def668d5df2", + "content-hash": "e4697fc3967676a20b4891042adb391a", "packages": [ { "name": "adhocore/jwt", @@ -279,16 +279,16 @@ }, { "name": "brick/math", - "version": "0.12.2", + "version": "0.12.3", "source": { "type": "git", "url": "https://github.com/brick/math.git", - "reference": "901eddb1e45a8e0f689302e40af871c181ecbe40" + "reference": "866551da34e9a618e64a819ee1e01c20d8a588ba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/brick/math/zipball/901eddb1e45a8e0f689302e40af871c181ecbe40", - "reference": "901eddb1e45a8e0f689302e40af871c181ecbe40", + "url": "https://api.github.com/repos/brick/math/zipball/866551da34e9a618e64a819ee1e01c20d8a588ba", + "reference": "866551da34e9a618e64a819ee1e01c20d8a588ba", "shasum": "" }, "require": { @@ -327,7 +327,7 @@ ], "support": { "issues": "https://github.com/brick/math/issues", - "source": "https://github.com/brick/math/tree/0.12.2" + "source": "https://github.com/brick/math/tree/0.12.3" }, "funding": [ { @@ -335,7 +335,7 @@ "type": "github" } ], - "time": "2025-02-26T10:21:45+00:00" + "time": "2025-02-28T13:11:00+00:00" }, { "name": "chillerlan/php-qrcode", @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.29.3", + "version": "v4.30.0", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7" + "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7", - "reference": "ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/e1d66682f6836aa87820400f0aa07d9eb566feb6", + "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.3" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.0" }, - "time": "2025-01-08T21:00:13+00:00" + "time": "2025-03-04T22:54:49+00:00" }, { "name": "jean85/pretty-package-versions", @@ -1237,16 +1237,16 @@ }, { "name": "open-telemetry/api", - "version": "1.2.2", + "version": "1.2.3", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "8b925df3047628968bc5be722468db1b98b82d51" + "reference": "199d7ddda88f5f5619fa73463f1a5a7149ccd1f1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/8b925df3047628968bc5be722468db1b98b82d51", - "reference": "8b925df3047628968bc5be722468db1b98b82d51", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/199d7ddda88f5f5619fa73463f1a5a7149ccd1f1", + "reference": "199d7ddda88f5f5619fa73463f1a5a7149ccd1f1", "shasum": "" }, "require": { @@ -1303,7 +1303,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-02-03T21:49:11+00:00" + "time": "2025-03-05T21:42:54+00:00" }, { "name": "open-telemetry/context", @@ -1366,16 +1366,16 @@ }, { "name": "open-telemetry/exporter-otlp", - "version": "1.2.0", + "version": "1.2.1", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/exporter-otlp.git", - "reference": "243d9657c44a06f740cf384f486afe954c2b725f" + "reference": "b7580440b7481a98da97aceabeb46e1b276c8747" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/243d9657c44a06f740cf384f486afe954c2b725f", - "reference": "243d9657c44a06f740cf384f486afe954c2b725f", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/b7580440b7481a98da97aceabeb46e1b276c8747", + "reference": "b7580440b7481a98da97aceabeb46e1b276c8747", "shasum": "" }, "require": { @@ -1426,7 +1426,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-08T23:50:03+00:00" + "time": "2025-03-06T23:21:56+00:00" }, { "name": "open-telemetry/gen-otlp-protobuf", @@ -2371,16 +2371,16 @@ }, { "name": "ramsey/collection", - "version": "2.0.0", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/ramsey/collection.git", - "reference": "a4b48764bfbb8f3a6a4d1aeb1a35bb5e9ecac4a5" + "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ramsey/collection/zipball/a4b48764bfbb8f3a6a4d1aeb1a35bb5e9ecac4a5", - "reference": "a4b48764bfbb8f3a6a4d1aeb1a35bb5e9ecac4a5", + "url": "https://api.github.com/repos/ramsey/collection/zipball/3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", + "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", "shasum": "" }, "require": { @@ -2388,25 +2388,22 @@ }, "require-dev": { "captainhook/plugin-composer": "^5.3", - "ergebnis/composer-normalize": "^2.28.3", - "fakerphp/faker": "^1.21", + "ergebnis/composer-normalize": "^2.45", + "fakerphp/faker": "^1.24", "hamcrest/hamcrest-php": "^2.0", - "jangregor/phpstan-prophecy": "^1.0", - "mockery/mockery": "^1.5", + "jangregor/phpstan-prophecy": "^2.1", + "mockery/mockery": "^1.6", "php-parallel-lint/php-console-highlighter": "^1.0", - "php-parallel-lint/php-parallel-lint": "^1.3", - "phpcsstandards/phpcsutils": "^1.0.0-rc1", - "phpspec/prophecy-phpunit": "^2.0", - "phpstan/extension-installer": "^1.2", - "phpstan/phpstan": "^1.9", - "phpstan/phpstan-mockery": "^1.1", - "phpstan/phpstan-phpunit": "^1.3", - "phpunit/phpunit": "^9.5", - "psalm/plugin-mockery": "^1.1", - "psalm/plugin-phpunit": "^0.18.4", - "ramsey/coding-standard": "^2.0.3", - "ramsey/conventional-commits": "^1.3", - "vimeo/psalm": "^5.4" + "php-parallel-lint/php-parallel-lint": "^1.4", + "phpspec/prophecy-phpunit": "^2.3", + "phpstan/extension-installer": "^1.4", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-mockery": "^2.0", + "phpstan/phpstan-phpunit": "^2.0", + "phpunit/phpunit": "^10.5", + "ramsey/coding-standard": "^2.3", + "ramsey/conventional-commits": "^1.6", + "roave/security-advisories": "dev-latest" }, "type": "library", "extra": { @@ -2444,19 +2441,9 @@ ], "support": { "issues": "https://github.com/ramsey/collection/issues", - "source": "https://github.com/ramsey/collection/tree/2.0.0" + "source": "https://github.com/ramsey/collection/tree/2.1.0" }, - "funding": [ - { - "url": "https://github.com/ramsey", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/ramsey/collection", - "type": "tidelift" - } - ], - "time": "2022-12-31T21:50:55+00:00" + "time": "2025-03-02T04:48:29+00:00" }, { "name": "ramsey/uuid", @@ -3519,6 +3506,61 @@ }, "time": "2025-02-12T09:12:44+00:00" }, + { + "name": "utopia-php/auth", + "version": "dev-dev", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/auth.git", + "reference": "b063a2317c48cc6f3dba1eab0298641b19accdcd" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/auth/zipball/b063a2317c48cc6f3dba1eab0298641b19accdcd", + "reference": "b063a2317c48cc6f3dba1eab0298641b19accdcd", + "shasum": "" + }, + "require": { + "ext-hash": "*", + "ext-scrypt": "*", + "ext-sodium": "*", + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "1.2.*", + "phpstan/phpstan": "1.9.x-dev", + "phpunit/phpunit": "^9.3", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Auth\\": "src/Auth" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Utopia PHP", + "email": "team@appwrite.io" + } + ], + "description": "A simple PHP authentication library", + "keywords": [ + "Authentication", + "auth", + "php", + "security" + ], + "support": { + "issues": "https://github.com/utopia-php/auth/issues", + "source": "https://github.com/utopia-php/auth/tree/dev" + }, + "time": "2025-03-09T22:50:59+00:00" + }, { "name": "utopia-php/cache", "version": "0.11.0", @@ -3880,16 +3922,16 @@ }, { "name": "utopia-php/fetch", - "version": "0.3.0", + "version": "0.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/fetch.git", - "reference": "02b12c05aec13399dcc2da8d51f908e328ab63f4" + "reference": "524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/fetch/zipball/02b12c05aec13399dcc2da8d51f908e328ab63f4", - "reference": "02b12c05aec13399dcc2da8d51f908e328ab63f4", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0", + "reference": "524dd50afa8c64670c4fb18f1df4db9b5bb4b3d0", "shasum": "" }, "require": { @@ -3913,22 +3955,22 @@ "description": "A simple library that provides an interface for making HTTP Requests.", "support": { "issues": "https://github.com/utopia-php/fetch/issues", - "source": "https://github.com/utopia-php/fetch/tree/0.3.0" + "source": "https://github.com/utopia-php/fetch/tree/0.3.1" }, - "time": "2025-01-17T06:11:10+00:00" + "time": "2025-03-05T18:08:55+00:00" }, { "name": "utopia-php/framework", - "version": "0.33.17", + "version": "0.33.19", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644" + "reference": "64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/73fac6fbce9f56282dba4e52a58cf836ec434644", - "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644", + "url": "https://api.github.com/repos/utopia-php/http/zipball/64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0", + "reference": "64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0", "shasum": "" }, "require": { @@ -3960,9 +4002,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.17" + "source": "https://github.com/utopia-php/http/tree/0.33.19" }, - "time": "2025-02-24T17:35:48+00:00" + "time": "2025-03-06T11:37:49+00:00" }, { "name": "utopia-php/image", @@ -4607,22 +4649,24 @@ }, { "name": "utopia-php/storage", - "version": "0.18.9", + "version": "0.18.10", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c" + "reference": "76f31158f4251abb207f7a9b16f7cb0bfdb3b39e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/1cf455404e8700b3093fd73d74a38d41cdced90c", - "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/76f31158f4251abb207f7a9b16f7cb0bfdb3b39e", + "reference": "76f31158f4251abb207f7a9b16f7cb0bfdb3b39e", "shasum": "" }, "require": { "ext-brotli": "*", + "ext-curl": "*", "ext-fileinfo": "*", "ext-lz4": "*", + "ext-simplexml": "*", "ext-snappy": "*", "ext-xz": "*", "ext-zlib": "*", @@ -4656,9 +4700,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.9" + "source": "https://github.com/utopia-php/storage/tree/0.18.10" }, - "time": "2025-02-11T13:10:40+00:00" + "time": "2025-03-03T10:47:54+00:00" }, { "name": "utopia-php/swoole", @@ -5051,16 +5095,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.1", + "version": "0.40.2", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "df180676b6fbde7832ae1495af3e2f3e8f700837" + "reference": "56f09482d9e2f223911277ab887f197402708049" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/df180676b6fbde7832ae1495af3e2f3e8f700837", - "reference": "df180676b6fbde7832ae1495af3e2f3e8f700837", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/56f09482d9e2f223911277ab887f197402708049", + "reference": "56f09482d9e2f223911277ab887f197402708049", "shasum": "" }, "require": { @@ -5096,9 +5140,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.1" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.2" }, - "time": "2025-02-26T07:07:10+00:00" + "time": "2025-03-06T16:31:03+00:00" }, { "name": "doctrine/annotations", @@ -8806,7 +8850,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/auth": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -8830,5 +8876,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php new file mode 100644 index 0000000000..c9833adcfb --- /dev/null +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -0,0 +1,829 @@ +desc('Builds worker') + ->inject('message') + ->inject('project') + ->inject('dbForPlatform') + ->inject('queueForEvents') + ->inject('queueForFunctions') + ->inject('queueForStatsUsage') + ->inject('cache') + ->inject('dbForProject') + ->inject('deviceForFunctions') + ->inject('isResourceBlocked') + ->inject('log') + ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) => + $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log)); + } + + /** + * @param Message $message + * @param Document $project + * @param Database $dbForPlatform + * @param Event $queueForEvents + * @param Func $queueForFunctions + * @param StatsUsage $queueForStatsUsage + * @param Cache $cache + * @param Database $dbForProject + * @param Device $deviceForFunctions + * @param Log $log + * @return void + * @throws \Utopia\Database\Exception + */ + public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void + { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new \Exception('Missing payload'); + } + + $type = $payload['type'] ?? ''; + $resource = new Document($payload['resource'] ?? []); + $deployment = new Document($payload['deployment'] ?? []); + $template = new Document($payload['template'] ?? []); + + $log->addTag('projectId', $project->getId()); + $log->addTag('type', $type); + + switch ($type) { + case BUILD_TYPE_DEPLOYMENT: + case BUILD_TYPE_RETRY: + Console::info('Creating build for deployment: ' . $deployment->getId()); + $github = new GitHub($cache); + $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log); + break; + + default: + throw new \Exception('Invalid build type'); + } + } + + /** + * @param Device $deviceForFunctions + * @param Func $queueForFunctions + * @param Event $queueForEvents + * @param StatsUsage $queueForStatsUsage + * @param Database $dbForPlatform + * @param Database $dbForProject + * @param GitHub $github + * @param Document $project + * @param Document $function + * @param Document $deployment + * @param Document $template + * @param Log $log + * @return void + * @throws \Utopia\Database\Exception + * @throws Exception + */ + protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void + { + $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); + + $functionId = $function->getId(); + $log->addTag('functionId', $function->getId()); + + $function = $dbForProject->getDocument('functions', $functionId); + if ($function->isEmpty()) { + throw new \Exception('Function not found'); + } + + if ($isResourceBlocked($project, RESOURCE_TYPE_FUNCTIONS, $functionId)) { + throw new \Exception('Function blocked'); + } + + $deploymentId = $deployment->getId(); + $log->addTag('deploymentId', $deploymentId); + + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + if ($deployment->isEmpty()) { + throw new \Exception('Deployment not found'); + } + + if (empty($deployment->getAttribute('entrypoint', ''))) { + throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".'); + } + + $version = $function->getAttribute('version', 'v2'); + $spec = Config::getParam('runtime-specifications')[$function->getAttribute('specification', APP_FUNCTION_SPECIFICATION_DEFAULT)]; + $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); + $key = $function->getAttribute('runtime'); + $runtime = $runtimes[$key] ?? null; + if (\is_null($runtime)) { + throw new \Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); + } + + // Realtime preparation + $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [ + 'functionId' => $function->getId(), + 'deploymentId' => $deployment->getId() + ]); + + $startTime = DateTime::now(); + $durationStart = \microtime(true); + $buildId = $deployment->getAttribute('buildId', ''); + $build = $dbForProject->getDocument('builds', $buildId); + $isNewBuild = empty($buildId); + if ($build->isEmpty()) { + $buildId = ID::unique(); + $build = $dbForProject->createDocument('builds', new Document([ + '$id' => $buildId, + '$permissions' => [], + 'startTime' => $startTime, + 'deploymentInternalId' => $deployment->getInternalId(), + 'deploymentId' => $deployment->getId(), + 'status' => 'processing', + 'path' => '', + 'runtime' => $function->getAttribute('runtime'), + 'source' => $deployment->getAttribute('path', ''), + 'sourceType' => strtolower($deviceForFunctions->getType()), + 'logs' => '', + 'endTime' => null, + 'duration' => 0, + 'size' => 0 + ])); + + $deployment->setAttribute('buildId', $build->getId()); + $deployment->setAttribute('buildInternalId', $build->getInternalId()); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } elseif ($build->getAttribute('status') === 'canceled') { + Console::info('Build has been canceled'); + return; + } else { + $build = $dbForProject->getDocument('builds', $buildId); + } + + $source = $deployment->getAttribute('path', ''); + $installationId = $deployment->getAttribute('installationId', ''); + $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', ''); + $providerCommitHash = $deployment->getAttribute('providerCommitHash', ''); + $isVcsEnabled = !empty($providerRepositoryId); + $owner = ''; + $repositoryName = ''; + + if ($isVcsEnabled) { + $installation = $dbForPlatform->getDocument('installations', $installationId); + $providerInstallationId = $installation->getAttribute('providerInstallationId'); + $privateKey = System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'); + $githubAppId = System::getEnv('_APP_VCS_GITHUB_APP_ID'); + + $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); + } + + try { + if ($isNewBuild && !$isVcsEnabled) { + // Non-VCS + Template + $templateRepositoryName = $template->getAttribute('repositoryName', ''); + $templateOwnerName = $template->getAttribute('ownerName', ''); + $templateVersion = $template->getAttribute('version', ''); + + $templateRootDirectory = $template->getAttribute('rootDirectory', ''); + $templateRootDirectory = \rtrim($templateRootDirectory, '/'); + $templateRootDirectory = \ltrim($templateRootDirectory, '.'); + $templateRootDirectory = \ltrim($templateRootDirectory, '/'); + + if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateVersion)) { + $stdout = ''; + $stderr = ''; + + // Clone template repo + $tmpTemplateDirectory = '/tmp/builds/' . $buildId . '-template'; + $gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateVersion, GitHub::CLONE_TYPE_TAG, $tmpTemplateDirectory, $templateRootDirectory); + $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to clone code repository: ' . $stderr); + } + + Console::execute('find ' . \escapeshellarg($tmpTemplateDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr); + + // Ensure directories + Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); + + $tmpPathFile = $tmpTemplateDirectory . '/code.tar.gz'; + + $localDevice = new Local(); + + if (substr($tmpTemplateDirectory, -1) !== '/') { + $tmpTemplateDirectory .= '/'; + } + + $tarParamDirectory = \escapeshellarg($tmpTemplateDirectory . (empty($templateRootDirectory) ? '' : '/' . $templateRootDirectory)); + Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax + + $source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); + $result = $localDevice->transfer($tmpPathFile, $source, $deviceForFunctions); + + if (!$result) { + throw new \Exception("Unable to move file"); + } + + Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr); + + $directorySize = $deviceForFunctions->getFileSize($source); + $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source)); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize)); + } + } elseif ($isNewBuild && $isVcsEnabled) { + // VCS and VCS+Temaplte + $tmpDirectory = '/tmp/builds/' . $buildId . '/code'; + $rootDirectory = $function->getAttribute('providerRootDirectory', ''); + $rootDirectory = \rtrim($rootDirectory, '/'); + $rootDirectory = \ltrim($rootDirectory, '.'); + $rootDirectory = \ltrim($rootDirectory, '/'); + + $owner = $github->getOwnerName($providerInstallationId); + $repositoryName = $github->getRepositoryName($providerRepositoryId); + + $cloneOwner = $deployment->getAttribute('providerRepositoryOwner', $owner); + $cloneRepository = $deployment->getAttribute('providerRepositoryName', $repositoryName); + + $branchName = $deployment->getAttribute('providerBranch'); + $commitHash = $deployment->getAttribute('providerCommitHash', ''); + + $cloneVersion = $branchName; + $cloneType = GitHub::CLONE_TYPE_BRANCH; + if (!empty($commitHash)) { + $cloneVersion = $commitHash; + $cloneType = GitHub::CLONE_TYPE_COMMIT; + } + + $gitCloneCommand = $github->generateCloneCommand($cloneOwner, $cloneRepository, $cloneVersion, $cloneType, $tmpDirectory, $rootDirectory); + $stdout = ''; + $stderr = ''; + + Console::execute('mkdir -p ' . \escapeshellarg('/tmp/builds/' . $buildId), '', $stdout, $stderr); + + if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { + Console::info('Build has been canceled'); + return; + } + + $exit = Console::execute($gitCloneCommand, '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to clone code repository: ' . $stderr); + } + + // Local refactoring for function folder with spaces + if (str_contains($rootDirectory, ' ')) { + $rootDirectoryWithoutSpaces = str_replace(' ', '', $rootDirectory); + $from = $tmpDirectory . '/' . $rootDirectory; + $to = $tmpDirectory . '/' . $rootDirectoryWithoutSpaces; + $exit = Console::execute('mv "' . \escapeshellarg($from) . '" "' . \escapeshellarg($to) . '"', '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to move function with spaces' . $stderr); + } + $rootDirectory = $rootDirectoryWithoutSpaces; + } + + + // Build from template + $templateRepositoryName = $template->getAttribute('repositoryName', ''); + $templateOwnerName = $template->getAttribute('ownerName', ''); + $templateVersion = $template->getAttribute('version', ''); + + $templateRootDirectory = $template->getAttribute('rootDirectory', ''); + $templateRootDirectory = \rtrim($templateRootDirectory, '/'); + $templateRootDirectory = \ltrim($templateRootDirectory, '.'); + $templateRootDirectory = \ltrim($templateRootDirectory, '/'); + + if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateVersion)) { + // Clone template repo + $tmpTemplateDirectory = '/tmp/builds/' . $buildId . '/template'; + + $gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateVersion, GitHub::CLONE_TYPE_TAG, $tmpTemplateDirectory, $templateRootDirectory); + $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to clone code repository: ' . $stderr); + } + + // Ensure directories + Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); + Console::execute('mkdir -p ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); + + // Merge template into user repo + Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); + + // Commit and push + $exit = Console::execute('git config --global user.email "team@appwrite.io" && git config --global user.name "Appwrite" && cd ' . \escapeshellarg($tmpDirectory) . ' && git add . && git commit -m "Create ' . \escapeshellarg($function->getAttribute('name', '')) . ' function" && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to push code repository: ' . $stderr); + } + + $exit = Console::execute('cd ' . \escapeshellarg($tmpDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to get vcs commit SHA: ' . $stderr); + } + + $providerCommitHash = \trim($stdout); + $authorUrl = "https://github.com/$cloneOwner"; + + $deployment->setAttribute('providerCommitHash', $providerCommitHash ?? ''); + $deployment->setAttribute('providerCommitAuthorUrl', $authorUrl); + $deployment->setAttribute('providerCommitAuthor', 'Appwrite'); + $deployment->setAttribute('providerCommitMessage', "Create '" . $function->getAttribute('name', '') . "' function"); + $deployment->setAttribute('providerCommitUrl', "https://github.com/$cloneOwner/$cloneRepository/commit/$providerCommitHash"); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + + /** + * Send realtime Event + */ + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $build, + project: $project + ); + Realtime::send( + projectId: 'console', + payload: $build->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + } + + $tmpPath = '/tmp/builds/' . $buildId; + $tmpPathFile = $tmpPath . '/code.tar.gz'; + $localDevice = new Local(); + + if (substr($tmpDirectory, -1) !== '/') { + $tmpDirectory .= '/'; + } + + $directorySize = $localDevice->getDirectorySize($tmpDirectory); + $functionsSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000'); + if ($directorySize > $functionsSizeLimit) { + throw new \Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / 1048576, 2) . ' MBs.'); + } + + Console::execute('find ' . \escapeshellarg($tmpDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr); + + $tarParamDirectory = '/tmp/builds/' . $buildId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory); + Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax + + $source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); + $result = $localDevice->transfer($tmpPathFile, $source, $deviceForFunctions); + + if (!$result) { + throw new \Exception("Unable to move file"); + } + + Console::execute('rm -rf ' . \escapeshellarg($tmpPath), '', $stdout, $stderr); + + $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source)); + + $directorySize = $deviceForFunctions->getFileSize($source); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize)); + + $this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform); + } + + /** Request the executor to build the code... */ + $build->setAttribute('status', 'building'); + $build = $dbForProject->updateDocument('builds', $buildId, $build); + + if ($isVcsEnabled) { + $this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform); + } + + /** Trigger Webhook */ + $deploymentModel = new Deployment(); + $deploymentUpdate = + $queueForEvents + ->setQueue(Event::WEBHOOK_QUEUE_NAME) + ->setClass(Event::WEBHOOK_CLASS_NAME) + ->setProject($project) + ->setEvent('functions.[functionId].deployments.[deploymentId].update') + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))); + + $deploymentUpdate->trigger(); + + /** Trigger Functions */ + $queueForFunctions + ->from($deploymentUpdate) + ->trigger(); + + /** Trigger Realtime */ + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $build, + project: $project + ); + + Realtime::send( + projectId: 'console', + payload: $build->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + + $vars = []; + + // Shared vars + foreach ($function->getAttribute('varsProject', []) as $var) { + $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); + } + + // Function vars + foreach ($function->getAttribute('vars', []) as $var) { + $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); + } + + $cpus = $spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT; + $memory = max($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT, 1024); // We have a minimum of 1024MB here because some runtimes can't compile with less memory than this. + + $jwtExpiry = (int)System::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900); + $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0); + $apiKey = $jwtObj->encode([ + 'projectId' => $project->getId(), + 'scopes' => $function->getAttribute('scopes', []) + ]); + + $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; + $hostname = System::getEnv('_APP_DOMAIN'); + $endpoint = $protocol . '://' . $hostname . "/v1"; + + // Appwrite vars + $vars = \array_merge($vars, [ + 'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint, + 'APPWRITE_FUNCTION_API_KEY' => API_KEY_DYNAMIC . '_' . $apiKey, + 'APPWRITE_FUNCTION_ID' => $function->getId(), + 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), + 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), + 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), + 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', + 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', + 'APPWRITE_FUNCTION_CPUS' => $cpus, + 'APPWRITE_FUNCTION_MEMORY' => $memory, + 'APPWRITE_VERSION' => APP_VERSION_STABLE, + 'APPWRITE_REGION' => $project->getAttribute('region'), + 'APPWRITE_DEPLOYMENT_TYPE' => $deployment->getAttribute('type', ''), + 'APPWRITE_VCS_REPOSITORY_ID' => $deployment->getAttribute('providerRepositoryId', ''), + 'APPWRITE_VCS_REPOSITORY_NAME' => $deployment->getAttribute('providerRepositoryName', ''), + 'APPWRITE_VCS_REPOSITORY_OWNER' => $deployment->getAttribute('providerRepositoryOwner', ''), + 'APPWRITE_VCS_REPOSITORY_URL' => $deployment->getAttribute('providerRepositoryUrl', ''), + 'APPWRITE_VCS_REPOSITORY_BRANCH' => $deployment->getAttribute('providerBranch', ''), + 'APPWRITE_VCS_REPOSITORY_BRANCH_URL' => $deployment->getAttribute('providerBranchUrl', ''), + 'APPWRITE_VCS_COMMIT_HASH' => $deployment->getAttribute('providerCommitHash', ''), + 'APPWRITE_VCS_COMMIT_MESSAGE' => $deployment->getAttribute('providerCommitMessage', ''), + 'APPWRITE_VCS_COMMIT_URL' => $deployment->getAttribute('providerCommitUrl', ''), + 'APPWRITE_VCS_COMMIT_AUTHOR_NAME' => $deployment->getAttribute('providerCommitAuthor', ''), + 'APPWRITE_VCS_COMMIT_AUTHOR_URL' => $deployment->getAttribute('providerCommitAuthorUrl', ''), + 'APPWRITE_VCS_ROOT_DIRECTORY' => $deployment->getAttribute('providerRootDirectory', ''), + ]); + + $command = $deployment->getAttribute('commands', ''); + + $response = null; + $err = null; + + if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { + Console::info('Build has been canceled'); + return; + } + + $isCanceled = false; + + Co::join([ + Co\go(function () use ($executor, &$response, $project, $deployment, $source, $function, $runtime, $vars, $command, $cpus, $memory, &$err) { + try { + $version = $function->getAttribute('version', 'v2'); + $command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . \trim(\escapeshellarg($command), "\'") . '"'; + + $response = $executor->createRuntime( + deploymentId: $deployment->getId(), + projectId: $project->getId(), + source: $source, + image: $runtime['image'], + version: $version, + cpus: $cpus, + memory: $memory, + remove: true, + entrypoint: $deployment->getAttribute('entrypoint'), + destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", + variables: $vars, + command: $command + ); + } catch (\Throwable $error) { + $err = $error; + } + }), + Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err, &$isCanceled) { + try { + $executor->getLogs( + deploymentId: $deployment->getId(), + projectId: $project->getId(), + callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $allEvents, $project, &$isCanceled) { + if ($isCanceled) { + return; + } + + // If we have response or error from concurrent coroutine, we already have latest logs + if ($response === null && $err === null) { + $build = $dbForProject->getDocument('builds', $build->getId()); + + if ($build->isEmpty()) { + throw new \Exception('Build not found'); + } + + if ($build->getAttribute('status') === 'canceled') { + $isCanceled = true; + Console::info('Ignoring realtime logs because build has been canceled'); + return; + } + + $logs = \mb_substr($logs, 0, null, 'UTF-8'); // Get only valid UTF8 part - removes leftover half-multibytes causing SQL errors + + $build = $build->setAttribute('logs', $build->getAttribute('logs', '') . $logs); + $build = $dbForProject->updateDocument('builds', $build->getId(), $build); + + /** + * Send realtime Event + */ + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $build, + project: $project + ); + Realtime::send( + projectId: 'console', + payload: $build->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + } + } + ); + } catch (\Throwable $error) { + if (empty($err)) { + $err = $error; + } + } + }), + ]); + + if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { + Console::info('Build has been canceled'); + return; + } + + if ($err) { + throw $err; + } + + $endTime = DateTime::now(); + $durationEnd = \microtime(true); + + $buildSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_BUILD_SIZE_LIMIT', '2000000000'); + if ($response['size'] > $buildSizeLimit) { + throw new \Exception('Build size should be less than ' . number_format($buildSizeLimit / 1048576, 2) . ' MBs.'); + } + + /** Update the build document */ + $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); + $build->setAttribute('endTime', $endTime); + $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); + $build->setAttribute('status', 'ready'); + $build->setAttribute('path', $response['path']); + $build->setAttribute('size', $response['size']); + $build->setAttribute('logs', $response['output']); + + $build = $dbForProject->updateDocument('builds', $buildId, $build); + + if ($isVcsEnabled) { + $this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform); + } + + Console::success("Build id: $buildId created"); + + /** Set auto deploy */ + if ($deployment->getAttribute('activate') === true) { + $function->setAttribute('deploymentInternalId', $deployment->getInternalId()); + $function->setAttribute('deployment', $deployment->getId()); + $function->setAttribute('live', true); + $function = $dbForProject->updateDocument('functions', $function->getId(), $function); + } + + if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { + Console::info('Build has been canceled'); + return; + } + + /** Update function schedule */ + + // Inform scheduler if function is still active + $schedule = $dbForPlatform->getDocument('schedules', $function->getAttribute('scheduleId')); + $schedule + ->setAttribute('resourceUpdatedAt', DateTime::now()) + ->setAttribute('schedule', $function->getAttribute('schedule')) + ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); + Authorization::skip(fn () => $dbForPlatform->updateDocument('schedules', $schedule->getId(), $schedule)); + } catch (\Throwable $th) { + if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { + Console::info('Build has been canceled'); + return; + } + + $endTime = DateTime::now(); + $durationEnd = \microtime(true); + $build->setAttribute('endTime', $endTime); + $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); + $build->setAttribute('status', 'failed'); + $build->setAttribute('logs', $th->getMessage()); + + $build = $dbForProject->updateDocument('builds', $buildId, $build); + + if ($isVcsEnabled) { + $this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform); + } + } finally { + /** + * Send realtime Event + */ + $target = Realtime::fromPayload( + // Pass first, most verbose event pattern + event: $allEvents[0], + payload: $build, + project: $project + ); + Realtime::send( + projectId: 'console', + payload: $build->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'] + ); + + /** Trigger usage queue */ + if ($build->getAttribute('status') === 'ready') { + $queueForStatsUsage + ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project + ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000); + } elseif ($build->getAttribute('status') === 'failed') { + $queueForStatsUsage + ->addMetric(METRIC_BUILDS_FAILED, 1) // per project + ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED), 1) // per function + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED), (int)$build->getAttribute('duration', 0) * 1000); + } + + $queueForStatsUsage + ->addMetric(METRIC_BUILDS, 1) // per project + ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) + ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) + ->setProject($project) + ->trigger(); + } + } + + /** + * @param string $status + * @param GitHub $github + * @param string $providerCommitHash + * @param string $owner + * @param string $repositoryName + * @param Document $project + * @param Document $function + * @param string $deploymentId + * @param Database $dbForProject + * @param Database $dbForPlatform + * @return void + * @throws Structure + * @throws \Utopia\Database\Exception + * @throws Authorization + * @throws Conflict + * @throws Restricted + */ + protected function runGitAction(string $status, GitHub $github, string $providerCommitHash, string $owner, string $repositoryName, Document $project, Document $function, string $deploymentId, Database $dbForProject, Database $dbForPlatform): void + { + if ($function->getAttribute('providerSilentMode', false) === true) { + return; + } + + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + $commentId = $deployment->getAttribute('providerCommentId', ''); + + if (!empty($providerCommitHash)) { + $message = match ($status) { + 'ready' => 'Build succeeded.', + 'failed' => 'Build failed.', + 'processing' => 'Building...', + default => $status + }; + + $state = match ($status) { + 'ready' => 'success', + 'failed' => 'failure', + 'processing' => 'pending', + default => $status + }; + + $functionName = $function->getAttribute('name'); + $projectName = $project->getAttribute('name'); + + $name = "{$functionName} ({$projectName})"; + + $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; + $hostname = System::getEnv('_APP_DOMAIN'); + $functionId = $function->getId(); + $projectId = $project->getId(); + $providerTargetUrl = $protocol . '://' . $hostname . "/console/project-$projectId/functions/function-$functionId"; + + $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, $state, $message, $providerTargetUrl, $name); + } + + if (!empty($commentId)) { + $retries = 0; + + while (true) { + $retries++; + + try { + $dbForPlatform->createDocument('vcsCommentLocks', new Document([ + '$id' => $commentId + ])); + break; + } catch (\Throwable $err) { + if ($retries >= 9) { + throw $err; + } + + \sleep(1); + } + } + + // Wrap in try/finally to ensure lock file gets deleted + try { + $comment = new Comment(); + $comment->parseComment($github->getComment($owner, $repositoryName, $commentId)); + $comment->addBuild($project, $function, $status, $deployment->getId(), ['type' => 'logs']); + $github->updateComment($owner, $repositoryName, $commentId, $comment->generateComment()); + } finally { + $dbForPlatform->deleteDocument('vcsCommentLocks', $commentId); + } + } + } +} From fadb24f29541a8a43768bae800844385b92b76eb Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 10 Mar 2025 00:21:49 +0100 Subject: [PATCH 003/245] revert change to build --- src/Appwrite/Platform/Workers/Builds.php | 829 ----------------------- 1 file changed, 829 deletions(-) delete mode 100644 src/Appwrite/Platform/Workers/Builds.php diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php deleted file mode 100644 index c9833adcfb..0000000000 --- a/src/Appwrite/Platform/Workers/Builds.php +++ /dev/null @@ -1,829 +0,0 @@ -desc('Builds worker') - ->inject('message') - ->inject('project') - ->inject('dbForPlatform') - ->inject('queueForEvents') - ->inject('queueForFunctions') - ->inject('queueForStatsUsage') - ->inject('cache') - ->inject('dbForProject') - ->inject('deviceForFunctions') - ->inject('isResourceBlocked') - ->inject('log') - ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) => - $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log)); - } - - /** - * @param Message $message - * @param Document $project - * @param Database $dbForPlatform - * @param Event $queueForEvents - * @param Func $queueForFunctions - * @param StatsUsage $queueForStatsUsage - * @param Cache $cache - * @param Database $dbForProject - * @param Device $deviceForFunctions - * @param Log $log - * @return void - * @throws \Utopia\Database\Exception - */ - public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void - { - $payload = $message->getPayload() ?? []; - - if (empty($payload)) { - throw new \Exception('Missing payload'); - } - - $type = $payload['type'] ?? ''; - $resource = new Document($payload['resource'] ?? []); - $deployment = new Document($payload['deployment'] ?? []); - $template = new Document($payload['template'] ?? []); - - $log->addTag('projectId', $project->getId()); - $log->addTag('type', $type); - - switch ($type) { - case BUILD_TYPE_DEPLOYMENT: - case BUILD_TYPE_RETRY: - Console::info('Creating build for deployment: ' . $deployment->getId()); - $github = new GitHub($cache); - $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log); - break; - - default: - throw new \Exception('Invalid build type'); - } - } - - /** - * @param Device $deviceForFunctions - * @param Func $queueForFunctions - * @param Event $queueForEvents - * @param StatsUsage $queueForStatsUsage - * @param Database $dbForPlatform - * @param Database $dbForProject - * @param GitHub $github - * @param Document $project - * @param Document $function - * @param Document $deployment - * @param Document $template - * @param Log $log - * @return void - * @throws \Utopia\Database\Exception - * @throws Exception - */ - protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void - { - $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); - - $functionId = $function->getId(); - $log->addTag('functionId', $function->getId()); - - $function = $dbForProject->getDocument('functions', $functionId); - if ($function->isEmpty()) { - throw new \Exception('Function not found'); - } - - if ($isResourceBlocked($project, RESOURCE_TYPE_FUNCTIONS, $functionId)) { - throw new \Exception('Function blocked'); - } - - $deploymentId = $deployment->getId(); - $log->addTag('deploymentId', $deploymentId); - - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - if ($deployment->isEmpty()) { - throw new \Exception('Deployment not found'); - } - - if (empty($deployment->getAttribute('entrypoint', ''))) { - throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".'); - } - - $version = $function->getAttribute('version', 'v2'); - $spec = Config::getParam('runtime-specifications')[$function->getAttribute('specification', APP_FUNCTION_SPECIFICATION_DEFAULT)]; - $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - $key = $function->getAttribute('runtime'); - $runtime = $runtimes[$key] ?? null; - if (\is_null($runtime)) { - throw new \Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); - } - - // Realtime preparation - $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [ - 'functionId' => $function->getId(), - 'deploymentId' => $deployment->getId() - ]); - - $startTime = DateTime::now(); - $durationStart = \microtime(true); - $buildId = $deployment->getAttribute('buildId', ''); - $build = $dbForProject->getDocument('builds', $buildId); - $isNewBuild = empty($buildId); - if ($build->isEmpty()) { - $buildId = ID::unique(); - $build = $dbForProject->createDocument('builds', new Document([ - '$id' => $buildId, - '$permissions' => [], - 'startTime' => $startTime, - 'deploymentInternalId' => $deployment->getInternalId(), - 'deploymentId' => $deployment->getId(), - 'status' => 'processing', - 'path' => '', - 'runtime' => $function->getAttribute('runtime'), - 'source' => $deployment->getAttribute('path', ''), - 'sourceType' => strtolower($deviceForFunctions->getType()), - 'logs' => '', - 'endTime' => null, - 'duration' => 0, - 'size' => 0 - ])); - - $deployment->setAttribute('buildId', $build->getId()); - $deployment->setAttribute('buildInternalId', $build->getInternalId()); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - } elseif ($build->getAttribute('status') === 'canceled') { - Console::info('Build has been canceled'); - return; - } else { - $build = $dbForProject->getDocument('builds', $buildId); - } - - $source = $deployment->getAttribute('path', ''); - $installationId = $deployment->getAttribute('installationId', ''); - $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', ''); - $providerCommitHash = $deployment->getAttribute('providerCommitHash', ''); - $isVcsEnabled = !empty($providerRepositoryId); - $owner = ''; - $repositoryName = ''; - - if ($isVcsEnabled) { - $installation = $dbForPlatform->getDocument('installations', $installationId); - $providerInstallationId = $installation->getAttribute('providerInstallationId'); - $privateKey = System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'); - $githubAppId = System::getEnv('_APP_VCS_GITHUB_APP_ID'); - - $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); - } - - try { - if ($isNewBuild && !$isVcsEnabled) { - // Non-VCS + Template - $templateRepositoryName = $template->getAttribute('repositoryName', ''); - $templateOwnerName = $template->getAttribute('ownerName', ''); - $templateVersion = $template->getAttribute('version', ''); - - $templateRootDirectory = $template->getAttribute('rootDirectory', ''); - $templateRootDirectory = \rtrim($templateRootDirectory, '/'); - $templateRootDirectory = \ltrim($templateRootDirectory, '.'); - $templateRootDirectory = \ltrim($templateRootDirectory, '/'); - - if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateVersion)) { - $stdout = ''; - $stderr = ''; - - // Clone template repo - $tmpTemplateDirectory = '/tmp/builds/' . $buildId . '-template'; - $gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateVersion, GitHub::CLONE_TYPE_TAG, $tmpTemplateDirectory, $templateRootDirectory); - $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); - - if ($exit !== 0) { - throw new \Exception('Unable to clone code repository: ' . $stderr); - } - - Console::execute('find ' . \escapeshellarg($tmpTemplateDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr); - - // Ensure directories - Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); - - $tmpPathFile = $tmpTemplateDirectory . '/code.tar.gz'; - - $localDevice = new Local(); - - if (substr($tmpTemplateDirectory, -1) !== '/') { - $tmpTemplateDirectory .= '/'; - } - - $tarParamDirectory = \escapeshellarg($tmpTemplateDirectory . (empty($templateRootDirectory) ? '' : '/' . $templateRootDirectory)); - Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax - - $source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $result = $localDevice->transfer($tmpPathFile, $source, $deviceForFunctions); - - if (!$result) { - throw new \Exception("Unable to move file"); - } - - Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr); - - $directorySize = $deviceForFunctions->getFileSize($source); - $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source)); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize)); - } - } elseif ($isNewBuild && $isVcsEnabled) { - // VCS and VCS+Temaplte - $tmpDirectory = '/tmp/builds/' . $buildId . '/code'; - $rootDirectory = $function->getAttribute('providerRootDirectory', ''); - $rootDirectory = \rtrim($rootDirectory, '/'); - $rootDirectory = \ltrim($rootDirectory, '.'); - $rootDirectory = \ltrim($rootDirectory, '/'); - - $owner = $github->getOwnerName($providerInstallationId); - $repositoryName = $github->getRepositoryName($providerRepositoryId); - - $cloneOwner = $deployment->getAttribute('providerRepositoryOwner', $owner); - $cloneRepository = $deployment->getAttribute('providerRepositoryName', $repositoryName); - - $branchName = $deployment->getAttribute('providerBranch'); - $commitHash = $deployment->getAttribute('providerCommitHash', ''); - - $cloneVersion = $branchName; - $cloneType = GitHub::CLONE_TYPE_BRANCH; - if (!empty($commitHash)) { - $cloneVersion = $commitHash; - $cloneType = GitHub::CLONE_TYPE_COMMIT; - } - - $gitCloneCommand = $github->generateCloneCommand($cloneOwner, $cloneRepository, $cloneVersion, $cloneType, $tmpDirectory, $rootDirectory); - $stdout = ''; - $stderr = ''; - - Console::execute('mkdir -p ' . \escapeshellarg('/tmp/builds/' . $buildId), '', $stdout, $stderr); - - if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { - Console::info('Build has been canceled'); - return; - } - - $exit = Console::execute($gitCloneCommand, '', $stdout, $stderr); - - if ($exit !== 0) { - throw new \Exception('Unable to clone code repository: ' . $stderr); - } - - // Local refactoring for function folder with spaces - if (str_contains($rootDirectory, ' ')) { - $rootDirectoryWithoutSpaces = str_replace(' ', '', $rootDirectory); - $from = $tmpDirectory . '/' . $rootDirectory; - $to = $tmpDirectory . '/' . $rootDirectoryWithoutSpaces; - $exit = Console::execute('mv "' . \escapeshellarg($from) . '" "' . \escapeshellarg($to) . '"', '', $stdout, $stderr); - - if ($exit !== 0) { - throw new \Exception('Unable to move function with spaces' . $stderr); - } - $rootDirectory = $rootDirectoryWithoutSpaces; - } - - - // Build from template - $templateRepositoryName = $template->getAttribute('repositoryName', ''); - $templateOwnerName = $template->getAttribute('ownerName', ''); - $templateVersion = $template->getAttribute('version', ''); - - $templateRootDirectory = $template->getAttribute('rootDirectory', ''); - $templateRootDirectory = \rtrim($templateRootDirectory, '/'); - $templateRootDirectory = \ltrim($templateRootDirectory, '.'); - $templateRootDirectory = \ltrim($templateRootDirectory, '/'); - - if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateVersion)) { - // Clone template repo - $tmpTemplateDirectory = '/tmp/builds/' . $buildId . '/template'; - - $gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateVersion, GitHub::CLONE_TYPE_TAG, $tmpTemplateDirectory, $templateRootDirectory); - $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); - - if ($exit !== 0) { - throw new \Exception('Unable to clone code repository: ' . $stderr); - } - - // Ensure directories - Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); - Console::execute('mkdir -p ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - - // Merge template into user repo - Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - - // Commit and push - $exit = Console::execute('git config --global user.email "team@appwrite.io" && git config --global user.name "Appwrite" && cd ' . \escapeshellarg($tmpDirectory) . ' && git add . && git commit -m "Create ' . \escapeshellarg($function->getAttribute('name', '')) . ' function" && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); - - if ($exit !== 0) { - throw new \Exception('Unable to push code repository: ' . $stderr); - } - - $exit = Console::execute('cd ' . \escapeshellarg($tmpDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr); - - if ($exit !== 0) { - throw new \Exception('Unable to get vcs commit SHA: ' . $stderr); - } - - $providerCommitHash = \trim($stdout); - $authorUrl = "https://github.com/$cloneOwner"; - - $deployment->setAttribute('providerCommitHash', $providerCommitHash ?? ''); - $deployment->setAttribute('providerCommitAuthorUrl', $authorUrl); - $deployment->setAttribute('providerCommitAuthor', 'Appwrite'); - $deployment->setAttribute('providerCommitMessage', "Create '" . $function->getAttribute('name', '') . "' function"); - $deployment->setAttribute('providerCommitUrl', "https://github.com/$cloneOwner/$cloneRepository/commit/$providerCommitHash"); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - - /** - * Send realtime Event - */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - } - - $tmpPath = '/tmp/builds/' . $buildId; - $tmpPathFile = $tmpPath . '/code.tar.gz'; - $localDevice = new Local(); - - if (substr($tmpDirectory, -1) !== '/') { - $tmpDirectory .= '/'; - } - - $directorySize = $localDevice->getDirectorySize($tmpDirectory); - $functionsSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000'); - if ($directorySize > $functionsSizeLimit) { - throw new \Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / 1048576, 2) . ' MBs.'); - } - - Console::execute('find ' . \escapeshellarg($tmpDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr); - - $tarParamDirectory = '/tmp/builds/' . $buildId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory); - Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax - - $source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $result = $localDevice->transfer($tmpPathFile, $source, $deviceForFunctions); - - if (!$result) { - throw new \Exception("Unable to move file"); - } - - Console::execute('rm -rf ' . \escapeshellarg($tmpPath), '', $stdout, $stderr); - - $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source)); - - $directorySize = $deviceForFunctions->getFileSize($source); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize)); - - $this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform); - } - - /** Request the executor to build the code... */ - $build->setAttribute('status', 'building'); - $build = $dbForProject->updateDocument('builds', $buildId, $build); - - if ($isVcsEnabled) { - $this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform); - } - - /** Trigger Webhook */ - $deploymentModel = new Deployment(); - $deploymentUpdate = - $queueForEvents - ->setQueue(Event::WEBHOOK_QUEUE_NAME) - ->setClass(Event::WEBHOOK_CLASS_NAME) - ->setProject($project) - ->setEvent('functions.[functionId].deployments.[deploymentId].update') - ->setParam('functionId', $function->getId()) - ->setParam('deploymentId', $deployment->getId()) - ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))); - - $deploymentUpdate->trigger(); - - /** Trigger Functions */ - $queueForFunctions - ->from($deploymentUpdate) - ->trigger(); - - /** Trigger Realtime */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - - $vars = []; - - // Shared vars - foreach ($function->getAttribute('varsProject', []) as $var) { - $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); - } - - // Function vars - foreach ($function->getAttribute('vars', []) as $var) { - $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); - } - - $cpus = $spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT; - $memory = max($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT, 1024); // We have a minimum of 1024MB here because some runtimes can't compile with less memory than this. - - $jwtExpiry = (int)System::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900); - $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0); - $apiKey = $jwtObj->encode([ - 'projectId' => $project->getId(), - 'scopes' => $function->getAttribute('scopes', []) - ]); - - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $hostname = System::getEnv('_APP_DOMAIN'); - $endpoint = $protocol . '://' . $hostname . "/v1"; - - // Appwrite vars - $vars = \array_merge($vars, [ - 'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint, - 'APPWRITE_FUNCTION_API_KEY' => API_KEY_DYNAMIC . '_' . $apiKey, - 'APPWRITE_FUNCTION_ID' => $function->getId(), - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), - 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), - 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_FUNCTION_CPUS' => $cpus, - 'APPWRITE_FUNCTION_MEMORY' => $memory, - 'APPWRITE_VERSION' => APP_VERSION_STABLE, - 'APPWRITE_REGION' => $project->getAttribute('region'), - 'APPWRITE_DEPLOYMENT_TYPE' => $deployment->getAttribute('type', ''), - 'APPWRITE_VCS_REPOSITORY_ID' => $deployment->getAttribute('providerRepositoryId', ''), - 'APPWRITE_VCS_REPOSITORY_NAME' => $deployment->getAttribute('providerRepositoryName', ''), - 'APPWRITE_VCS_REPOSITORY_OWNER' => $deployment->getAttribute('providerRepositoryOwner', ''), - 'APPWRITE_VCS_REPOSITORY_URL' => $deployment->getAttribute('providerRepositoryUrl', ''), - 'APPWRITE_VCS_REPOSITORY_BRANCH' => $deployment->getAttribute('providerBranch', ''), - 'APPWRITE_VCS_REPOSITORY_BRANCH_URL' => $deployment->getAttribute('providerBranchUrl', ''), - 'APPWRITE_VCS_COMMIT_HASH' => $deployment->getAttribute('providerCommitHash', ''), - 'APPWRITE_VCS_COMMIT_MESSAGE' => $deployment->getAttribute('providerCommitMessage', ''), - 'APPWRITE_VCS_COMMIT_URL' => $deployment->getAttribute('providerCommitUrl', ''), - 'APPWRITE_VCS_COMMIT_AUTHOR_NAME' => $deployment->getAttribute('providerCommitAuthor', ''), - 'APPWRITE_VCS_COMMIT_AUTHOR_URL' => $deployment->getAttribute('providerCommitAuthorUrl', ''), - 'APPWRITE_VCS_ROOT_DIRECTORY' => $deployment->getAttribute('providerRootDirectory', ''), - ]); - - $command = $deployment->getAttribute('commands', ''); - - $response = null; - $err = null; - - if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { - Console::info('Build has been canceled'); - return; - } - - $isCanceled = false; - - Co::join([ - Co\go(function () use ($executor, &$response, $project, $deployment, $source, $function, $runtime, $vars, $command, $cpus, $memory, &$err) { - try { - $version = $function->getAttribute('version', 'v2'); - $command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . \trim(\escapeshellarg($command), "\'") . '"'; - - $response = $executor->createRuntime( - deploymentId: $deployment->getId(), - projectId: $project->getId(), - source: $source, - image: $runtime['image'], - version: $version, - cpus: $cpus, - memory: $memory, - remove: true, - entrypoint: $deployment->getAttribute('entrypoint'), - destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", - variables: $vars, - command: $command - ); - } catch (\Throwable $error) { - $err = $error; - } - }), - Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err, &$isCanceled) { - try { - $executor->getLogs( - deploymentId: $deployment->getId(), - projectId: $project->getId(), - callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $allEvents, $project, &$isCanceled) { - if ($isCanceled) { - return; - } - - // If we have response or error from concurrent coroutine, we already have latest logs - if ($response === null && $err === null) { - $build = $dbForProject->getDocument('builds', $build->getId()); - - if ($build->isEmpty()) { - throw new \Exception('Build not found'); - } - - if ($build->getAttribute('status') === 'canceled') { - $isCanceled = true; - Console::info('Ignoring realtime logs because build has been canceled'); - return; - } - - $logs = \mb_substr($logs, 0, null, 'UTF-8'); // Get only valid UTF8 part - removes leftover half-multibytes causing SQL errors - - $build = $build->setAttribute('logs', $build->getAttribute('logs', '') . $logs); - $build = $dbForProject->updateDocument('builds', $build->getId(), $build); - - /** - * Send realtime Event - */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - } - } - ); - } catch (\Throwable $error) { - if (empty($err)) { - $err = $error; - } - } - }), - ]); - - if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { - Console::info('Build has been canceled'); - return; - } - - if ($err) { - throw $err; - } - - $endTime = DateTime::now(); - $durationEnd = \microtime(true); - - $buildSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_BUILD_SIZE_LIMIT', '2000000000'); - if ($response['size'] > $buildSizeLimit) { - throw new \Exception('Build size should be less than ' . number_format($buildSizeLimit / 1048576, 2) . ' MBs.'); - } - - /** Update the build document */ - $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); - $build->setAttribute('endTime', $endTime); - $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); - $build->setAttribute('status', 'ready'); - $build->setAttribute('path', $response['path']); - $build->setAttribute('size', $response['size']); - $build->setAttribute('logs', $response['output']); - - $build = $dbForProject->updateDocument('builds', $buildId, $build); - - if ($isVcsEnabled) { - $this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform); - } - - Console::success("Build id: $buildId created"); - - /** Set auto deploy */ - if ($deployment->getAttribute('activate') === true) { - $function->setAttribute('deploymentInternalId', $deployment->getInternalId()); - $function->setAttribute('deployment', $deployment->getId()); - $function->setAttribute('live', true); - $function = $dbForProject->updateDocument('functions', $function->getId(), $function); - } - - if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { - Console::info('Build has been canceled'); - return; - } - - /** Update function schedule */ - - // Inform scheduler if function is still active - $schedule = $dbForPlatform->getDocument('schedules', $function->getAttribute('scheduleId')); - $schedule - ->setAttribute('resourceUpdatedAt', DateTime::now()) - ->setAttribute('schedule', $function->getAttribute('schedule')) - ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment'))); - Authorization::skip(fn () => $dbForPlatform->updateDocument('schedules', $schedule->getId(), $schedule)); - } catch (\Throwable $th) { - if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { - Console::info('Build has been canceled'); - return; - } - - $endTime = DateTime::now(); - $durationEnd = \microtime(true); - $build->setAttribute('endTime', $endTime); - $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); - $build->setAttribute('status', 'failed'); - $build->setAttribute('logs', $th->getMessage()); - - $build = $dbForProject->updateDocument('builds', $buildId, $build); - - if ($isVcsEnabled) { - $this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform); - } - } finally { - /** - * Send realtime Event - */ - $target = Realtime::fromPayload( - // Pass first, most verbose event pattern - event: $allEvents[0], - payload: $build, - project: $project - ); - Realtime::send( - projectId: 'console', - payload: $build->getArrayCopy(), - events: $allEvents, - channels: $target['channels'], - roles: $target['roles'] - ); - - /** Trigger usage queue */ - if ($build->getAttribute('status') === 'ready') { - $queueForStatsUsage - ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000); - } elseif ($build->getAttribute('status') === 'failed') { - $queueForStatsUsage - ->addMetric(METRIC_BUILDS_FAILED, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED), 1) // per function - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED), (int)$build->getAttribute('duration', 0) * 1000); - } - - $queueForStatsUsage - ->addMetric(METRIC_BUILDS, 1) // per project - ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) - ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->setProject($project) - ->trigger(); - } - } - - /** - * @param string $status - * @param GitHub $github - * @param string $providerCommitHash - * @param string $owner - * @param string $repositoryName - * @param Document $project - * @param Document $function - * @param string $deploymentId - * @param Database $dbForProject - * @param Database $dbForPlatform - * @return void - * @throws Structure - * @throws \Utopia\Database\Exception - * @throws Authorization - * @throws Conflict - * @throws Restricted - */ - protected function runGitAction(string $status, GitHub $github, string $providerCommitHash, string $owner, string $repositoryName, Document $project, Document $function, string $deploymentId, Database $dbForProject, Database $dbForPlatform): void - { - if ($function->getAttribute('providerSilentMode', false) === true) { - return; - } - - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - $commentId = $deployment->getAttribute('providerCommentId', ''); - - if (!empty($providerCommitHash)) { - $message = match ($status) { - 'ready' => 'Build succeeded.', - 'failed' => 'Build failed.', - 'processing' => 'Building...', - default => $status - }; - - $state = match ($status) { - 'ready' => 'success', - 'failed' => 'failure', - 'processing' => 'pending', - default => $status - }; - - $functionName = $function->getAttribute('name'); - $projectName = $project->getAttribute('name'); - - $name = "{$functionName} ({$projectName})"; - - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $hostname = System::getEnv('_APP_DOMAIN'); - $functionId = $function->getId(); - $projectId = $project->getId(); - $providerTargetUrl = $protocol . '://' . $hostname . "/console/project-$projectId/functions/function-$functionId"; - - $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, $state, $message, $providerTargetUrl, $name); - } - - if (!empty($commentId)) { - $retries = 0; - - while (true) { - $retries++; - - try { - $dbForPlatform->createDocument('vcsCommentLocks', new Document([ - '$id' => $commentId - ])); - break; - } catch (\Throwable $err) { - if ($retries >= 9) { - throw $err; - } - - \sleep(1); - } - } - - // Wrap in try/finally to ensure lock file gets deleted - try { - $comment = new Comment(); - $comment->parseComment($github->getComment($owner, $repositoryName, $commentId)); - $comment->addBuild($project, $function, $status, $deployment->getId(), ['type' => 'logs']); - $github->updateComment($owner, $repositoryName, $commentId, $comment->generateComment()); - } finally { - $dbForPlatform->deleteDocument('vcsCommentLocks', $commentId); - } - } - } -} From a058925252d79e302d3ea86701df1e7a35cb55e3 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 10 Mar 2025 00:23:22 +0100 Subject: [PATCH 004/245] Code formatting --- app/controllers/api/users.php | 19 +++++++++---------- app/controllers/shared/api.php | 12 ++++++------ 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 4a0c6013ed..31b6e61bad 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -29,6 +29,15 @@ use MaxMind\Db\Reader; use Utopia\App; use Utopia\Audit\Audit; use Utopia\Auth\Hash; +use Utopia\Auth\Hashes\Argon2; +use Utopia\Auth\Hashes\Bcrypt; +use Utopia\Auth\Hashes\MD5; +use Utopia\Auth\Hashes\PHPass; +use Utopia\Auth\Hashes\Plaintext; +use Utopia\Auth\Hashes\Scrypt; +use Utopia\Auth\Hashes\ScryptModified; +use Utopia\Auth\Hashes\Sha; +use Utopia\Auth\Proofs\Password as ProofsPassword; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -54,15 +63,6 @@ use Utopia\Validator\Integer; use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -use Utopia\Auth\Hashes\Argon2; -use Utopia\Auth\Hashes\Bcrypt; -use Utopia\Auth\Hashes\MD5; -use Utopia\Auth\Hashes\PHPass; -use Utopia\Auth\Hashes\Scrypt; -use Utopia\Auth\Hashes\ScryptModified; -use Utopia\Auth\Hashes\Sha; -use Utopia\Auth\Hashes\Plaintext; -use Utopia\Auth\Proofs\Password as ProofsPassword; /** TODO: Remove function when we move to using utopia/platform */ function createUser(Hash $hash, string $userId, ?string $email, ?string $password, ?string $phone, string $name, Document $project, Database $dbForProject, Hooks $hooks): Document @@ -2515,4 +2515,3 @@ App::get('/v1/users/usage') 'sessions' => $usage[$metrics[1]]['data'], ]), Response::MODEL_USAGE_USERS); }); - diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index f87ddf9730..bc9ce80b0d 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -202,18 +202,18 @@ App::init() /** * Handle user authentication and session validation. - * + * * This function follows a series of steps to determine the appropriate user session * based on cookies, headers, and JWT tokens. - * + * * Process: - * + * * Project & Role Validation: * 1. Check if the project is empty. If so, throw an exception. * 2. Get the roles configuration. * 3. Determine the role for the user based on the user document. * 4. Get the scopes for the role. - * + * * API Key Authentication: * 5. If there is an API key: * - Verify no user session exists simultaneously @@ -237,7 +237,7 @@ App::init() * 12. Validate MFA requirements: * - Check if MFA is enabled * - Verify email status - * - Verify phone status + * - Verify phone status * - Verify authenticator status * 13. Handle Multi-Factor Authentication: * - Check remaining required factors @@ -255,7 +255,7 @@ App::init() // Step 3: Determine role for user // TODO get scopes from the identity instead of the user roles config. The identity will containn the scopes the user authorized for the access token. - + $role = $user->isEmpty() ? Role::guests()->toString() : Role::users()->toString(); From d1318cf0161a25275a9dd69de3a3d8b35d68e96c Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 10 Mar 2025 02:19:57 +0100 Subject: [PATCH 005/245] Updated dependencies --- composer.lock | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index ce424076c3..6adef9098a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "58ad2e1375ec47d944b96864d4aae63f", + "content-hash": "5820d9145556499015ac03aef8a7fb39", "packages": [ { "name": "adhocore/jwt", @@ -5990,6 +5990,65 @@ ], "time": "2025-01-26T19:54:45+00:00" }, + { + "name": "phpstan/phpstan", + "version": "1.8.11", + "source": { + "type": "git", + "url": "https://github.com/phpstan/phpstan.git", + "reference": "46e223dd68a620da18855c23046ddb00940b4014" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/46e223dd68a620da18855c23046ddb00940b4014", + "reference": "46e223dd68a620da18855c23046ddb00940b4014", + "shasum": "" + }, + "require": { + "php": "^7.2|^8.0" + }, + "conflict": { + "phpstan/phpstan-shim": "*" + }, + "bin": [ + "phpstan", + "phpstan.phar" + ], + "type": "library", + "autoload": { + "files": [ + "bootstrap.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "PHPStan - PHP Static Analysis Tool", + "keywords": [ + "dev", + "static analysis" + ], + "support": { + "issues": "https://github.com/phpstan/phpstan/issues", + "source": "https://github.com/phpstan/phpstan/tree/1.8.11" + }, + "funding": [ + { + "url": "https://github.com/ondrejmirtes", + "type": "github" + }, + { + "url": "https://github.com/phpstan", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpstan/phpstan", + "type": "tidelift" + } + ], + "time": "2022-10-24T15:45:13+00:00" + }, { "name": "phpunit/php-code-coverage", "version": "9.2.32", From c08355df8dd311cb2fd66388688e4d7bec8d1524 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 16 Mar 2025 10:34:45 +0100 Subject: [PATCH 006/245] Updated composer --- composer.lock | 110 +++++++++++++++++++++++++------------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/composer.lock b/composer.lock index b8b61754ec..94a4ec905b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6e6444246ba207b6bf88c20340835fb5", + "content-hash": "a7592b2898066e8fe4eecf4fe94db6ed", "packages": [ { "name": "adhocore/jwt", @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.30.0", + "version": "v4.30.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6" + "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/e1d66682f6836aa87820400f0aa07d9eb566feb6", - "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/f29ba8a30dfd940efb3a8a75dc44446539101f24", + "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.0" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.1" }, - "time": "2025-03-04T22:54:49+00:00" + "time": "2025-03-13T21:08:17+00:00" }, { "name": "jean85/pretty-package-versions", @@ -3364,16 +3364,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.51.0", + "version": "0.52.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "661687b03277f1d202a0e8cf9da6e58c97da2b5e" + "reference": "a0d6421e7e5baa3ac02755496dca9fdeaa814b93" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/661687b03277f1d202a0e8cf9da6e58c97da2b5e", - "reference": "661687b03277f1d202a0e8cf9da6e58c97da2b5e", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/a0d6421e7e5baa3ac02755496dca9fdeaa814b93", + "reference": "a0d6421e7e5baa3ac02755496dca9fdeaa814b93", "shasum": "" }, "require": { @@ -3381,7 +3381,7 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/database": "0.60.*" + "utopia-php/database": "0.*.*" }, "require-dev": { "laravel/pint": "1.*", @@ -3409,9 +3409,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.51.0" + "source": "https://github.com/utopia-php/abuse/tree/0.52.0" }, - "time": "2025-02-17T11:10:18+00:00" + "time": "2025-03-06T03:48:29+00:00" }, { "name": "utopia-php/analytics", @@ -3461,21 +3461,21 @@ }, { "name": "utopia-php/audit", - "version": "0.54.0", + "version": "0.55.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975" + "reference": "9f8cfe5fa5d5011b8dbf93b710236dfa91dc5518" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975", - "reference": "1b0cb8ac6bfbd7703e3f9a753c6ba59ff1c39975", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/9f8cfe5fa5d5011b8dbf93b710236dfa91dc5518", + "reference": "9f8cfe5fa5d5011b8dbf93b710236dfa91dc5518", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/database": "0.60.*" + "utopia-php/database": "0.*.*" }, "require-dev": { "laravel/pint": "1.*", @@ -3502,9 +3502,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.54.0" + "source": "https://github.com/utopia-php/audit/tree/0.55.0" }, - "time": "2025-02-25T07:21:07+00:00" + "time": "2025-03-06T03:47:47+00:00" }, { "name": "utopia-php/auth", @@ -3760,16 +3760,16 @@ }, { "name": "utopia-php/database", - "version": "0.60.6", + "version": "0.61.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "f3c9aa964b39c6205069f038a26e709a15541406" + "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/f3c9aa964b39c6205069f038a26e709a15541406", - "reference": "f3c9aa964b39c6205069f038a26e709a15541406", + "url": "https://api.github.com/repos/utopia-php/database/zipball/349fbdf4bc088f7775c7dfb8b80239a617a88436", + "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436", "shasum": "" }, "require": { @@ -3810,9 +3810,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.60.6" + "source": "https://github.com/utopia-php/database/tree/0.61.2" }, - "time": "2025-03-05T01:23:14+00:00" + "time": "2025-03-15T11:47:42+00:00" }, { "name": "utopia-php/detector", @@ -4259,16 +4259,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.20", + "version": "0.6.22", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba" + "reference": "a0269746bd318ff0993f5aa008675b971689d5b5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba", - "reference": "8c9ba52196f50aaef4aa1903f0d8fe0c8d9997ba", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/a0269746bd318ff0993f5aa008675b971689d5b5", + "reference": "a0269746bd318ff0993f5aa008675b971689d5b5", "shasum": "" }, "require": { @@ -4276,7 +4276,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.60.*", + "utopia-php/database": "0.61.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4309,9 +4309,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.20" + "source": "https://github.com/utopia-php/migration/tree/0.6.22" }, - "time": "2025-02-17T11:02:15+00:00" + "time": "2025-03-13T07:35:55+00:00" }, { "name": "utopia-php/mongo", @@ -4425,16 +4425,16 @@ }, { "name": "utopia-php/platform", - "version": "0.7.3", + "version": "0.7.4", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "463c2d817c893d7dbb678c2eac7a8291f2710e25" + "reference": "a5b93d8177702ec458c3af9137663133c012b71b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/463c2d817c893d7dbb678c2eac7a8291f2710e25", - "reference": "463c2d817c893d7dbb678c2eac7a8291f2710e25", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/a5b93d8177702ec458c3af9137663133c012b71b", + "reference": "a5b93d8177702ec458c3af9137663133c012b71b", "shasum": "" }, "require": { @@ -4443,7 +4443,7 @@ "php": ">=8.0", "utopia-php/cli": "0.15.*", "utopia-php/framework": "0.33.*", - "utopia-php/queue": "0.8.*" + "utopia-php/queue": "0.9.*" }, "require-dev": { "laravel/pint": "1.2.*", @@ -4469,9 +4469,9 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/0.7.3" + "source": "https://github.com/utopia-php/platform/tree/0.7.4" }, - "time": "2025-02-04T15:09:00+00:00" + "time": "2025-03-13T13:00:12+00:00" }, { "name": "utopia-php/pools", @@ -4579,16 +4579,16 @@ }, { "name": "utopia-php/queue", - "version": "0.8.6", + "version": "0.9.0", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "b713b997285c29d120bbcbe3d6e93762d850f87c" + "reference": "077075f1d57afa430f76c35ed3bf4616e0eee8e7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/b713b997285c29d120bbcbe3d6e93762d850f87c", - "reference": "b713b997285c29d120bbcbe3d6e93762d850f87c", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/077075f1d57afa430f76c35ed3bf4616e0eee8e7", + "reference": "077075f1d57afa430f76c35ed3bf4616e0eee8e7", "shasum": "" }, "require": { @@ -4638,9 +4638,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.8.6" + "source": "https://github.com/utopia-php/queue/tree/0.9.0" }, - "time": "2025-02-10T03:35:00+00:00" + "time": "2025-03-13T12:22:41+00:00" }, { "name": "utopia-php/registry", @@ -5417,16 +5417,16 @@ }, { "name": "laravel/pint", - "version": "v1.21.1", + "version": "v1.21.2", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86" + "reference": "370772e7d9e9da087678a0edf2b11b6960e40558" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/c44bffbb2334e90fba560933c45948fa4a3f3e86", - "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86", + "url": "https://api.github.com/repos/laravel/pint/zipball/370772e7d9e9da087678a0edf2b11b6960e40558", + "reference": "370772e7d9e9da087678a0edf2b11b6960e40558", "shasum": "" }, "require": { @@ -5437,9 +5437,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.70.2", - "illuminate/view": "^11.44.1", - "larastan/larastan": "^3.1.0", + "friendsofphp/php-cs-fixer": "^3.72.0", + "illuminate/view": "^11.44.2", + "larastan/larastan": "^3.2.0", "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3", @@ -5479,7 +5479,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-03-11T03:22:21+00:00" + "time": "2025-03-14T22:31:42+00:00" }, { "name": "matthiasmullie/minify", From 706ce4b3b6ad5d63f3c80bc4e6c4582f7c6c560a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 16 Mar 2025 18:30:48 +0100 Subject: [PATCH 007/245] WIP - cleaning up auth managment --- app/controllers/api/account.php | 79 +++++++++++++++++++++++---------- app/controllers/api/teams.php | 19 +++++--- app/controllers/api/users.php | 11 ++++- app/init.php | 44 ++++++++++-------- composer.lock | 8 ++-- tests/unit/Auth/AuthTest.php | 15 ++++++- 6 files changed, 120 insertions(+), 56 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 20f64496ac..0778bb32c1 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -38,6 +38,7 @@ use MaxMind\Db\Reader; use Utopia\Abuse\Abuse; use Utopia\App; use Utopia\Audit\Audit as EventAudit; +use Utopia\Auth\Store; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -154,7 +155,7 @@ function sendSessionAlert(Locale $locale, Document $user, Document $project, Doc }; -$createSession = function (string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Mail $queueForMails) { +$createSession = function (string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Mail $queueForMails, Store $store) { $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); $isAppUser = Auth::isAppUser($roles); @@ -255,16 +256,21 @@ $createSession = function (string $userId, string $secret, Request $request, Res ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()); + $encoded = $store + ->setProperty('id', $user->getId()) + ->setProperty('secret', $sessionSecret) + ->encode(); + if (!Config::getParam('domainVerification')) { - $response->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $sessionSecret)])); + $response->addHeader('X-Fallback-Cookies', \json_encode([$store->getKey() => $encoded])); } $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); $protocol = $request->getProtocol(); $response - ->addCookie(Auth::$cookieName . '_legacy', Auth::encodeSession($user->getId(), $sessionSecret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, Auth::encodeSession($user->getId(), $sessionSecret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) + ->addCookie(Auth::$cookieName . '_legacy', $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) + ->addCookie(Auth::$cookieName, $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) ->setStatusCode(Response::STATUS_CODE_CREATED); $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); @@ -273,7 +279,7 @@ $createSession = function (string $userId, string $secret, Request $request, Res ->setAttribute('current', true) ->setAttribute('countryName', $countryName) ->setAttribute('expire', $expire) - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $sessionSecret) : '') + ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $encoded : '') ; $response->dynamic($session, Response::MODEL_SESSION); @@ -881,7 +887,8 @@ App::post('/v1/account/sessions/email') ->inject('queueForEvents') ->inject('queueForMails') ->inject('hooks') - ->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Mail $queueForMails, Hooks $hooks) { + ->inject('store') + ->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Mail $queueForMails, Hooks $hooks, Store $store) { $email = \strtolower($email); $protocol = $request->getProtocol(); @@ -947,17 +954,20 @@ App::post('/v1/account/sessions/email') Permission::delete(Role::user($user->getId())), ])); + $encoded = $store + ->setProperty('id', $user->getId()) + ->setProperty('secret', $secret) + ->encode(); + if (!Config::getParam('domainVerification')) { - $response - ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])) - ; + $response->addHeader('X-Fallback-Cookies', \json_encode([$store->getKey() => $encoded])); } $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); $response - ->addCookie(Auth::$cookieName . '_legacy', Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) + ->addCookie(Auth::$cookieName . '_legacy', $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) + ->addCookie(Auth::$cookieName, $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) ->setStatusCode(Response::STATUS_CODE_CREATED) ; @@ -966,7 +976,7 @@ App::post('/v1/account/sessions/email') $session ->setAttribute('current', true) ->setAttribute('countryName', $countryName) - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $secret) : '') + ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $encoded : '') ; $queueForEvents @@ -1017,7 +1027,8 @@ App::post('/v1/account/sessions/anonymous') ->inject('dbForProject') ->inject('geodb') ->inject('queueForEvents') - ->action(function (Request $request, Response $response, Locale $locale, Document $user, Document $project, Database $dbForProject, Reader $geodb, Event $queueForEvents) { + ->inject('store') + ->action(function (Request $request, Response $response, Locale $locale, Document $user, Document $project, Database $dbForProject, Reader $geodb, Event $queueForEvents, Store $store) { $protocol = $request->getProtocol(); $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); @@ -1106,15 +1117,20 @@ App::post('/v1/account/sessions/anonymous') ->setParam('sessionId', $session->getId()) ; + $encoded = $store + ->setProperty('id', $user->getId()) + ->setProperty('secret', $secret) + ->encode(); + if (!Config::getParam('domainVerification')) { - $response->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])); + $response->addHeader('X-Fallback-Cookies', \json_encode([$store->getKey() => $encoded])); } $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); $response - ->addCookie(Auth::$cookieName . '_legacy', Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) + ->addCookie(Auth::$cookieName . '_legacy', $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) + ->addCookie(Auth::$cookieName, $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) ->setStatusCode(Response::STATUS_CODE_CREATED) ; @@ -1123,7 +1139,7 @@ App::post('/v1/account/sessions/anonymous') $session ->setAttribute('current', true) ->setAttribute('countryName', $countryName) - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $secret) : '') + ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $encoded : '') ; $response->dynamic($session, Response::MODEL_SESSION); @@ -1163,6 +1179,7 @@ App::post('/v1/account/sessions/token') ->inject('geodb') ->inject('queueForEvents') ->inject('queueForMails') + ->inject('store') ->action($createSession); App::get('/v1/account/sessions/oauth2/:provider') @@ -1327,7 +1344,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') ->inject('dbForProject') ->inject('geodb') ->inject('queueForEvents') - ->action(function (string $provider, string $code, string $state, string $error, string $error_description, Request $request, Response $response, Document $project, Document $user, Database $dbForProject, Reader $geodb, Event $queueForEvents) use ($oauthDefaultSuccess) { + ->inject('store') + ->action(function (string $provider, string $code, string $state, string $error, string $error_description, Request $request, Response $response, Document $project, Document $user, Database $dbForProject, Reader $geodb, Event $queueForEvents, Store $store) use ($oauthDefaultSuccess) { $protocol = $request->getProtocol(); $callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId(); $defaultState = ['success' => $project->getAttribute('url', ''), 'failure' => '']; @@ -1711,8 +1729,13 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $session->setAttribute('expire', $expire); + $encoded = $store + ->setProperty('id', $user->getId()) + ->setProperty('secret', $secret) + ->encode(); + if (!Config::getParam('domainVerification')) { - $response->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])); + $response->addHeader('X-Fallback-Cookies', \json_encode([$store->getKey() => $encoded])); } $queueForEvents @@ -1726,12 +1749,12 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $query['project'] = $project->getId(); $query['domain'] = Config::getParam('cookieDomain'); $query['key'] = Auth::$cookieName; - $query['secret'] = Auth::encodeSession($user->getId(), $secret); + $query['secret'] = $encoded; } $response - ->addCookie(Auth::$cookieName . '_legacy', Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')); + ->addCookie(Auth::$cookieName . '_legacy', $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) + ->addCookie(Auth::$cookieName, $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')); } if (isset($sessionUpgrade) && $sessionUpgrade) { @@ -2358,6 +2381,7 @@ App::put('/v1/account/sessions/magic-url') ->inject('geodb') ->inject('queueForEvents') ->inject('queueForMails') + ->inject('store') ->action($createSession); App::put('/v1/account/sessions/phone') @@ -2395,6 +2419,7 @@ App::put('/v1/account/sessions/phone') ->inject('geodb') ->inject('queueForEvents') ->inject('queueForMails') + ->inject('store') ->action($createSession); App::post('/v1/account/tokens/phone') @@ -2434,7 +2459,8 @@ App::post('/v1/account/tokens/phone') ->inject('timelimit') ->inject('queueForStatsUsage') ->inject('plan') - ->action(function (string $userId, string $phone, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan) { + ->inject('store') + ->action(function (string $userId, string $phone, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, Store $store) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -2600,8 +2626,13 @@ App::post('/v1/account/tokens/phone') $queueForEvents ->setPayload($response->output($token, Response::MODEL_TOKEN), sensitive: ['secret']); + $encoded = $store + ->setProperty('id', $user->getId()) + ->setProperty('secret', $secret) + ->encode(); + // Hide secret for clients - $token->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $secret) : ''); + $token->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $encoded : ''); $response ->setStatusCode(Response::STATUS_CODE_CREATED) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index b45c9fd3b9..60d6960445 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -27,6 +27,7 @@ use MaxMind\Db\Reader; use Utopia\Abuse\Abuse; use Utopia\App; use Utopia\Audit\Audit; +use Utopia\Auth\Store; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -1126,7 +1127,8 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') ->inject('project') ->inject('geodb') ->inject('queueForEvents') - ->action(function (string $teamId, string $membershipId, string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Reader $geodb, Event $queueForEvents) { + ->inject('store') + ->action(function (string $teamId, string $membershipId, string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Reader $geodb, Event $queueForEvents, Store $store) { $protocol = $request->getProtocol(); $membership = $dbForProject->getDocument('memberships', $membershipId); @@ -1206,13 +1208,18 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') Authorization::setRole(Role::user($userId)->toString()); if (!Config::getParam('domainVerification')) { - $response->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])); + $encoded = $store + ->setProperty('id', $user->getId()) + ->setProperty('secret', $secret) + ->encode(); + + $response->addHeader('X-Fallback-Cookies', \json_encode([$store->getKey() => $encoded])); } $response ->addCookie( - name: Auth::$cookieName . '_legacy', - value: Auth::encodeSession($user->getId(), $secret), + name: $store->getKey() . '_legacy', + value: $encoded, expire: (new \DateTime($expire))->getTimestamp(), path: '/', domain: Config::getParam('cookieDomain'), @@ -1220,8 +1227,8 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') httponly: true ) ->addCookie( - name: Auth::$cookieName, - value: Auth::encodeSession($user->getId(), $secret), + name: $store->getKey(), + value: $encoded, expire: (new \DateTime($expire))->getTimestamp(), path: '/', domain: Config::getParam('cookieDomain'), diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index f5cefd365b..e1753fd9b4 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -38,6 +38,7 @@ use Utopia\Auth\Hashes\Scrypt; use Utopia\Auth\Hashes\ScryptModified; use Utopia\Auth\Hashes\Sha; use Utopia\Auth\Proofs\Password as ProofsPassword; +use Utopia\Auth\Store; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -2014,7 +2015,8 @@ App::post('/v1/users/:userId/sessions') ->inject('locale') ->inject('geodb') ->inject('queueForEvents') - ->action(function (string $userId, Request $request, Response $response, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents) { + ->inject('store') + ->action(function (string $userId, Request $request, Response $response, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Store $store) { $user = $dbForProject->getDocument('users', $userId); if ($user->isEmpty()) { throw new Exception(Exception::USER_NOT_FOUND); @@ -2057,8 +2059,13 @@ App::post('/v1/users/:userId/sessions') $dbForProject->purgeCachedDocument('users', $user->getId()); + $encoded = $store + ->setProperty('id', $user->getId()) + ->setProperty('secret', $secret) + ->encode(); + $session - ->setAttribute('secret', Auth::encodeSession($user->getId(), $secret)) + ->setAttribute('secret', $encoded) ->setAttribute('countryName', $countryName); $queueForEvents diff --git a/app/init.php b/app/init.php index 8219517834..123edd3915 100644 --- a/app/init.php +++ b/app/init.php @@ -51,6 +51,7 @@ use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; +use Utopia\Auth\Store; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; @@ -1286,13 +1287,14 @@ App::setResource('clients', function ($request, $console, $project) { return \array_unique($clients); }, ['request', 'console', 'project']); -App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForPlatform) { +App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForPlatform, Store $store) { /** @var Appwrite\Utopia\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForProject */ /** @var Utopia\Database\Database $dbForPlatform */ /** @var string $mode */ + /** @var Utopia\Auth\Store $store */ /** * Handles user authentication and session validation. @@ -1315,62 +1317,64 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons Authorization::setDefaultStatus(true); - Auth::setCookieName('a_session_' . $project->getId()); + $store->setKey('a_session_' . $project->getId()); if (APP_MODE_ADMIN === $mode) { - Auth::setCookieName('a_session_' . $console->getId()); + $store->setKey('a_session_' . $console->getId()); } - $session = Auth::decodeSession( + $store->decode( $request->getCookie( - Auth::$cookieName, // Get sessions - $request->getCookie(Auth::$cookieName . '_legacy', '') + $store->getKey(), // Get sessions + $request->getCookie($store->getKey() . '_legacy', '') ) ); + var_dump($store); + // Get session from header for SSR clients - if (empty($session['id']) && empty($session['secret'])) { + if (empty($store->getProperty('id', '')) && empty($store->getProperty('secret', ''))) { $sessionHeader = $request->getHeader('x-appwrite-session', ''); if (!empty($sessionHeader)) { - $session = Auth::decodeSession($sessionHeader); + $store->decode($sessionHeader); } } // Get fallback session from old clients (no SameSite support) or clients who block 3rd-party cookies - if ($response) { + if ($response) { // if in http context - add debug header $response->addHeader('X-Debug-Fallback', 'false'); } - if (empty($session['id']) && empty($session['secret'])) { + if (empty($store->getProperty('id', '')) && empty($store->getProperty('secret', ''))) { if ($response) { $response->addHeader('X-Debug-Fallback', 'true'); } $fallback = $request->getHeader('x-fallback-cookies', ''); $fallback = \json_decode($fallback, true); - $session = Auth::decodeSession(((isset($fallback[Auth::$cookieName])) ? $fallback[Auth::$cookieName] : '')); + $store->decode(((isset($fallback[$store->getKey()])) ? $fallback[$store->getKey()] : '')); } - Auth::$unique = $session['id'] ?? ''; - Auth::$secret = $session['secret'] ?? ''; + // Auth::$unique = $session['id'] ?? ''; + // Auth::$secret = $session['secret'] ?? ''; if (APP_MODE_ADMIN !== $mode) { if ($project->isEmpty()) { $user = new Document([]); } else { if ($project->getId() === 'console') { - $user = $dbForPlatform->getDocument('users', Auth::$unique); + $user = $dbForPlatform->getDocument('users', $store->getProperty('id', '')); } else { - $user = $dbForProject->getDocument('users', Auth::$unique); + $user = $dbForProject->getDocument('users', $store->getProperty('id', '')); } } } else { - $user = $dbForPlatform->getDocument('users', Auth::$unique); + $user = $dbForPlatform->getDocument('users', $store->getProperty('id', '')); } if ( $user->isEmpty() // Check a document has been found in the DB - || !Auth::sessionVerify($user->getAttribute('sessions', []), Auth::$secret) + || !Auth::sessionVerify($user->getAttribute('sessions', []), $store->getProperty('secret', '')) ) { // Validate user has valid login token $user = new Document([]); } @@ -1411,7 +1415,7 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons $dbForPlatform->setMetadata('user', $user->getId()); return $user; -}, ['mode', 'project', 'console', 'request', 'response', 'dbForProject', 'dbForPlatform']); +}, ['mode', 'project', 'console', 'request', 'response', 'dbForProject', 'dbForPlatform', 'store']); App::setResource('project', function ($dbForPlatform, $request, $console) { /** @var Appwrite\Utopia\Request $request */ @@ -1976,3 +1980,7 @@ App::setResource('apiKey', function (Request $request, Document $project): ?Key return Key::decode($project, $key); }, ['request', 'project']); + +App::setResource('store', function () { + return new Store(); +}); diff --git a/composer.lock b/composer.lock index 94a4ec905b..1997076f5e 100644 --- a/composer.lock +++ b/composer.lock @@ -3512,12 +3512,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/auth.git", - "reference": "b063a2317c48cc6f3dba1eab0298641b19accdcd" + "reference": "ada77726740eb7180a4cee762cdf0c1beb0dc3a3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/auth/zipball/b063a2317c48cc6f3dba1eab0298641b19accdcd", - "reference": "b063a2317c48cc6f3dba1eab0298641b19accdcd", + "url": "https://api.github.com/repos/utopia-php/auth/zipball/ada77726740eb7180a4cee762cdf0c1beb0dc3a3", + "reference": "ada77726740eb7180a4cee762cdf0c1beb0dc3a3", "shasum": "" }, "require": { @@ -3559,7 +3559,7 @@ "issues": "https://github.com/utopia-php/auth/issues", "source": "https://github.com/utopia-php/auth/tree/dev" }, - "time": "2025-03-09T22:50:59+00:00" + "time": "2025-03-16T16:32:38+00:00" }, { "name": "utopia-php/cache", diff --git a/tests/unit/Auth/AuthTest.php b/tests/unit/Auth/AuthTest.php index 705da42879..147ad17977 100644 --- a/tests/unit/Auth/AuthTest.php +++ b/tests/unit/Auth/AuthTest.php @@ -4,6 +4,7 @@ namespace Tests\Unit\Auth; use Appwrite\Auth\Auth; use PHPUnit\Framework\TestCase; +use Utopia\Auth\Store; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; @@ -36,8 +37,18 @@ class AuthTest extends TestCase $secret = 'secret'; $session = 'eyJpZCI6ImlkIiwic2VjcmV0Ijoic2VjcmV0In0='; - $this->assertEquals(Auth::encodeSession($id, $secret), $session); - $this->assertEquals(Auth::decodeSession($session), ['id' => $id, 'secret' => $secret]); + $store = new Store(); + + $encoded = $store + ->setProperty('id', $id) + ->setProperty('secret', $secret) + ->encode(); + + $decoded = $store->decode($encoded); + + $this->assertEquals($encoded, $session); + $this->assertEquals($decoded->getProperty('id'), $id); + $this->assertEquals($decoded->getProperty('secret'), $secret); } public function testHash(): void From cc724218b00459aeccf26e44449dabf0c36348a3 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 16 Mar 2025 18:32:46 +0100 Subject: [PATCH 008/245] fixed formatting --- tests/unit/Auth/AuthTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/Auth/AuthTest.php b/tests/unit/Auth/AuthTest.php index 147ad17977..68865306ae 100644 --- a/tests/unit/Auth/AuthTest.php +++ b/tests/unit/Auth/AuthTest.php @@ -45,7 +45,7 @@ class AuthTest extends TestCase ->encode(); $decoded = $store->decode($encoded); - + $this->assertEquals($encoded, $session); $this->assertEquals($decoded->getProperty('id'), $id); $this->assertEquals($decoded->getProperty('secret'), $secret); From a5e57a9a6715d272f572752c36c2a52c9272bdd1 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 16 Mar 2025 18:54:55 +0100 Subject: [PATCH 009/245] Work in progress, moving to instance based auth --- app/controllers/api/account.php | 53 ++++++++-------- app/init.php | 11 +--- app/realtime.php | 10 +-- src/Appwrite/Auth/Auth.php | 63 ------------------- .../Functions/Http/Executions/Create.php | 7 ++- tests/unit/Auth/AuthTest.php | 29 --------- 6 files changed, 41 insertions(+), 132 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 0778bb32c1..3ac17e953e 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -533,15 +533,15 @@ App::get('/v1/account/sessions') ->inject('response') ->inject('user') ->inject('locale') - ->inject('project') - ->action(function (Response $response, Document $user, Locale $locale, Document $project) { + ->inject('store') + ->action(function (Response $response, Document $user, Locale $locale, Store $store) { $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); $isAppUser = Auth::isAppUser($roles); $sessions = $user->getAttribute('sessions', []); - $current = Auth::sessionVerify($sessions, Auth::$secret); + $current = Auth::sessionVerify($sessions, $store->getProperty('secret', '')); foreach ($sessions as $key => $session) {/** @var Document $session */ $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); @@ -587,7 +587,8 @@ App::delete('/v1/account/sessions') ->inject('locale') ->inject('queueForEvents') ->inject('queueForDeletes') - ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $queueForEvents, Delete $queueForDeletes) { + ->inject('store') + ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $queueForEvents, Delete $queueForDeletes, Store $store) { $protocol = $request->getProtocol(); $sessions = $user->getAttribute('sessions', []); @@ -603,13 +604,13 @@ App::delete('/v1/account/sessions') ->setAttribute('current', false) ->setAttribute('countryName', $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown'))); - if ($session->getAttribute('secret') == Auth::hash(Auth::$secret)) { + if ($session->getAttribute('secret') == Auth::hash($store->getProperty('secret', ''))) { $session->setAttribute('current', true); // If current session delete the cookies too $response - ->addCookie(Auth::$cookieName . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')); + ->addCookie($store->getKey() . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) + ->addCookie($store->getKey(), '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')); // Use current session for events. $queueForEvents @@ -652,8 +653,8 @@ App::get('/v1/account/sessions/:sessionId') ->inject('response') ->inject('user') ->inject('locale') - ->inject('project') - ->action(function (?string $sessionId, Response $response, Document $user, Locale $locale, Document $project) { + ->inject('store') + ->action(function (?string $sessionId, Response $response, Document $user, Locale $locale, Store $store) { $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); @@ -661,7 +662,7 @@ App::get('/v1/account/sessions/:sessionId') $sessions = $user->getAttribute('sessions', []); $sessionId = ($sessionId === 'current') - ? Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret) + ? Auth::sessionVerify($user->getAttribute('sessions'), $store->getProperty('secret', '')) : $sessionId; foreach ($sessions as $session) {/** @var Document $session */ @@ -669,7 +670,7 @@ App::get('/v1/account/sessions/:sessionId') $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session - ->setAttribute('current', ($session->getAttribute('secret') == Auth::hash(Auth::$secret))) + ->setAttribute('current', ($session->getAttribute('secret') == Auth::hash($store->getProperty('secret', '')))) ->setAttribute('countryName', $countryName) ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $session->getAttribute('secret', '') : '') ; @@ -711,12 +712,12 @@ App::delete('/v1/account/sessions/:sessionId') ->inject('locale') ->inject('queueForEvents') ->inject('queueForDeletes') - ->inject('project') - ->action(function (?string $sessionId, ?\DateTime $requestTimestamp, Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $queueForEvents, Delete $queueForDeletes, Document $project) { + ->inject('store') + ->action(function (?string $sessionId, ?\DateTime $requestTimestamp, Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $queueForEvents, Delete $queueForDeletes, Store $store) { $protocol = $request->getProtocol(); $sessionId = ($sessionId === 'current') - ? Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret) + ? Auth::sessionVerify($user->getAttribute('sessions'), $store->getProperty('secret', '')) : $sessionId; $sessions = $user->getAttribute('sessions', []); @@ -735,7 +736,7 @@ App::delete('/v1/account/sessions/:sessionId') $session->setAttribute('current', false); - if ($session->getAttribute('secret') == Auth::hash(Auth::$secret)) { // If current session delete the cookies too + if ($session->getAttribute('secret') == Auth::hash($store->getProperty('secret', ''))) { // If current session delete the cookies too $session ->setAttribute('current', true) ->setAttribute('countryName', $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown'))); @@ -745,8 +746,8 @@ App::delete('/v1/account/sessions/:sessionId') } $response - ->addCookie(Auth::$cookieName . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')); + ->addCookie($store->getKey() . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) + ->addCookie($store->getKey(), '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')); } $dbForProject->purgeCachedDocument('users', $user->getId()); @@ -796,10 +797,11 @@ App::patch('/v1/account/sessions/:sessionId') ->inject('dbForProject') ->inject('project') ->inject('queueForEvents') - ->action(function (?string $sessionId, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents) { + ->inject('store') + ->action(function (?string $sessionId, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents, Store $store) { $sessionId = ($sessionId === 'current') - ? Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret) + ? Auth::sessionVerify($user->getAttribute('sessions'), $store->getProperty('secret', '')) : $sessionId; $sessions = $user->getAttribute('sessions', []); @@ -1482,7 +1484,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') } $sessions = $user->getAttribute('sessions', []); - $current = Auth::sessionVerify($sessions, Auth::$secret); + $current = Auth::sessionVerify($sessions, $store->getProperty('secret', '')); if ($current) { // Delete current session of new one. $currentDocument = $dbForProject->getDocument('sessions', $current); @@ -2662,15 +2664,15 @@ App::post('/v1/account/jwts') ->label('abuse-key', 'url:{url},userId:{userId}') ->inject('response') ->inject('user') - ->inject('dbForProject') - ->action(function (Response $response, Document $user, Database $dbForProject) { + ->inject('store') + ->action(function (Response $response, Document $user, Store $store) { $sessions = $user->getAttribute('sessions', []); $current = new Document(); foreach ($sessions as $session) { /** @var Utopia\Database\Document $session */ - if ($session->getAttribute('secret') == Auth::hash(Auth::$secret)) { // If current session delete the cookies too + if ($session->getAttribute('secret') == Auth::hash($store->getProperty('secret', ''))) { // If current session delete the cookies too $current = $session; } } @@ -4650,7 +4652,8 @@ App::post('/v1/account/targets/push') ->inject('request') ->inject('response') ->inject('dbForProject') - ->action(function (string $targetId, string $identifier, string $providerId, Event $queueForEvents, Document $user, Request $request, Response $response, Database $dbForProject) { + ->inject('store') + ->action(function (string $targetId, string $identifier, string $providerId, Event $queueForEvents, Document $user, Request $request, Response $response, Database $dbForProject, Store $store) { $targetId = $targetId == 'unique()' ? ID::unique() : $targetId; $provider = Authorization::skip(fn () => $dbForProject->getDocument('providers', $providerId)); @@ -4666,7 +4669,7 @@ App::post('/v1/account/targets/push') $device = $detector->getDevice(); - $sessionId = Auth::sessionVerify($user->getAttribute('sessions', []), Auth::$secret); + $sessionId = Auth::sessionVerify($user->getAttribute('sessions', []), $store->getProperty('secret', '')); $session = $dbForProject->getDocument('sessions', $sessionId); try { diff --git a/app/init.php b/app/init.php index 123edd3915..e54b7b3cc8 100644 --- a/app/init.php +++ b/app/init.php @@ -1330,8 +1330,6 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons ) ); - var_dump($store); - // Get session from header for SSR clients if (empty($store->getProperty('id', '')) && empty($store->getProperty('secret', ''))) { $sessionHeader = $request->getHeader('x-appwrite-session', ''); @@ -1355,9 +1353,6 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons $store->decode(((isset($fallback[$store->getKey()])) ? $fallback[$store->getKey()] : '')); } - // Auth::$unique = $session['id'] ?? ''; - // Auth::$secret = $session['secret'] ?? ''; - if (APP_MODE_ADMIN !== $mode) { if ($project->isEmpty()) { $user = new Document([]); @@ -1433,13 +1428,13 @@ App::setResource('project', function ($dbForPlatform, $request, $console) { return $project; }, ['dbForPlatform', 'request', 'console']); -App::setResource('session', function (Document $user) { +App::setResource('session', function (Document $user, Store $store) { if ($user->isEmpty()) { return; } $sessions = $user->getAttribute('sessions', []); - $sessionId = Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret); + $sessionId = Auth::sessionVerify($user->getAttribute('sessions'), $store->getProperty('secret', '')); if (!$sessionId) { return; @@ -1452,7 +1447,7 @@ App::setResource('session', function (Document $user) { } return; -}, ['user']); +}, ['user', 'store']); App::setResource('console', function () { return new Document(Config::getParam('console')); diff --git a/app/realtime.php b/app/realtime.php index 86f9c85fdd..d65a2559b5 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -15,6 +15,7 @@ use Swoole\Timer; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; +use Utopia\Auth\Store; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -649,15 +650,14 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Payload is not valid.'); } - $session = Auth::decodeSession($message['data']['session']); - Auth::$unique = $session['id'] ?? ''; - Auth::$secret = $session['secret'] ?? ''; + $store = new Store(); + $store->decode($message['data']['session']); - $user = $database->getDocument('users', Auth::$unique); + $user = $database->getDocument('users', $store->getProperty('id', '')); if ( empty($user->getId()) // Check a document has been found in the DB - || !Auth::sessionVerify($user->getAttribute('sessions', []), Auth::$secret) // Validate user has valid login token + || !Auth::sessionVerify($user->getAttribute('sessions', []), $store->getProperty('secret', '')) // Validate user has valid login token ) { // cookie not valid throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Session is not valid.'); diff --git a/src/Appwrite/Auth/Auth.php b/src/Appwrite/Auth/Auth.php index 9af5045fa4..ee1233fc10 100644 --- a/src/Appwrite/Auth/Auth.php +++ b/src/Appwrite/Auth/Auth.php @@ -108,48 +108,6 @@ class Auth */ public static $cookieNamePreview = 'a_jwt_console'; - /** - * User Unique ID. - * - * @var string - */ - public static $unique = ''; - - /** - * User Secret Key. - * - * @var string - */ - public static $secret = ''; - - /** - * Set Cookie Name. - * - * @param $string - * - * @return string - */ - public static function setCookieName($string) - { - return self::$cookieName = $string; - } - - /** - * Encode Session. - * - * @param string $id - * @param string $secret - * - * @return string - */ - public static function encodeSession($id, $secret) - { - return \base64_encode(\json_encode([ - 'id' => $id, - 'secret' => $secret, - ])); - } - /** * Token type to session provider mapping. */ @@ -171,27 +129,6 @@ class Auth } } - /** - * Decode Session. - * - * @param string $session - * - * @return array - * - * @throws \Exception - */ - public static function decodeSession($session) - { - $session = \json_decode(\base64_decode($session), true); - $default = ['id' => null, 'secret' => '']; - - if (!\is_array($session)) { - return $default; - } - - return \array_merge($default, $session); - } - /** * Encode. * diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index 2e47c04276..3a7030742e 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -19,6 +19,7 @@ use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; use Executor\Executor; use MaxMind\Db\Reader; +use Utopia\Auth\Store; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; @@ -93,6 +94,7 @@ class Create extends Base ->inject('queueForStatsUsage') ->inject('queueForFunctions') ->inject('geodb') + ->inject('store') ->callback([$this, 'action']); } @@ -113,7 +115,8 @@ class Create extends Base Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, - Reader $geodb + Reader $geodb, + Store $store ) { $async = \strval($async) === 'true' || \strval($async) === '1'; @@ -190,7 +193,7 @@ class Create extends Base foreach ($sessions as $session) { /** @var Utopia\Database\Document $session */ - if ($session->getAttribute('secret') == Auth::hash(Auth::$secret)) { // If current session delete the cookies too + if ($session->getAttribute('secret') == Auth::hash($store->getProperty('secret', ''))) { // If current session delete the cookies too $current = $session; } } diff --git a/tests/unit/Auth/AuthTest.php b/tests/unit/Auth/AuthTest.php index 68865306ae..2b19439da6 100644 --- a/tests/unit/Auth/AuthTest.php +++ b/tests/unit/Auth/AuthTest.php @@ -4,7 +4,6 @@ namespace Tests\Unit\Auth; use Appwrite\Auth\Auth; use PHPUnit\Framework\TestCase; -use Utopia\Auth\Store; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; @@ -23,34 +22,6 @@ class AuthTest extends TestCase Authorization::setRole(Role::any()->toString()); } - public function testCookieName(): void - { - $name = 'cookie-name'; - - $this->assertEquals(Auth::setCookieName($name), $name); - $this->assertEquals(Auth::$cookieName, $name); - } - - public function testEncodeDecodeSession(): void - { - $id = 'id'; - $secret = 'secret'; - $session = 'eyJpZCI6ImlkIiwic2VjcmV0Ijoic2VjcmV0In0='; - - $store = new Store(); - - $encoded = $store - ->setProperty('id', $id) - ->setProperty('secret', $secret) - ->encode(); - - $decoded = $store->decode($encoded); - - $this->assertEquals($encoded, $session); - $this->assertEquals($decoded->getProperty('id'), $id); - $this->assertEquals($decoded->getProperty('secret'), $secret); - } - public function testHash(): void { $secret = 'secret'; From 9346efc24a1fb7b594a49316781b3c88347e1b74 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 16 Mar 2025 19:35:42 +0100 Subject: [PATCH 010/245] Fixed some tests --- app/controllers/api/account.php | 22 +++++++++++----------- app/controllers/api/users.php | 2 +- composer.lock | 8 ++++---- src/Appwrite/Auth/Auth.php | 5 ----- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 3ac17e953e..4264300cce 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -269,8 +269,8 @@ $createSession = function (string $userId, string $secret, Request $request, Res $protocol = $request->getProtocol(); $response - ->addCookie(Auth::$cookieName . '_legacy', $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) + ->addCookie($store->getKey() . '_legacy', $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) + ->addCookie($store->getKey(), $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) ->setStatusCode(Response::STATUS_CODE_CREATED); $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); @@ -968,8 +968,8 @@ App::post('/v1/account/sessions/email') $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); $response - ->addCookie(Auth::$cookieName . '_legacy', $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) + ->addCookie($store->getKey() . '_legacy', $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) + ->addCookie($store->getKey(), $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) ->setStatusCode(Response::STATUS_CODE_CREATED) ; @@ -1131,8 +1131,8 @@ App::post('/v1/account/sessions/anonymous') $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); $response - ->addCookie(Auth::$cookieName . '_legacy', $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) + ->addCookie($store->getKey() . '_legacy', $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) + ->addCookie($store->getKey(), $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) ->setStatusCode(Response::STATUS_CODE_CREATED) ; @@ -1750,13 +1750,13 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') if ($state['success']['path'] == $oauthDefaultSuccess) { $query['project'] = $project->getId(); $query['domain'] = Config::getParam('cookieDomain'); - $query['key'] = Auth::$cookieName; + $query['key'] = $store->getKey(); $query['secret'] = $encoded; } $response - ->addCookie(Auth::$cookieName . '_legacy', $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')); + ->addCookie($store->getKey() . '_legacy', $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) + ->addCookie($store->getKey(), $encoded, (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')); } if (isset($sessionUpgrade) && $sessionUpgrade) { @@ -3161,8 +3161,8 @@ App::patch('/v1/account/status') $protocol = $request->getProtocol(); $response - ->addCookie(Auth::$cookieName . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) + ->addCookie($store->getKey() . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) + ->addCookie($store->getKey(), '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) ; $response->dynamic($user, Response::MODEL_ACCOUNT); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index e1753fd9b4..2fbbc423ba 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1313,7 +1313,7 @@ App::patch('/v1/users/:userId/password') // Create Argon2 hasher with default settings $hasher = new Argon2(); - $hasher->setMemoryCost(65536); + $hasher->setMemoryCost(2048); $hasher->setTimeCost(4); $hasher->setThreads(3); diff --git a/composer.lock b/composer.lock index 1997076f5e..050a92a1e3 100644 --- a/composer.lock +++ b/composer.lock @@ -3512,12 +3512,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/auth.git", - "reference": "ada77726740eb7180a4cee762cdf0c1beb0dc3a3" + "reference": "966fbfefb27be94e3363f07279787d5cf8a66b95" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/auth/zipball/ada77726740eb7180a4cee762cdf0c1beb0dc3a3", - "reference": "ada77726740eb7180a4cee762cdf0c1beb0dc3a3", + "url": "https://api.github.com/repos/utopia-php/auth/zipball/966fbfefb27be94e3363f07279787d5cf8a66b95", + "reference": "966fbfefb27be94e3363f07279787d5cf8a66b95", "shasum": "" }, "require": { @@ -3559,7 +3559,7 @@ "issues": "https://github.com/utopia-php/auth/issues", "source": "https://github.com/utopia-php/auth/tree/dev" }, - "time": "2025-03-16T16:32:38+00:00" + "time": "2025-03-16T18:32:00+00:00" }, { "name": "utopia-php/cache", diff --git a/src/Appwrite/Auth/Auth.php b/src/Appwrite/Auth/Auth.php index ee1233fc10..b0cba3235e 100644 --- a/src/Appwrite/Auth/Auth.php +++ b/src/Appwrite/Auth/Auth.php @@ -98,11 +98,6 @@ class Auth */ public const MFA_RECENT_DURATION = 1800; // 30 mins - /** - * @var string - */ - public static $cookieName = 'a_session'; - /** * @var string */ From f618a8b5630eca0df69939b1ee0ab2f34b450375 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sun, 16 Mar 2025 22:16:02 +0100 Subject: [PATCH 011/245] Fixed account test --- app/controllers/api/account.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 4264300cce..9445d937b7 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -3145,7 +3145,8 @@ App::patch('/v1/account/status') ->inject('user') ->inject('dbForProject') ->inject('queueForEvents') - ->action(function (?\DateTime $requestTimestamp, Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { + ->inject('store') + ->action(function (?\DateTime $requestTimestamp, Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Store $store) { $user->setAttribute('status', false); From 1ce84f1650f1df409bad72d00ca7995c226cbcb6 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 17 Mar 2025 12:39:35 +0100 Subject: [PATCH 012/245] WIP --- app/controllers/api/account.php | 14 +++++++++----- app/controllers/api/teams.php | 11 +++++++---- app/init.php | 21 ++++++++++++++++++++- src/Appwrite/Auth/Auth.php | 14 -------------- src/Appwrite/Platform/Tasks/Install.php | 3 ++- tests/unit/Auth/AuthTest.php | 6 ------ 6 files changed, 38 insertions(+), 31 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 9445d937b7..3abedb1f3f 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -38,6 +38,7 @@ use MaxMind\Db\Reader; use Utopia\Abuse\Abuse; use Utopia\App; use Utopia\Audit\Audit as EventAudit; +use Utopia\Auth\Proofs\Password as ProofsPassword; use Utopia\Auth\Store; use Utopia\Config\Config; use Utopia\Database\Database; @@ -364,7 +365,9 @@ App::post('/v1/account') $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]); $passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0; - $password = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS); + $proof = new ProofsPassword(); + $hash = $proof->hash($password); + try { $userId = $userId == 'unique()' ? ID::unique() : $userId; $user->setAttributes([ @@ -377,7 +380,7 @@ App::post('/v1/account') 'email' => $email, 'emailVerification' => false, 'status' => true, - 'password' => $password, + 'password' => $hash, 'passwordHistory' => $passwordHistory > 0 ? [$password] : [], 'passwordUpdate' => DateTime::now(), 'hash' => Auth::DEFAULT_ALGO, @@ -942,7 +945,7 @@ App::post('/v1/account/sessions/email') // Re-hash if not using recommended algo if ($user->getAttribute('hash') !== Auth::DEFAULT_ALGO) { $user - ->setAttribute('password', Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS)) + ->setAttribute('password', (new ProofsPassword())->hash($password)) ->setAttribute('hash', Auth::DEFAULT_ALGO) ->setAttribute('hashOptions', Auth::DEFAULT_ALGO_OPTIONS); $dbForProject->updateDocument('users', $user->getId(), $user); @@ -2930,13 +2933,14 @@ App::patch('/v1/account/email') ->inject('queueForEvents') ->inject('project') ->inject('hooks') - ->action(function (string $email, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Document $project, Hooks $hooks) { + ->inject('proofForPassword') + ->action(function (string $email, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Document $project, Hooks $hooks, ProofsPassword $proofForPassword) { // passwordUpdate will be empty if the user has never set a password $passwordUpdate = $user->getAttribute('passwordUpdate'); if ( !empty($passwordUpdate) && - !Auth::passwordVerify($password, $user->getAttribute('password'), $user->getAttribute('hash'), $user->getAttribute('hashOptions')) + !$proofForPassword->verify($password, $user->getAttribute('password'), $user->getAttribute('hash'), $user->getAttribute('hashOptions')) ) { // Double check user password throw new Exception(Exception::USER_INVALID_CREDENTIALS); } diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 60d6960445..b1b4445de0 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -27,6 +27,7 @@ use MaxMind\Db\Reader; use Utopia\Abuse\Abuse; use Utopia\App; use Utopia\Audit\Audit; +use Utopia\Auth\Proofs\Password; use Utopia\Auth\Store; use Utopia\Config\Config; use Utopia\Database\Database; @@ -469,7 +470,8 @@ App::post('/v1/teams/:teamId/memberships') ->inject('timelimit') ->inject('queueForStatsUsage') ->inject('plan') - ->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan) { + ->inject('proofForPassword') + ->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, Password $proofForPassword) { $isAppUser = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -542,6 +544,7 @@ App::post('/v1/teams/:teamId/memberships') try { $userId = ID::unique(); + $hash = $proofForPassword->hash($proofForPassword->generate()); $invitee = Authorization::skip(fn () => $dbForProject->createDocument('users', new Document([ '$id' => $userId, '$permissions' => [ @@ -555,9 +558,9 @@ App::post('/v1/teams/:teamId/memberships') 'emailVerification' => false, 'status' => true, // TODO: Set password empty? - 'password' => Auth::passwordHash(Auth::passwordGenerator(), Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS), - 'hash' => Auth::DEFAULT_ALGO, - 'hashOptions' => Auth::DEFAULT_ALGO_OPTIONS, + 'password' => $hash, + 'hash' => $proofForPassword->getHash()->getName(), + 'hashOptions' => $proofForPassword->getHash()->getOptions(), /** * Set the password update time to 0 for users created using * team invite and OAuth to allow password updates without an diff --git a/app/init.php b/app/init.php index e54b7b3cc8..ffab067206 100644 --- a/app/init.php +++ b/app/init.php @@ -51,6 +51,9 @@ use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; +use Utopia\Auth\Proofs\Code; +use Utopia\Auth\Proofs\Password; +use Utopia\Auth\Proofs\Token; use Utopia\Auth\Store; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Adapter\Sharding; @@ -1976,6 +1979,22 @@ App::setResource('apiKey', function (Request $request, Document $project): ?Key return Key::decode($project, $key); }, ['request', 'project']); -App::setResource('store', function () { +App::setResource('store', function (): Store { return new Store(); }); + +App::setResource('proofForPassword', function (): Password { + return new Password(); +}); + +App::setResource('proofForToken', function (): Token { + return new Token(); +}); + +App::setResource('proofForCode', function (): Code { + return new Code(); +}); + + + + diff --git a/src/Appwrite/Auth/Auth.php b/src/Appwrite/Auth/Auth.php index b0cba3235e..d47d9ec4b5 100644 --- a/src/Appwrite/Auth/Auth.php +++ b/src/Appwrite/Auth/Auth.php @@ -237,20 +237,6 @@ class Auth } } - /** - * Password Generator. - * - * Generate random password string - * - * @param int $length - * - * @return string - */ - public static function passwordGenerator(int $length = 20): string - { - return \bin2hex(\random_bytes($length)); - } - /** * Token Generator. * diff --git a/src/Appwrite/Platform/Tasks/Install.php b/src/Appwrite/Platform/Tasks/Install.php index c7b1f72453..9b6ec64154 100644 --- a/src/Appwrite/Platform/Tasks/Install.php +++ b/src/Appwrite/Platform/Tasks/Install.php @@ -6,6 +6,7 @@ use Appwrite\Auth\Auth; use Appwrite\Docker\Compose; use Appwrite\Docker\Env; use Appwrite\Utopia\View; +use Utopia\Auth\Proofs\Password; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Platform\Action; @@ -162,7 +163,7 @@ class Install extends Action } if ($var['filter'] === 'password') { - $input[$var['name']] = Auth::passwordGenerator(); + $input[$var['name']] = (new Password())->generate(); continue; } } diff --git a/tests/unit/Auth/AuthTest.php b/tests/unit/Auth/AuthTest.php index 2b19439da6..c2057394d3 100644 --- a/tests/unit/Auth/AuthTest.php +++ b/tests/unit/Auth/AuthTest.php @@ -163,12 +163,6 @@ class AuthTest extends TestCase $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'md8')); } - public function testPasswordGenerator(): void - { - $this->assertEquals(\mb_strlen(Auth::passwordGenerator()), 40); - $this->assertEquals(\mb_strlen(Auth::passwordGenerator(5)), 10); - } - public function testTokenGenerator(): void { $this->assertEquals(\strlen(Auth::tokenGenerator()), 256); From fdb83c51f72b93e8b7d65d5c7410af23f7fc5cb0 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 17 Mar 2025 12:46:07 +0100 Subject: [PATCH 013/245] formatting --- app/init/resources.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/init/resources.php b/app/init/resources.php index 51b7ff0616..f04deaba89 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -23,6 +23,10 @@ use Appwrite\Network\Validator\Origin; use Appwrite\Utopia\Request; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; +use Utopia\Auth\Proofs\Code; +use Utopia\Auth\Proofs\Password; +use Utopia\Auth\Proofs\Token; +use Utopia\Auth\Store; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -48,10 +52,6 @@ use Utopia\Storage\Storage; use Utopia\System\System; use Utopia\Validator\Hostname; use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub; -use Utopia\Auth\Store; -use Utopia\Auth\Proofs\Password; -use Utopia\Auth\Proofs\Token; -use Utopia\Auth\Proofs\Code; // Runtime Execution App::setResource('log', fn () => new Log()); From 8aa57141730d9a8191711aff7b3705e47780340f Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 17 Mar 2025 21:44:31 +0100 Subject: [PATCH 014/245] cleanups --- app/config/collections/common.php | 4 +- app/config/console.php | 2 +- app/config/roles.php | 12 +- app/controllers/api/account.php | 171 +++++++------- app/controllers/api/projects.php | 2 +- app/controllers/api/teams.php | 8 +- app/controllers/api/users.php | 10 +- app/controllers/shared/api.php | 10 +- app/controllers/shared/api/auth.php | 2 +- app/init/constants.php | 67 +++++- composer.lock | 34 +-- src/Appwrite/Auth/Auth.php | 217 ++---------------- src/Appwrite/Auth/Hash.php | 62 ----- src/Appwrite/Auth/Key.php | 8 +- .../Auth/Validator/PasswordHistory.php | 6 +- src/Appwrite/Migration/Version/V16.php | 2 +- src/Appwrite/Migration/Version/V17.php | 2 +- src/Appwrite/Migration/Version/V20.php | 10 +- src/Appwrite/Platform/Workers/Audits.php | 2 +- src/Appwrite/Platform/Workers/Deletes.php | 2 +- .../Utopia/Response/Model/Project.php | 4 +- .../Projects/ProjectsConsoleClientTest.php | 6 +- tests/unit/Auth/AuthTest.php | 101 ++++---- tests/unit/Auth/KeyTest.php | 4 +- .../unit/Messaging/MessagingChannelsTest.php | 4 +- 25 files changed, 293 insertions(+), 459 deletions(-) delete mode 100644 src/Appwrite/Auth/Hash.php diff --git a/app/config/collections/common.php b/app/config/collections/common.php index f68400e226..7e7da1c94d 100644 --- a/app/config/collections/common.php +++ b/app/config/collections/common.php @@ -173,7 +173,7 @@ return [ 'size' => 256, 'signed' => true, 'required' => false, - 'default' => Auth::DEFAULT_ALGO, + 'default' => '', 'array' => false, 'filters' => [], ], @@ -184,7 +184,7 @@ return [ 'size' => 65535, 'signed' => true, 'required' => false, - 'default' => Auth::DEFAULT_ALGO_OPTIONS, + 'default' => new \stdClass(), 'array' => false, 'filters' => ['json'], ], diff --git a/app/config/console.php b/app/config/console.php index e37c9b7836..a0988ffaf9 100644 --- a/app/config/console.php +++ b/app/config/console.php @@ -38,7 +38,7 @@ $console = [ 'mockNumbers' => [], 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user - 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds + 'duration' => TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' ], 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], diff --git a/app/config/roles.php b/app/config/roles.php index a4abee0c45..2c0499855f 100644 --- a/app/config/roles.php +++ b/app/config/roles.php @@ -84,7 +84,7 @@ $admins = [ ]; return [ - Auth::USER_ROLE_GUESTS => [ + USER_ROLE_GUESTS => [ 'label' => 'Guests', 'scopes' => [ 'global', @@ -102,23 +102,23 @@ return [ 'execution.write', ], ], - Auth::USER_ROLE_USERS => [ + USER_ROLE_USERS => [ 'label' => 'Users', 'scopes' => \array_merge($member), ], - Auth::USER_ROLE_ADMIN => [ + USER_ROLE_ADMIN => [ 'label' => 'Admin', 'scopes' => \array_merge($admins), ], - Auth::USER_ROLE_DEVELOPER => [ + USER_ROLE_DEVELOPER => [ 'label' => 'Developer', 'scopes' => \array_merge($admins), ], - Auth::USER_ROLE_OWNER => [ + USER_ROLE_OWNER => [ 'label' => 'Owner', 'scopes' => \array_merge($member, $admins), ], - Auth::USER_ROLE_APPS => [ + USER_ROLE_APPS => [ 'label' => 'Applications', 'scopes' => ['global', 'health.read', 'graphql'], ], diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 3abedb1f3f..812b454cac 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -176,17 +176,17 @@ $createSession = function (string $userId, string $secret, Request $request, Res $user->setAttributes($userFromRequest->getArrayCopy()); - $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $duration = $project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; $detector = new Detector($request->getUserAgent('UNKNOWN')); $record = $geodb->get($request->getIP()); - $sessionSecret = Auth::tokenGenerator(Auth::TOKEN_LENGTH_SESSION); + $sessionSecret = Auth::tokenGenerator(TOKEN_LENGTH_SESSION); $factor = (match ($verifiedToken->getAttribute('type')) { - Auth::TOKEN_TYPE_MAGIC_URL, - Auth::TOKEN_TYPE_OAUTH2, - Auth::TOKEN_TYPE_EMAIL => Type::EMAIL, - Auth::TOKEN_TYPE_PHONE => Type::PHONE, - Auth::TOKEN_TYPE_GENERIC => 'token', + TOKEN_TYPE_MAGIC_URL, + TOKEN_TYPE_OAUTH2, + TOKEN_TYPE_EMAIL => Type::EMAIL, + TOKEN_TYPE_PHONE => Type::PHONE, + TOKEN_TYPE_GENERIC => 'token', default => throw new Exception(Exception::USER_INVALID_TOKEN) }); @@ -221,11 +221,11 @@ $createSession = function (string $userId, string $secret, Request $request, Res $dbForProject->purgeCachedDocument('users', $user->getId()); // Magic URL + Email OTP - if ($verifiedToken->getAttribute('type') === Auth::TOKEN_TYPE_MAGIC_URL || $verifiedToken->getAttribute('type') === Auth::TOKEN_TYPE_EMAIL) { + if ($verifiedToken->getAttribute('type') === TOKEN_TYPE_MAGIC_URL || $verifiedToken->getAttribute('type') === TOKEN_TYPE_EMAIL) { $user->setAttribute('emailVerification', true); } - if ($verifiedToken->getAttribute('type') === Auth::TOKEN_TYPE_PHONE) { + if ($verifiedToken->getAttribute('type') === TOKEN_TYPE_PHONE) { $user->setAttribute('phoneVerification', true); } @@ -236,8 +236,8 @@ $createSession = function (string $userId, string $secret, Request $request, Res } $isAllowedTokenType = match ($verifiedToken->getAttribute('type')) { - Auth::TOKEN_TYPE_MAGIC_URL, - Auth::TOKEN_TYPE_EMAIL => false, + TOKEN_TYPE_MAGIC_URL, + TOKEN_TYPE_EMAIL => false, default => true }; @@ -383,8 +383,8 @@ App::post('/v1/account') 'password' => $hash, 'passwordHistory' => $passwordHistory > 0 ? [$password] : [], 'passwordUpdate' => DateTime::now(), - 'hash' => Auth::DEFAULT_ALGO, - 'hashOptions' => Auth::DEFAULT_ALGO_OPTIONS, + 'hash' => $proof->getHash()->getName(), + 'hashOptions' => $proof->getHash()->getOptions(), 'registration' => DateTime::now(), 'reset' => false, 'name' => $name, @@ -821,7 +821,7 @@ App::patch('/v1/account/sessions/:sessionId') } // Extend session - $authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $authDuration = $project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; $session->setAttribute('expire', DateTime::addSeconds(new \DateTime(), $authDuration)); // Refresh OAuth access token @@ -893,7 +893,8 @@ App::post('/v1/account/sessions/email') ->inject('queueForMails') ->inject('hooks') ->inject('store') - ->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Mail $queueForMails, Hooks $hooks, Store $store) { + ->inject('proofForPassword') + ->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Mail $queueForMails, Hooks $hooks, Store $store, ProofsPassword $proofForPassword) { $email = \strtolower($email); $protocol = $request->getProtocol(); @@ -901,7 +902,9 @@ App::post('/v1/account/sessions/email') Query::equal('email', [$email]), ]); - if ($profile->isEmpty() || empty($profile->getAttribute('passwordUpdate')) || !Auth::passwordVerify($password, $profile->getAttribute('password'), $profile->getAttribute('hash'), $profile->getAttribute('hashOptions'))) { + $userProofForPassword = ProofsPassword::createHash($profile->getAttribute('hash'), $profile->getAttribute('hashOptions')); + + if ($profile->isEmpty() || empty($profile->getAttribute('passwordUpdate')) || !$userProofForPassword->verify($password, $profile->getAttribute('password'))) { throw new Exception(Exception::USER_INVALID_CREDENTIALS); } @@ -917,16 +920,16 @@ App::post('/v1/account/sessions/email') $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, false]); - $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $duration = $project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; $detector = new Detector($request->getUserAgent('UNKNOWN')); $record = $geodb->get($request->getIP()); - $secret = Auth::tokenGenerator(Auth::TOKEN_LENGTH_SESSION); + $secret = Auth::tokenGenerator(TOKEN_LENGTH_SESSION); $session = new Document(array_merge( [ '$id' => ID::unique(), 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), - 'provider' => Auth::SESSION_PROVIDER_EMAIL, + 'provider' => SESSION_PROVIDER_EMAIL, 'providerUid' => $email, 'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak 'userAgent' => $request->getUserAgent('UNKNOWN'), @@ -943,11 +946,11 @@ App::post('/v1/account/sessions/email') Authorization::setRole(Role::user($user->getId())->toString()); // Re-hash if not using recommended algo - if ($user->getAttribute('hash') !== Auth::DEFAULT_ALGO) { + if ($user->getAttribute('hash') !== $proofForPassword->getHash()->getName()) { $user ->setAttribute('password', (new ProofsPassword())->hash($password)) - ->setAttribute('hash', Auth::DEFAULT_ALGO) - ->setAttribute('hashOptions', Auth::DEFAULT_ALGO_OPTIONS); + ->setAttribute('hash', $proofForPassword->getHash()->getName()) + ->setAttribute('hashOptions', $proofForPassword->getHash()->getOptions()); $dbForProject->updateDocument('users', $user->getId(), $user); } @@ -1033,7 +1036,8 @@ App::post('/v1/account/sessions/anonymous') ->inject('geodb') ->inject('queueForEvents') ->inject('store') - ->action(function (Request $request, Response $response, Locale $locale, Document $user, Document $project, Database $dbForProject, Reader $geodb, Event $queueForEvents, Store $store) { + ->inject('proofForPassword') + ->action(function (Request $request, Response $response, Locale $locale, Document $user, Document $project, Database $dbForProject, Reader $geodb, Event $queueForEvents, Store $store, ProofsPassword $proofForPassword) { $protocol = $request->getProtocol(); $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); @@ -1065,8 +1069,8 @@ App::post('/v1/account/sessions/anonymous') 'emailVerification' => false, 'status' => true, 'password' => null, - 'hash' => Auth::DEFAULT_ALGO, - 'hashOptions' => Auth::DEFAULT_ALGO_OPTIONS, + 'hash' => $proofForPassword->getHash()->getName(), + 'hashOptions' => $proofForPassword->getHash()->getOptions(), 'passwordUpdate' => null, 'registration' => DateTime::now(), 'reset' => false, @@ -1084,17 +1088,17 @@ App::post('/v1/account/sessions/anonymous') Authorization::skip(fn () => $dbForProject->createDocument('users', $user)); // Create session token - $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $duration = $project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; $detector = new Detector($request->getUserAgent('UNKNOWN')); $record = $geodb->get($request->getIP()); - $secret = Auth::tokenGenerator(Auth::TOKEN_LENGTH_SESSION); + $secret = Auth::tokenGenerator(TOKEN_LENGTH_SESSION); $session = new Document(array_merge( [ '$id' => ID::unique(), 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), - 'provider' => Auth::SESSION_PROVIDER_ANONYMOUS, + 'provider' => SESSION_PROVIDER_ANONYMOUS, 'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak 'userAgent' => $request->getUserAgent('UNKNOWN'), 'ip' => $request->getIP(), @@ -1350,7 +1354,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') ->inject('geodb') ->inject('queueForEvents') ->inject('store') - ->action(function (string $provider, string $code, string $state, string $error, string $error_description, Request $request, Response $response, Document $project, Document $user, Database $dbForProject, Reader $geodb, Event $queueForEvents, Store $store) use ($oauthDefaultSuccess) { + ->inject('proofForPassword') + ->action(function (string $provider, string $code, string $state, string $error, string $error_description, Request $request, Response $response, Document $project, Document $user, Database $dbForProject, Reader $geodb, Event $queueForEvents, Store $store, ProofsPassword $proofForPassword) use ($oauthDefaultSuccess) { $protocol = $request->getProtocol(); $callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId(); $defaultState = ['success' => $project->getAttribute('url', ''), 'failure' => '']; @@ -1568,8 +1573,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') 'emailVerification' => true, 'status' => true, // Email should already be authenticated by OAuth2 provider 'password' => null, - 'hash' => Auth::DEFAULT_ALGO, - 'hashOptions' => Auth::DEFAULT_ALGO_OPTIONS, + 'hash' => $proofForPassword->getHash()->getName(), + 'hashOptions' => $proofForPassword->getHash()->getOptions(), 'passwordUpdate' => null, 'registration' => DateTime::now(), 'reset' => false, @@ -1668,17 +1673,17 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $state['success'] = URLParser::parse($state['success']); $query = URLParser::parseQuery($state['success']['query']); - $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $duration = $project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); // If the `token` param is set, we will return the token in the query string if ($state['token']) { - $secret = Auth::tokenGenerator(Auth::TOKEN_LENGTH_OAUTH2); + $secret = Auth::tokenGenerator(TOKEN_LENGTH_OAUTH2); $token = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), - 'type' => Auth::TOKEN_TYPE_OAUTH2, + 'type' => TOKEN_TYPE_OAUTH2, 'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), @@ -1707,7 +1712,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') } else { $detector = new Detector($request->getUserAgent('UNKNOWN')); $record = $geodb->get($request->getIP()); - $secret = Auth::tokenGenerator(Auth::TOKEN_LENGTH_SESSION); + $secret = Auth::tokenGenerator(TOKEN_LENGTH_SESSION); $session = new Document(array_merge([ '$id' => ID::unique(), @@ -1902,7 +1907,8 @@ App::post('/v1/account/tokens/magic-url') ->inject('locale') ->inject('queueForEvents') ->inject('queueForMails') - ->action(function (string $userId, string $email, string $url, bool $phrase, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Locale $locale, Event $queueForEvents, Mail $queueForMails) { + ->inject('proofForPassword') + ->action(function (string $userId, string $email, string $url, bool $phrase, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Locale $locale, Event $queueForEvents, Mail $queueForMails, ProofsPassword $proofForPassword) { if (empty(System::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP disabled'); } @@ -1951,8 +1957,8 @@ App::post('/v1/account/tokens/magic-url') 'emailVerification' => false, 'status' => true, 'password' => null, - 'hash' => Auth::DEFAULT_ALGO, - 'hashOptions' => Auth::DEFAULT_ALGO_OPTIONS, + 'hash' => $proofForPassword->getHash()->getName(), + 'hashOptions' => $proofForPassword->getHash()->getOptions(), 'passwordUpdate' => null, 'registration' => DateTime::now(), 'reset' => false, @@ -1970,14 +1976,14 @@ App::post('/v1/account/tokens/magic-url') Authorization::skip(fn () => $dbForProject->createDocument('users', $user)); } - $tokenSecret = Auth::tokenGenerator(Auth::TOKEN_LENGTH_MAGIC_URL); - $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_CONFIRM)); + $tokenSecret = Auth::tokenGenerator(TOKEN_LENGTH_MAGIC_URL); + $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM)); $token = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), - 'type' => Auth::TOKEN_TYPE_MAGIC_URL, + 'type' => TOKEN_TYPE_MAGIC_URL, 'secret' => Auth::hash($tokenSecret), // One way hash encryption to protect DB leak 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), @@ -2150,7 +2156,8 @@ App::post('/v1/account/tokens/email') ->inject('locale') ->inject('queueForEvents') ->inject('queueForMails') - ->action(function (string $userId, string $email, bool $phrase, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Locale $locale, Event $queueForEvents, Mail $queueForMails) { + ->inject('proofForPassword') + ->action(function (string $userId, string $email, bool $phrase, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Locale $locale, Event $queueForEvents, Mail $queueForMails, ProofsPassword $proofForPassword) { if (empty(System::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP disabled'); } @@ -2198,8 +2205,8 @@ App::post('/v1/account/tokens/email') 'emailVerification' => false, 'status' => true, 'password' => null, - 'hash' => Auth::DEFAULT_ALGO, - 'hashOptions' => Auth::DEFAULT_ALGO_OPTIONS, + 'hash' => $proofForPassword->getHash()->getName(), + 'hashOptions' => $proofForPassword->getHash()->getOptions(), 'passwordUpdate' => null, 'registration' => DateTime::now(), 'reset' => false, @@ -2216,13 +2223,13 @@ App::post('/v1/account/tokens/email') } $tokenSecret = Auth::codeGenerator(6); - $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_OTP)); + $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_OTP)); $token = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), - 'type' => Auth::TOKEN_TYPE_EMAIL, + 'type' => TOKEN_TYPE_EMAIL, 'secret' => Auth::hash($tokenSecret), // One way hash encryption to protect DB leak 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), @@ -2549,13 +2556,13 @@ App::post('/v1/account/tokens/phone') } $secret ??= Auth::codeGenerator(); - $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_OTP)); + $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_OTP)); $token = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), - 'type' => Auth::TOKEN_TYPE_PHONE, + 'type' => TOKEN_TYPE_PHONE, 'secret' => Auth::hash($secret), 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), @@ -2861,14 +2868,16 @@ App::patch('/v1/account/password') ->inject('dbForProject') ->inject('queueForEvents') ->inject('hooks') - ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) { + ->inject('proofForPassword') + ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks, ProofsPassword $proofForPassword) { + $userProofForPassword = ProofsPassword::createHash($user->getAttribute('hash'), $user->getAttribute('hashOptions')); // Check old password only if its an existing user. - if (!empty($user->getAttribute('passwordUpdate')) && !Auth::passwordVerify($oldPassword, $user->getAttribute('password'), $user->getAttribute('hash'), $user->getAttribute('hashOptions'))) { // Double check user password + if (!empty($user->getAttribute('passwordUpdate')) && !$userProofForPassword->verify($oldPassword, $user->getAttribute('password'))) { // Double check user password throw new Exception(Exception::USER_INVALID_CREDENTIALS); } - $newPassword = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS); + $newPassword = $proofForPassword->hash($password); $historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0; $history = $user->getAttribute('passwordHistory', []); if ($historyLimit > 0) { @@ -2894,8 +2903,8 @@ App::patch('/v1/account/password') ->setAttribute('password', $newPassword) ->setAttribute('passwordHistory', $history) ->setAttribute('passwordUpdate', DateTime::now()) - ->setAttribute('hash', Auth::DEFAULT_ALGO) - ->setAttribute('hashOptions', Auth::DEFAULT_ALGO_OPTIONS); + ->setAttribute('hash', $proofForPassword->getHash()->getName()) + ->setAttribute('hashOptions', $proofForPassword->getHash()->getOptions()); $user = $dbForProject->withRequestTimestamp($requestTimestamp, fn () => $dbForProject->updateDocument('users', $user->getId(), $user)); @@ -2938,9 +2947,11 @@ App::patch('/v1/account/email') // passwordUpdate will be empty if the user has never set a password $passwordUpdate = $user->getAttribute('passwordUpdate'); + $userProofForPassword = ProofsPassword::createHash($user->getAttribute('hash'), $user->getAttribute('hashOptions')); + if ( !empty($passwordUpdate) && - !$proofForPassword->verify($password, $user->getAttribute('password'), $user->getAttribute('hash'), $user->getAttribute('hashOptions')) + !$userProofForPassword->verify($password, $user->getAttribute('password')) ) { // Double check user password throw new Exception(Exception::USER_INVALID_CREDENTIALS); } @@ -2967,9 +2978,9 @@ App::patch('/v1/account/email') if (empty($passwordUpdate)) { $user - ->setAttribute('password', Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS)) - ->setAttribute('hash', Auth::DEFAULT_ALGO) - ->setAttribute('hashOptions', Auth::DEFAULT_ALGO_OPTIONS) + ->setAttribute('password', $proofForPassword->hash($password)) + ->setAttribute('hash', $proofForPassword->getHash()->getName()) + ->setAttribute('hashOptions', $proofForPassword->getHash()->getOptions()) ->setAttribute('passwordUpdate', DateTime::now()); } @@ -3030,13 +3041,16 @@ App::patch('/v1/account/phone') ->inject('queueForEvents') ->inject('project') ->inject('hooks') - ->action(function (string $phone, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Document $project, Hooks $hooks) { + ->inject('proofForPassword') + ->action(function (string $phone, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Document $project, Hooks $hooks, ProofsPassword $proofForPassword) { // passwordUpdate will be empty if the user has never set a password $passwordUpdate = $user->getAttribute('passwordUpdate'); + $userProofForPassword = ProofsPassword::createHash($user->getAttribute('hash'), $user->getAttribute('hashOptions')); + if ( !empty($passwordUpdate) && - !Auth::passwordVerify($password, $user->getAttribute('password'), $user->getAttribute('hash'), $user->getAttribute('hashOptions')) + !$userProofForPassword->verify($password, $user->getAttribute('password')) ) { // Double check user password throw new Exception(Exception::USER_INVALID_CREDENTIALS); } @@ -3060,9 +3074,9 @@ App::patch('/v1/account/phone') if (empty($passwordUpdate)) { $user - ->setAttribute('password', Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS)) - ->setAttribute('hash', Auth::DEFAULT_ALGO) - ->setAttribute('hashOptions', Auth::DEFAULT_ALGO_OPTIONS) + ->setAttribute('password', $proofForPassword->hash($password)) + ->setAttribute('hash', $proofForPassword->getHash()->getName()) + ->setAttribute('hashOptions', $proofForPassword->getHash()->getOptions()) ->setAttribute('passwordUpdate', DateTime::now()); } @@ -3233,14 +3247,14 @@ App::post('/v1/account/recovery') throw new Exception(Exception::USER_BLOCKED); } - $expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_RECOVERY); + $expire = DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_RECOVERY); - $secret = Auth::tokenGenerator(Auth::TOKEN_LENGTH_RECOVERY); + $secret = Auth::tokenGenerator(TOKEN_LENGTH_RECOVERY); $recovery = new Document([ '$id' => ID::unique(), 'userId' => $profile->getId(), 'userInternalId' => $profile->getInternalId(), - 'type' => Auth::TOKEN_TYPE_RECOVERY, + 'type' => TOKEN_TYPE_RECOVERY, 'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), @@ -3391,7 +3405,8 @@ App::put('/v1/account/recovery') ->inject('project') ->inject('queueForEvents') ->inject('hooks') - ->action(function (string $userId, string $secret, string $password, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents, Hooks $hooks) { + ->inject('proofForPassword') + ->action(function (string $userId, string $secret, string $password, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents, Hooks $hooks, ProofsPassword $proofForPassword) { $profile = $dbForProject->getDocument('users', $userId); if ($profile->isEmpty()) { @@ -3399,7 +3414,7 @@ App::put('/v1/account/recovery') } $tokens = $profile->getAttribute('tokens', []); - $verifiedToken = Auth::tokenVerify($tokens, Auth::TOKEN_TYPE_RECOVERY, $secret); + $verifiedToken = Auth::tokenVerify($tokens, TOKEN_TYPE_RECOVERY, $secret); if (!$verifiedToken) { throw new Exception(Exception::USER_INVALID_TOKEN); @@ -3407,7 +3422,7 @@ App::put('/v1/account/recovery') Authorization::setRole(Role::user($profile->getId())->toString()); - $newPassword = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS); + $newPassword = $proofForPassword->hash($password); $historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0; $history = $profile->getAttribute('passwordHistory', []); @@ -3427,8 +3442,8 @@ App::put('/v1/account/recovery') ->setAttribute('password', $newPassword) ->setAttribute('passwordHistory', $history) ->setAttribute('passwordUpdate', DateTime::now()) - ->setAttribute('hash', Auth::DEFAULT_ALGO) - ->setAttribute('hashOptions', Auth::DEFAULT_ALGO_OPTIONS) + ->setAttribute('hash', $proofForPassword->getHash()->getName()) + ->setAttribute('hashOptions', $proofForPassword->getHash()->getOptions()) ->setAttribute('emailVerification', true)); $user->setAttributes($profile->getArrayCopy()); @@ -3495,14 +3510,14 @@ App::post('/v1/account/verification') $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); $isAppUser = Auth::isAppUser($roles); - $verificationSecret = Auth::tokenGenerator(Auth::TOKEN_LENGTH_VERIFICATION); - $expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_CONFIRM); + $verificationSecret = Auth::tokenGenerator(TOKEN_LENGTH_VERIFICATION); + $expire = DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM); $verification = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), - 'type' => Auth::TOKEN_TYPE_VERIFICATION, + 'type' => TOKEN_TYPE_VERIFICATION, 'secret' => Auth::hash($verificationSecret), // One way hash encryption to protect DB leak 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), @@ -3658,7 +3673,7 @@ App::put('/v1/account/verification') } $tokens = $profile->getAttribute('tokens', []); - $verifiedToken = Auth::tokenVerify($tokens, Auth::TOKEN_TYPE_VERIFICATION, $secret); + $verifiedToken = Auth::tokenVerify($tokens, TOKEN_TYPE_VERIFICATION, $secret); if (!$verifiedToken) { throw new Exception(Exception::USER_INVALID_TOKEN); @@ -3750,13 +3765,13 @@ App::post('/v1/account/verification/phone') } $secret ??= Auth::codeGenerator(); - $expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_CONFIRM); + $expire = DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM); $verification = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), - 'type' => Auth::TOKEN_TYPE_PHONE, + 'type' => TOKEN_TYPE_PHONE, 'secret' => Auth::hash($secret), 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), @@ -3880,7 +3895,7 @@ App::put('/v1/account/verification/phone') throw new Exception(Exception::USER_NOT_FOUND); } - $verifiedToken = Auth::tokenVerify($user->getAttribute('tokens', []), Auth::TOKEN_TYPE_PHONE, $secret); + $verifiedToken = Auth::tokenVerify($user->getAttribute('tokens', []), TOKEN_TYPE_PHONE, $secret); if (!$verifiedToken) { throw new Exception(Exception::USER_INVALID_TOKEN); @@ -4354,7 +4369,7 @@ App::post('/v1/account/mfa/challenge') ->inject('plan') ->action(function (string $factor, Response $response, Database $dbForProject, Document $user, Locale $locale, Document $project, Request $request, Event $queueForEvents, Messaging $queueForMessaging, Mail $queueForMails, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan) { - $expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_CONFIRM); + $expire = DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM); $code = Auth::codeGenerator(); $challenge = new Document([ 'userId' => $user->getId(), diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 48d20cd17f..bb6055f1e8 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -117,7 +117,7 @@ App::post('/v1/projects') 'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT, 'passwordHistory' => 0, 'passwordDictionary' => false, - 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, + 'duration' => TOKEN_EXPIRATION_LOGIN_LONG, 'personalDataCheck' => false, 'mockNumbers' => [], 'sessionAlerts' => false, diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index b1b4445de0..b406bd03d7 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -451,7 +451,7 @@ App::post('/v1/teams/:teamId/memberships') ; $roles = array_keys(Config::getParam('roles', [])); array_filter($roles, function ($role) { - return !in_array($role, [Auth::USER_ROLE_APPS, Auth::USER_ROLE_GUESTS, Auth::USER_ROLE_USERS]); + return !in_array($role, [USER_ROLE_APPS, USER_ROLE_GUESTS, USER_ROLE_USERS]); }); return new ArrayList(new WhiteList($roles), APP_LIMIT_ARRAY_PARAMS_SIZE); } @@ -1038,7 +1038,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId') ; $roles = array_keys(Config::getParam('roles', [])); array_filter($roles, function ($role) { - return !in_array($role, [Auth::USER_ROLE_APPS, Auth::USER_ROLE_GUESTS, Auth::USER_ROLE_USERS]); + return !in_array($role, [USER_ROLE_APPS, USER_ROLE_GUESTS, USER_ROLE_USERS]); }); return new ArrayList(new WhiteList($roles), APP_LIMIT_ARRAY_PARAMS_SIZE); } @@ -1184,7 +1184,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') $detector = new Detector($request->getUserAgent('UNKNOWN')); $record = $geodb->get($request->getIP()); - $authDuration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $authDuration = $project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; $expire = DateTime::addSeconds(new \DateTime(), $authDuration); $secret = Auth::tokenGenerator(); $session = new Document(array_merge([ @@ -1196,7 +1196,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') ], 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), - 'provider' => Auth::SESSION_PROVIDER_EMAIL, + 'provider' => SESSION_PROVIDER_EMAIL, 'providerUid' => $user->getAttribute('email'), 'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak 'userAgent' => $request->getUserAgent('UNKNOWN'), diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 2fbbc423ba..1be2cdf236 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -2022,11 +2022,11 @@ App::post('/v1/users/:userId/sessions') throw new Exception(Exception::USER_NOT_FOUND); } - $secret = Auth::tokenGenerator(Auth::TOKEN_LENGTH_SESSION); + $secret = Auth::tokenGenerator(TOKEN_LENGTH_SESSION); $detector = new Detector($request->getUserAgent('UNKNOWN')); $record = $geodb->get($request->getIP()); - $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $duration = $project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); $session = new Document(array_merge( @@ -2034,7 +2034,7 @@ App::post('/v1/users/:userId/sessions') '$id' => ID::unique(), 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), - 'provider' => Auth::SESSION_PROVIDER_SERVER, + 'provider' => SESSION_PROVIDER_SERVER, 'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak 'userAgent' => $request->getUserAgent('UNKNOWN'), 'factors' => ['server'], @@ -2099,7 +2099,7 @@ App::post('/v1/users/:userId/tokens') )) ->param('userId', '', new UID(), 'User ID.') ->param('length', 6, new Range(4, 128), 'Token length in characters. The default length is 6 characters', true) - ->param('expire', Auth::TOKEN_EXPIRATION_GENERIC, new Range(60, Auth::TOKEN_EXPIRATION_LOGIN_LONG), 'Token expiration period in seconds. The default expiration is 15 minutes.', true) + ->param('expire', TOKEN_EXPIRATION_GENERIC, new Range(60, TOKEN_EXPIRATION_LOGIN_LONG), 'Token expiration period in seconds. The default expiration is 15 minutes.', true) ->inject('request') ->inject('response') ->inject('dbForProject') @@ -2118,7 +2118,7 @@ App::post('/v1/users/:userId/tokens') '$id' => ID::unique(), 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), - 'type' => Auth::TOKEN_TYPE_GENERIC, + 'type' => TOKEN_TYPE_GENERIC, 'secret' => Auth::hash($secret), 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 4fcdc12017..dfec602932 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -282,11 +282,11 @@ App::init() Authorization::setDefaultStatus(false); // Handle special app role case - if ($apiKey->getRole() === Auth::USER_ROLE_APPS) { + if ($apiKey->getRole() === USER_ROLE_APPS) { $user = new Document([ '$id' => '', 'status' => true, - 'type' => Auth::ACTIVITY_TYPE_APP, + 'type' => ACTIVITY_TYPE_APP, 'email' => 'app.' . $project->getId() . '@service.' . $request->getHostname(), 'password' => '', 'name' => $apiKey->getName(), @@ -551,7 +551,7 @@ App::init() if (!$user->isEmpty()) { $userClone = clone $user; // $user doesn't support `type` and can cause unintended effects. - $userClone->setAttribute('type', Auth::ACTIVITY_TYPE_USER); + $userClone->setAttribute('type', ACTIVITY_TYPE_USER); $queueForAudits->setUser($userClone); } @@ -773,7 +773,7 @@ App::shutdown() if (!$user->isEmpty()) { $userClone = clone $user; // $user doesn't support `type` and can cause unintended effects. - $userClone->setAttribute('type', Auth::ACTIVITY_TYPE_USER); + $userClone->setAttribute('type', ACTIVITY_TYPE_USER); $queueForAudits->setUser($userClone); } elseif ($queueForAudits->getUser() === null || $queueForAudits->getUser()->isEmpty()) { /** @@ -787,7 +787,7 @@ App::shutdown() $user = new Document([ '$id' => '', 'status' => true, - 'type' => Auth::ACTIVITY_TYPE_GUEST, + 'type' => ACTIVITY_TYPE_GUEST, 'email' => 'guest.' . $project->getId() . '@service.' . $request->getHostname(), 'password' => '', 'name' => 'Guest', diff --git a/app/controllers/shared/api/auth.php b/app/controllers/shared/api/auth.php index ecabc641ec..8f5e981362 100644 --- a/app/controllers/shared/api/auth.php +++ b/app/controllers/shared/api/auth.php @@ -20,7 +20,7 @@ App::init() $lastUpdate = $session->getAttribute('mfaUpdatedAt'); if (!empty($lastUpdate)) { $now = DateTime::now(); - $maxAllowedDate = DateTime::addSeconds(new \DateTime($lastUpdate), Auth::MFA_RECENT_DURATION); // Maximum date until session is considered safe before asking for another challenge + $maxAllowedDate = DateTime::addSeconds(new \DateTime($lastUpdate), MFA_RECENT_DURATION); // Maximum date until session is considered safe before asking for another challenge $isSessionFresh = DateTime::formatTz($maxAllowedDate) >= DateTime::formatTz($now); } diff --git a/app/init/constants.php b/app/init/constants.php index d46d3ed79c..44f8a36562 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -72,6 +72,72 @@ const APP_PLATFORM_SERVER = 'server'; const APP_PLATFORM_CLIENT = 'client'; const APP_PLATFORM_CONSOLE = 'console'; +// User Roles +const USER_ROLE_ANY = 'any'; +const USER_ROLE_GUESTS = 'guests'; +const USER_ROLE_USERS = 'users'; +const USER_ROLE_ADMIN = 'admin'; +const USER_ROLE_DEVELOPER = 'developer'; +const USER_ROLE_OWNER = 'owner'; +const USER_ROLE_APPS = 'apps'; +const USER_ROLE_SYSTEM = 'system'; + +/** + * Token Expiration times. + */ +const TOKEN_EXPIRATION_LOGIN_LONG = 31536000; /* 1 year */ +const TOKEN_EXPIRATION_LOGIN_SHORT = 3600; /* 1 hour */ +const TOKEN_EXPIRATION_RECOVERY = 3600; /* 1 hour */ +const TOKEN_EXPIRATION_CONFIRM = 3600 * 1; /* 1 hour */ +const TOKEN_EXPIRATION_OTP = 60 * 15; /* 15 minutes */ +const TOKEN_EXPIRATION_GENERIC = 60 * 15; /* 15 minutes */ + +/** + * Token Lengths. + */ +const TOKEN_LENGTH_MAGIC_URL = 64; +const TOKEN_LENGTH_VERIFICATION = 256; +const TOKEN_LENGTH_RECOVERY = 256; +const TOKEN_LENGTH_OAUTH2 = 64; +const TOKEN_LENGTH_SESSION = 256; + +/** + * Token Types. + */ +const TOKEN_TYPE_LOGIN = 1; // Deprecated +const TOKEN_TYPE_VERIFICATION = 2; +const TOKEN_TYPE_RECOVERY = 3; +const TOKEN_TYPE_INVITE = 4; +const TOKEN_TYPE_MAGIC_URL = 5; +const TOKEN_TYPE_PHONE = 6; +const TOKEN_TYPE_OAUTH2 = 7; +const TOKEN_TYPE_GENERIC = 8; +const TOKEN_TYPE_EMAIL = 9; // OTP + +/** + * Session Providers. + */ +const SESSION_PROVIDER_EMAIL = 'email'; +const SESSION_PROVIDER_ANONYMOUS = 'anonymous'; +const SESSION_PROVIDER_MAGIC_URL = 'magic-url'; +const SESSION_PROVIDER_PHONE = 'phone'; +const SESSION_PROVIDER_OAUTH2 = 'oauth2'; +const SESSION_PROVIDER_TOKEN = 'token'; +const SESSION_PROVIDER_SERVER = 'server'; + +/** + * Activity associated with user or the app. + */ +const ACTIVITY_TYPE_APP = 'app'; +const ACTIVITY_TYPE_USER = 'user'; +const ACTIVITY_TYPE_GUEST = 'guest'; + +/** + * MFA + */ +const MFA_RECENT_DURATION = 1800; // 30 mins + + // Database Reconnect const DATABASE_RECONNECT_SLEEP = 2; const DATABASE_RECONNECT_MAX_ATTEMPTS = 10; @@ -244,7 +310,6 @@ const METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId const METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage'; // Resource types - const RESOURCE_TYPE_PROJECTS = 'projects'; const RESOURCE_TYPE_FUNCTIONS = 'functions'; const RESOURCE_TYPE_SITES = 'sites'; diff --git a/composer.lock b/composer.lock index 050a92a1e3..f3377c1fec 100644 --- a/composer.lock +++ b/composer.lock @@ -3512,12 +3512,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/auth.git", - "reference": "966fbfefb27be94e3363f07279787d5cf8a66b95" + "reference": "ed49b9e481030ba5e589140b41a9f4be1486310f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/auth/zipball/966fbfefb27be94e3363f07279787d5cf8a66b95", - "reference": "966fbfefb27be94e3363f07279787d5cf8a66b95", + "url": "https://api.github.com/repos/utopia-php/auth/zipball/ed49b9e481030ba5e589140b41a9f4be1486310f", + "reference": "ed49b9e481030ba5e589140b41a9f4be1486310f", "shasum": "" }, "require": { @@ -3559,7 +3559,7 @@ "issues": "https://github.com/utopia-php/auth/issues", "source": "https://github.com/utopia-php/auth/tree/dev" }, - "time": "2025-03-16T18:32:00+00:00" + "time": "2025-03-17T19:57:57+00:00" }, { "name": "utopia-php/cache", @@ -4860,16 +4860,16 @@ }, { "name": "utopia-php/telemetry", - "version": "0.1.0", + "version": "0.1.1", "source": { "type": "git", "url": "https://github.com/utopia-php/telemetry.git", - "reference": "d35f2f0632f4ee0be63fb7ace6a94a6adda71a80" + "reference": "437f0021777f0e575dfb9e8a1a081b3aed75e33f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/d35f2f0632f4ee0be63fb7ace6a94a6adda71a80", - "reference": "d35f2f0632f4ee0be63fb7ace6a94a6adda71a80", + "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/437f0021777f0e575dfb9e8a1a081b3aed75e33f", + "reference": "437f0021777f0e575dfb9e8a1a081b3aed75e33f", "shasum": "" }, "require": { @@ -4890,7 +4890,7 @@ "type": "library", "autoload": { "psr-4": { - "Utopia\\": "src/" + "Utopia\\Telemetry\\": "src/Telemetry" } }, "notification-url": "https://packagist.org/downloads/", @@ -4904,9 +4904,9 @@ ], "support": { "issues": "https://github.com/utopia-php/telemetry/issues", - "source": "https://github.com/utopia-php/telemetry/tree/0.1.0" + "source": "https://github.com/utopia-php/telemetry/tree/0.1.1" }, - "time": "2024-11-13T10:29:53+00:00" + "time": "2025-03-17T11:57:52+00:00" }, { "name": "utopia-php/vcs", @@ -5143,16 +5143,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.40.7", + "version": "0.40.9", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "9e89b0bc4d8e6c81817d27096629f34a149fa873" + "reference": "dbb45a5db22cdc3368fe2573c07ba6088f188fa4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/9e89b0bc4d8e6c81817d27096629f34a149fa873", - "reference": "9e89b0bc4d8e6c81817d27096629f34a149fa873", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/dbb45a5db22cdc3368fe2573c07ba6088f188fa4", + "reference": "dbb45a5db22cdc3368fe2573c07ba6088f188fa4", "shasum": "" }, "require": { @@ -5188,9 +5188,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.40.7" + "source": "https://github.com/appwrite/sdk-generator/tree/0.40.9" }, - "time": "2025-03-12T08:43:55+00:00" + "time": "2025-03-17T18:39:14+00:00" }, { "name": "doctrine/annotations", diff --git a/src/Appwrite/Auth/Auth.php b/src/Appwrite/Auth/Auth.php index d47d9ec4b5..18b863b15b 100644 --- a/src/Appwrite/Auth/Auth.php +++ b/src/Appwrite/Auth/Auth.php @@ -2,13 +2,6 @@ namespace Appwrite\Auth; -use Appwrite\Auth\Hash\Argon2; -use Appwrite\Auth\Hash\Bcrypt; -use Appwrite\Auth\Hash\Md5; -use Appwrite\Auth\Hash\Phpass; -use Appwrite\Auth\Hash\Scrypt; -use Appwrite\Auth\Hash\Scryptmodified; -use Appwrite\Auth\Hash\Sha; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\Role; @@ -17,87 +10,6 @@ use Utopia\Database\Validator\Roles; class Auth { - public const SUPPORTED_ALGOS = [ - 'argon2', - 'bcrypt', - 'md5', - 'sha', - 'phpass', - 'scrypt', - 'scryptMod', - 'plaintext' - ]; - - public const DEFAULT_ALGO = 'argon2'; - public const DEFAULT_ALGO_OPTIONS = ['type' => 'argon2', 'memoryCost' => 2048, 'timeCost' => 4, 'threads' => 3]; - - /** - * User Roles. - */ - public const USER_ROLE_ANY = 'any'; - public const USER_ROLE_GUESTS = 'guests'; - public const USER_ROLE_USERS = 'users'; - public const USER_ROLE_ADMIN = 'admin'; - public const USER_ROLE_DEVELOPER = 'developer'; - public const USER_ROLE_OWNER = 'owner'; - public const USER_ROLE_APPS = 'apps'; - public const USER_ROLE_SYSTEM = 'system'; - - /** - * Activity associated with user or the app. - */ - public const ACTIVITY_TYPE_APP = 'app'; - public const ACTIVITY_TYPE_USER = 'user'; - public const ACTIVITY_TYPE_GUEST = 'guest'; - - /** - * Token Types. - */ - public const TOKEN_TYPE_LOGIN = 1; // Deprecated - public const TOKEN_TYPE_VERIFICATION = 2; - public const TOKEN_TYPE_RECOVERY = 3; - public const TOKEN_TYPE_INVITE = 4; - public const TOKEN_TYPE_MAGIC_URL = 5; - public const TOKEN_TYPE_PHONE = 6; - public const TOKEN_TYPE_OAUTH2 = 7; - public const TOKEN_TYPE_GENERIC = 8; - public const TOKEN_TYPE_EMAIL = 9; // OTP - - /** - * Session Providers. - */ - public const SESSION_PROVIDER_EMAIL = 'email'; - public const SESSION_PROVIDER_ANONYMOUS = 'anonymous'; - public const SESSION_PROVIDER_MAGIC_URL = 'magic-url'; - public const SESSION_PROVIDER_PHONE = 'phone'; - public const SESSION_PROVIDER_OAUTH2 = 'oauth2'; - public const SESSION_PROVIDER_TOKEN = 'token'; - public const SESSION_PROVIDER_SERVER = 'server'; - - /** - * Token Expiration times. - */ - public const TOKEN_EXPIRATION_LOGIN_LONG = 31536000; /* 1 year */ - public const TOKEN_EXPIRATION_LOGIN_SHORT = 3600; /* 1 hour */ - public const TOKEN_EXPIRATION_RECOVERY = 3600; /* 1 hour */ - public const TOKEN_EXPIRATION_CONFIRM = 3600 * 1; /* 1 hour */ - public const TOKEN_EXPIRATION_OTP = 60 * 15; /* 15 minutes */ - public const TOKEN_EXPIRATION_GENERIC = 60 * 15; /* 15 minutes */ - - /** - * Token Lengths. - */ - public const TOKEN_LENGTH_MAGIC_URL = 64; - public const TOKEN_LENGTH_VERIFICATION = 256; - public const TOKEN_LENGTH_RECOVERY = 256; - public const TOKEN_LENGTH_OAUTH2 = 64; - public const TOKEN_LENGTH_SESSION = 256; - - /** - * MFA - */ - public const MFA_RECENT_DURATION = 1800; // 30 mins - /** * @var string */ @@ -109,18 +21,18 @@ class Auth public static function getSessionProviderByTokenType(int $type): string { switch ($type) { - case Auth::TOKEN_TYPE_VERIFICATION: - case Auth::TOKEN_TYPE_RECOVERY: - case Auth::TOKEN_TYPE_INVITE: - return Auth::SESSION_PROVIDER_EMAIL; - case Auth::TOKEN_TYPE_MAGIC_URL: - return Auth::SESSION_PROVIDER_MAGIC_URL; - case Auth::TOKEN_TYPE_PHONE: - return Auth::SESSION_PROVIDER_PHONE; - case Auth::TOKEN_TYPE_OAUTH2: - return Auth::SESSION_PROVIDER_OAUTH2; + case TOKEN_TYPE_VERIFICATION: + case TOKEN_TYPE_RECOVERY: + case TOKEN_TYPE_INVITE: + return SESSION_PROVIDER_EMAIL; + case TOKEN_TYPE_MAGIC_URL: + return SESSION_PROVIDER_MAGIC_URL; + case TOKEN_TYPE_PHONE: + return SESSION_PROVIDER_PHONE; + case TOKEN_TYPE_OAUTH2: + return SESSION_PROVIDER_OAUTH2; default: - return Auth::SESSION_PROVIDER_TOKEN; + return SESSION_PROVIDER_TOKEN; } } @@ -138,105 +50,6 @@ class Auth return \hash('sha256', $string); } - /** - * Password Hash. - * - * One way string hashing for user passwords - * - * @param string $string - * @param string $algo hashing algorithm to use - * @param array $options algo-specific options - * - * @return bool|string|null - */ - public static function passwordHash(string $string, string $algo, array $options = []) - { - // Plain text not supported, just an alias. Switch to recommended algo - if ($algo === 'plaintext') { - $algo = Auth::DEFAULT_ALGO; - $options = Auth::DEFAULT_ALGO_OPTIONS; - } - - if (!\in_array($algo, Auth::SUPPORTED_ALGOS)) { - throw new \Exception('Hashing algorithm \'' . $algo . '\' is not supported.'); - } - - switch ($algo) { - case 'argon2': - $hasher = new Argon2($options); - return $hasher->hash($string); - case 'bcrypt': - $hasher = new Bcrypt($options); - return $hasher->hash($string); - case 'md5': - $hasher = new Md5($options); - return $hasher->hash($string); - case 'sha': - $hasher = new Sha($options); - return $hasher->hash($string); - case 'phpass': - $hasher = new Phpass($options); - return $hasher->hash($string); - case 'scrypt': - $hasher = new Scrypt($options); - return $hasher->hash($string); - case 'scryptMod': - $hasher = new Scryptmodified($options); - return $hasher->hash($string); - default: - throw new \Exception('Hashing algorithm \'' . $algo . '\' is not supported.'); - } - } - - /** - * Password verify. - * - * @param string $plain - * @param string $hash - * @param string $algo hashing algorithm used to hash - * @param array $options algo-specific options - * - * @return bool - */ - public static function passwordVerify(string $plain, string $hash, string $algo, array $options = []) - { - // Plain text not supported, just an alias. Switch to recommended algo - if ($algo === 'plaintext') { - $algo = Auth::DEFAULT_ALGO; - $options = Auth::DEFAULT_ALGO_OPTIONS; - } - - if (!\in_array($algo, Auth::SUPPORTED_ALGOS)) { - throw new \Exception('Hashing algorithm \'' . $algo . '\' is not supported.'); - } - - switch ($algo) { - case 'argon2': - $hasher = new Argon2($options); - return $hasher->verify($plain, $hash); - case 'bcrypt': - $hasher = new Bcrypt($options); - return $hasher->verify($plain, $hash); - case 'md5': - $hasher = new Md5($options); - return $hasher->verify($plain, $hash); - case 'sha': - $hasher = new Sha($options); - return $hasher->verify($plain, $hash); - case 'phpass': - $hasher = new Phpass($options); - return $hasher->verify($plain, $hash); - case 'scrypt': - $hasher = new Scrypt($options); - return $hasher->verify($plain, $hash); - case 'scryptMod': - $hasher = new Scryptmodified($options); - return $hasher->verify($plain, $hash); - default: - throw new \Exception('Hashing algorithm \'' . $algo . '\' is not supported.'); - } - } - /** * Token Generator. * @@ -339,9 +152,9 @@ class Auth public static function isPrivilegedUser(array $roles): bool { if ( - in_array(self::USER_ROLE_OWNER, $roles) || - in_array(self::USER_ROLE_DEVELOPER, $roles) || - in_array(self::USER_ROLE_ADMIN, $roles) + in_array(USER_ROLE_OWNER, $roles) || + in_array(USER_ROLE_DEVELOPER, $roles) || + in_array(USER_ROLE_ADMIN, $roles) ) { return true; } @@ -358,7 +171,7 @@ class Auth */ public static function isAppUser(array $roles): bool { - if (in_array(self::USER_ROLE_APPS, $roles)) { + if (in_array(USER_ROLE_APPS, $roles)) { return true; } diff --git a/src/Appwrite/Auth/Hash.php b/src/Appwrite/Auth/Hash.php deleted file mode 100644 index 7134057581..0000000000 --- a/src/Appwrite/Auth/Hash.php +++ /dev/null @@ -1,62 +0,0 @@ -setOptions($options); - } - - /** - * Set hashing algo options - * - * @param array $options Hashing-algo specific options - */ - public function setOptions(array $options): self - { - $this->options = \array_merge([], $this->getDefaultOptions(), $options); - return $this; - } - - /** - * Get hashing algo options - * - * @return array $options Hashing-algo specific options - */ - public function getOptions(): array - { - return $this->options; - } - - /** - * @param string $password Input password to hash - * - * @return string hash - */ - abstract public function hash(string $password): string; - - /** - * @param string $password Input password to validate - * @param string $hash Hash to verify password against - * - * @return boolean true if password matches hash - */ - abstract public function verify(string $password, string $hash): bool; - - /** - * Get default options for specific hashing algo - * - * @return array options named array - */ - abstract public function getDefaultOptions(): array; -} diff --git a/src/Appwrite/Auth/Key.php b/src/Appwrite/Auth/Key.php index 89c28c4727..fb6d2ceafe 100644 --- a/src/Appwrite/Auth/Key.php +++ b/src/Appwrite/Auth/Key.php @@ -104,16 +104,16 @@ class Key $secret = $key; } - $role = Auth::USER_ROLE_APPS; + $role = USER_ROLE_APPS; $roles = Config::getParam('roles', []); - $scopes = $roles[Auth::USER_ROLE_APPS]['scopes'] ?? []; + $scopes = $roles[USER_ROLE_APPS]['scopes'] ?? []; $expired = false; $guestKey = new Key( $project->getId(), $type, - Auth::USER_ROLE_GUESTS, - $roles[Auth::USER_ROLE_GUESTS]['scopes'] ?? [], + USER_ROLE_GUESTS, + $roles[USER_ROLE_GUESTS]['scopes'] ?? [], 'UNKNOWN' ); diff --git a/src/Appwrite/Auth/Validator/PasswordHistory.php b/src/Appwrite/Auth/Validator/PasswordHistory.php index f623ca180d..7677deafc0 100644 --- a/src/Appwrite/Auth/Validator/PasswordHistory.php +++ b/src/Appwrite/Auth/Validator/PasswordHistory.php @@ -2,7 +2,7 @@ namespace Appwrite\Auth\Validator; -use Appwrite\Auth\Auth; +use Utopia\Auth\Proofs\Password as ProofsPassword; /** * Password. @@ -45,8 +45,10 @@ class PasswordHistory extends Password */ public function isValid($value): bool { + $proofForPassword = ProofsPassword::createHash($this->algo, $this->algoOptions); + foreach ($this->history as $hash) { - if (!empty($hash) && Auth::passwordVerify($value, $hash, $this->algo, $this->algoOptions)) { + if (!empty($hash) && $proofForPassword->verify($value, $hash)) { return false; } } diff --git a/src/Appwrite/Migration/Version/V16.php b/src/Appwrite/Migration/Version/V16.php index 49f244598e..203505ce26 100644 --- a/src/Appwrite/Migration/Version/V16.php +++ b/src/Appwrite/Migration/Version/V16.php @@ -118,7 +118,7 @@ class V16 extends Migration * Set default authDuration */ $document->setAttribute('auths', array_merge($document->getAttribute('auths', []), [ - 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG + 'duration' => TOKEN_EXPIRATION_LOGIN_LONG ])); /** diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index 96c890c65d..46b4715a65 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -270,7 +270,7 @@ class V17 extends Migration * Set hashOptions type */ $document->setAttribute('hashOptions', array_merge($document->getAttribute('hashOptions', []), [ - 'type' => $document->getAttribute('hash', Auth::DEFAULT_ALGO) + 'type' => $document->getAttribute('hash', 'argon2') ])); break; } diff --git a/src/Appwrite/Migration/Version/V20.php b/src/Appwrite/Migration/Version/V20.php index 5a0807cedf..93115ed5ae 100644 --- a/src/Appwrite/Migration/Version/V20.php +++ b/src/Appwrite/Migration/Version/V20.php @@ -632,15 +632,15 @@ class V20 extends Migration } break; case 'sessions': - $duration = $this->project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $duration = $this->project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; $expire = DateTime::addSeconds(new \DateTime(), $duration); $document->setAttribute('expire', $expire); $factors = match ($document->getAttribute('provider')) { - Auth::SESSION_PROVIDER_EMAIL => ['password'], - Auth::SESSION_PROVIDER_PHONE => ['phone'], - Auth::SESSION_PROVIDER_ANONYMOUS => ['anonymous'], - Auth::SESSION_PROVIDER_TOKEN => ['token'], + SESSION_PROVIDER_EMAIL => ['password'], + SESSION_PROVIDER_PHONE => ['phone'], + SESSION_PROVIDER_ANONYMOUS => ['anonymous'], + SESSION_PROVIDER_TOKEN => ['token'], default => ['email'], }; diff --git a/src/Appwrite/Platform/Workers/Audits.php b/src/Appwrite/Platform/Workers/Audits.php index ed5ff8010a..c605d78b27 100644 --- a/src/Appwrite/Platform/Workers/Audits.php +++ b/src/Appwrite/Platform/Workers/Audits.php @@ -83,7 +83,7 @@ class Audits extends Action $userName = $user->getAttribute('name', ''); $userEmail = $user->getAttribute('email', ''); - $userType = $user->getAttribute('type', Auth::ACTIVITY_TYPE_USER); + $userType = $user->getAttribute('type', ACTIVITY_TYPE_USER); // Create event data $eventData = [ diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 95d58f8003..22d40f83fa 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -716,7 +716,7 @@ class Deletes extends Action private function deleteExpiredSessions(Document $project, callable $getProjectDB): void { $dbForProject = $getProjectDB($project); - $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $duration = $project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration); // Delete Sessions diff --git a/src/Appwrite/Utopia/Response/Model/Project.php b/src/Appwrite/Utopia/Response/Model/Project.php index fbbe062531..367796f5f4 100644 --- a/src/Appwrite/Utopia/Response/Model/Project.php +++ b/src/Appwrite/Utopia/Response/Model/Project.php @@ -105,7 +105,7 @@ class Project extends Model ->addRule('authDuration', [ 'type' => self::TYPE_INTEGER, 'description' => 'Session duration in seconds.', - 'default' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, + 'default' => TOKEN_EXPIRATION_LOGIN_LONG, 'example' => 60, ]) ->addRule('authLimit', [ @@ -359,7 +359,7 @@ class Project extends Model $auth = Config::getParam('auth', []); $document->setAttribute('authLimit', $authValues['limit'] ?? 0); - $document->setAttribute('authDuration', $authValues['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG); + $document->setAttribute('authDuration', $authValues['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG); $document->setAttribute('authSessionsLimit', $authValues['maxSessions'] ?? APP_LIMIT_USER_SESSIONS_DEFAULT); $document->setAttribute('authPasswordHistory', $authValues['passwordHistory'] ?? 0); $document->setAttribute('authPasswordDictionary', $authValues['passwordDictionary'] ?? false); diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index ed9171e46a..9e15b632bd 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -787,7 +787,7 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(Auth::TOKEN_EXPIRATION_LOGIN_LONG, $response['body']['authDuration']); // 1 Year + $this->assertEquals(TOKEN_EXPIRATION_LOGIN_LONG, $response['body']['authDuration']); // 1 Year /** * Test for SUCCESS @@ -931,7 +931,7 @@ class ProjectsConsoleClientTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, + 'duration' => TOKEN_EXPIRATION_LOGIN_LONG, ]); $this->assertEquals(200, $response['headers']['status-code']); @@ -944,7 +944,7 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(Auth::TOKEN_EXPIRATION_LOGIN_LONG, $response['body']['authDuration']); // 1 Year + $this->assertEquals(TOKEN_EXPIRATION_LOGIN_LONG, $response['body']['authDuration']); // 1 Year return ['projectId' => $projectId]; } diff --git a/tests/unit/Auth/AuthTest.php b/tests/unit/Auth/AuthTest.php index c2057394d3..e8cf938ce9 100644 --- a/tests/unit/Auth/AuthTest.php +++ b/tests/unit/Auth/AuthTest.php @@ -4,6 +4,7 @@ namespace Tests\Unit\Auth; use Appwrite\Auth\Auth; use PHPUnit\Framework\TestCase; +use Utopia\Auth\Proofs\Password; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; @@ -37,7 +38,7 @@ class AuthTest extends TestCase // Bcrypt - Version Y $plain = 'secret'; $hash = '$2y$08$PDbMtV18J1KOBI9tIYabBuyUwBrtXPGhLxCy9pWP6xkldVOKLrLKy'; - $generatedHash = Auth::passwordHash($plain, 'bcrypt'); + $generatedHash = Password::createHash('bcrypt')->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'bcrypt')); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'bcrypt')); $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'bcrypt')); @@ -45,7 +46,7 @@ class AuthTest extends TestCase // Bcrypt - Version A $plain = 'test123'; $hash = '$2a$12$3f2ZaARQ1AmhtQWx2nmQpuXcWfTj1YV2/Hl54e8uKxIzJe3IfwLiu'; - $generatedHash = Auth::passwordHash($plain, 'bcrypt'); + $generatedHash = Password::createHash('bcrypt')->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'bcrypt')); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'bcrypt')); $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'bcrypt')); @@ -53,7 +54,7 @@ class AuthTest extends TestCase // Bcrypt - Cost 5 $plain = 'hello-world'; $hash = '$2a$05$IjrtSz6SN7UJ6Sh3l.b5jODEvEG2LMJTPAHIaLWRvlWx7if3VMkFO'; - $generatedHash = Auth::passwordHash($plain, 'bcrypt'); + $generatedHash = Password::createHash('bcrypt')->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'bcrypt')); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'bcrypt')); $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'bcrypt')); @@ -61,7 +62,7 @@ class AuthTest extends TestCase // Bcrypt - Cost 15 $plain = 'super-secret-password'; $hash = '$2a$15$DS0ZzbsFZYumH/E4Qj5oeOHnBcM3nCCsCA2m4Goigat/0iMVQC4Na'; - $generatedHash = Auth::passwordHash($plain, 'bcrypt'); + $generatedHash = Password::createHash('bcrypt')->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'bcrypt')); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'bcrypt')); $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'bcrypt')); @@ -69,7 +70,7 @@ class AuthTest extends TestCase // MD5 - Short $plain = 'appwrite'; $hash = '144fa7eaa4904e8ee120651997f70dcc'; - $generatedHash = Auth::passwordHash($plain, 'md5'); + $generatedHash = Password::createHash('md5')->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'md5')); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'md5')); $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'md5')); @@ -77,7 +78,7 @@ class AuthTest extends TestCase // MD5 - Long $plain = 'AppwriteIsAwesomeBackendAsAServiceThatIsAlsoOpenSourced'; $hash = '8410e96cf7ac64e0b84c3f8517a82616'; - $generatedHash = Auth::passwordHash($plain, 'md5'); + $generatedHash = Password::createHash('md5')->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'md5')); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'md5')); $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'md5')); @@ -85,7 +86,7 @@ class AuthTest extends TestCase // PHPass $plain = 'pass123'; $hash = '$P$BVKPmJBZuLch27D4oiMRTEykGLQ9tX0'; - $generatedHash = Auth::passwordHash($plain, 'phpass'); + $generatedHash = Password::createHash('phpass')->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'phpass')); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'phpass')); $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'phpass')); @@ -93,7 +94,7 @@ class AuthTest extends TestCase // SHA $plain = 'developersAreAwesome!'; $hash = '2455118438cb125354b89bb5888346e9bd23355462c40df393fab514bf2220b5a08e4e2d7b85d7327595a450d0ac965cc6661152a46a157c66d681bed20a4735'; - $generatedHash = Auth::passwordHash($plain, 'sha'); + $generatedHash = Password::createHash('sha')->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'sha')); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'sha')); $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'sha')); @@ -101,7 +102,7 @@ class AuthTest extends TestCase // Argon2 $plain = 'safe-argon-password'; $hash = '$argon2id$v=19$m=2048,t=3,p=4$MWc5NWRmc2QxZzU2$41mp7rSgBZ49YxLbbxIac7aRaxfp5/e1G45ckwnK0g8'; - $generatedHash = Auth::passwordHash($plain, 'argon2'); + $generatedHash = Password::createHash('argon2')->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'argon2')); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'argon2')); $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'argon2')); @@ -109,7 +110,7 @@ class AuthTest extends TestCase // Scrypt $plain = 'some-scrypt-password'; $hash = 'b448ad7ba88b653b5b56b8053a06806724932d0751988bc9cd0ef7ff059e8ba8a020e1913b7069a650d3f99a1559aba0221f2c277826919513a054e76e339028'; - $generatedHash = Auth::passwordHash($plain, 'scrypt', [ 'salt' => 'some-salt', 'length' => 64, 'costCpu' => 16384, 'costMemory' => 12, 'costParallel' => 2]); + $generatedHash = Password::createHash('scrypt')->setOptions([ 'salt' => 'some-salt', 'length' => 64, 'costCpu' => 16384, 'costMemory' => 12, 'costParallel' => 2])->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'scrypt', [ 'salt' => 'some-salt', 'length' => 64, 'costCpu' => 16384, 'costMemory' => 12, 'costParallel' => 2])); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'scrypt', [ 'salt' => 'some-salt', 'length' => 64, 'costCpu' => 16384, 'costMemory' => 12, 'costParallel' => 2])); @@ -126,7 +127,7 @@ class AuthTest extends TestCase // Provider #1 (Database) $plain = 'example-password'; $hash = '$2a$10$3bIGRWUes86CICsuchGLj.e.BqdCdg2/1Ud9LvBhJr0j7Dze8PBdS'; - $generatedHash = Auth::passwordHash($plain, 'bcrypt'); + $generatedHash = Password::createHash('bcrypt')->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'bcrypt')); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'bcrypt')); $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'bcrypt')); @@ -134,7 +135,7 @@ class AuthTest extends TestCase // Provider #2 (Blog) $plain = 'your-password'; $hash = '$P$BkiNDJTpAWXtpaMhEUhUdrv7M0I1g6.'; - $generatedHash = Auth::passwordHash($plain, 'phpass'); + $generatedHash = Password::createHash('phpass')->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'phpass')); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'phpass')); $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'phpass')); @@ -147,7 +148,7 @@ class AuthTest extends TestCase $signerKey = 'XyEKE9RcTDeLEsL/RjwPDBv/RqDl8fb3gpYEOQaPihbxf1ZAtSOHCjuAAa7Q3oHpCYhXSN9tizHgVOwn6krflQ=='; $options = [ 'salt' => $salt, 'saltSeparator' => $saltSeparator, 'signerKey' => $signerKey ]; - $generatedHash = Auth::passwordHash($plain, 'scryptMod', $options); + $generatedHash = Password::createHash('scryptMod')->hash($plain, $options); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'scryptMod', $options)); $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'scryptMod', $options)); $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'scryptMod', $options)); @@ -159,7 +160,7 @@ class AuthTest extends TestCase // Bcrypt - Cost 5 $plain = 'whatIsMd8?!?'; - $generatedHash = Auth::passwordHash($plain, 'md8'); + $generatedHash = Password::createHash('md8')->hash($plain); $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'md8')); } @@ -187,14 +188,14 @@ class AuthTest extends TestCase new Document([ '$id' => ID::custom('token1'), 'secret' => $hash, - 'provider' => Auth::SESSION_PROVIDER_EMAIL, + 'provider' => SESSION_PROVIDER_EMAIL, 'providerUid' => 'test@example.com', 'expire' => DateTime::addSeconds(new \DateTime(), $expireTime1), ]), new Document([ '$id' => ID::custom('token2'), 'secret' => 'secret2', - 'provider' => Auth::SESSION_PROVIDER_EMAIL, + 'provider' => SESSION_PROVIDER_EMAIL, 'providerUid' => 'test@example.com', 'expire' => DateTime::addSeconds(new \DateTime(), $expireTime1), ]), @@ -206,14 +207,14 @@ class AuthTest extends TestCase new Document([ // Correct secret and type time, wrong expire time '$id' => ID::custom('token1'), 'secret' => $hash, - 'provider' => Auth::SESSION_PROVIDER_EMAIL, + 'provider' => SESSION_PROVIDER_EMAIL, 'providerUid' => 'test@example.com', 'expire' => DateTime::addSeconds(new \DateTime(), $expireTime2), ]), new Document([ '$id' => ID::custom('token2'), 'secret' => 'secret2', - 'provider' => Auth::SESSION_PROVIDER_EMAIL, + 'provider' => SESSION_PROVIDER_EMAIL, 'providerUid' => 'test@example.com', 'expire' => DateTime::addSeconds(new \DateTime(), $expireTime2), ]), @@ -232,13 +233,13 @@ class AuthTest extends TestCase $tokens1 = [ new Document([ '$id' => ID::custom('token1'), - 'type' => Auth::TOKEN_TYPE_RECOVERY, + 'type' => TOKEN_TYPE_RECOVERY, 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), 60 * 60 * 24)), 'secret' => $hash, ]), new Document([ '$id' => ID::custom('token2'), - 'type' => Auth::TOKEN_TYPE_RECOVERY, + 'type' => TOKEN_TYPE_RECOVERY, 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -60 * 60 * 24)), 'secret' => 'secret2', ]), @@ -247,13 +248,13 @@ class AuthTest extends TestCase $tokens2 = [ new Document([ // Correct secret and type time, wrong expire time '$id' => ID::custom('token1'), - 'type' => Auth::TOKEN_TYPE_RECOVERY, + 'type' => TOKEN_TYPE_RECOVERY, 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -60 * 60 * 24)), 'secret' => $hash, ]), new Document([ '$id' => ID::custom('token2'), - 'type' => Auth::TOKEN_TYPE_RECOVERY, + 'type' => TOKEN_TYPE_RECOVERY, 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -60 * 60 * 24)), 'secret' => 'secret2', ]), @@ -262,25 +263,25 @@ class AuthTest extends TestCase $tokens3 = [ // Correct secret and expire time, wrong type new Document([ '$id' => ID::custom('token1'), - 'type' => Auth::TOKEN_TYPE_INVITE, + 'type' => TOKEN_TYPE_INVITE, 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), 60 * 60 * 24)), 'secret' => $hash, ]), new Document([ '$id' => ID::custom('token2'), - 'type' => Auth::TOKEN_TYPE_RECOVERY, + 'type' => TOKEN_TYPE_RECOVERY, 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -60 * 60 * 24)), 'secret' => 'secret2', ]), ]; - $this->assertEquals(Auth::tokenVerify($tokens1, Auth::TOKEN_TYPE_RECOVERY, $secret), $tokens1[0]); + $this->assertEquals(Auth::tokenVerify($tokens1, TOKEN_TYPE_RECOVERY, $secret), $tokens1[0]); $this->assertEquals(Auth::tokenVerify($tokens1, null, $secret), $tokens1[0]); - $this->assertEquals(Auth::tokenVerify($tokens1, Auth::TOKEN_TYPE_RECOVERY, 'false-secret'), false); - $this->assertEquals(Auth::tokenVerify($tokens2, Auth::TOKEN_TYPE_RECOVERY, $secret), false); - $this->assertEquals(Auth::tokenVerify($tokens2, Auth::TOKEN_TYPE_RECOVERY, 'false-secret'), false); - $this->assertEquals(Auth::tokenVerify($tokens3, Auth::TOKEN_TYPE_RECOVERY, $secret), false); - $this->assertEquals(Auth::tokenVerify($tokens3, Auth::TOKEN_TYPE_RECOVERY, 'false-secret'), false); + $this->assertEquals(Auth::tokenVerify($tokens1, TOKEN_TYPE_RECOVERY, 'false-secret'), false); + $this->assertEquals(Auth::tokenVerify($tokens2, TOKEN_TYPE_RECOVERY, $secret), false); + $this->assertEquals(Auth::tokenVerify($tokens2, TOKEN_TYPE_RECOVERY, 'false-secret'), false); + $this->assertEquals(Auth::tokenVerify($tokens3, TOKEN_TYPE_RECOVERY, $secret), false); + $this->assertEquals(Auth::tokenVerify($tokens3, TOKEN_TYPE_RECOVERY, 'false-secret'), false); } public function testIsPrivilegedUser(): void @@ -288,16 +289,16 @@ class AuthTest extends TestCase $this->assertEquals(false, Auth::isPrivilegedUser([])); $this->assertEquals(false, Auth::isPrivilegedUser([Role::guests()->toString()])); $this->assertEquals(false, Auth::isPrivilegedUser([Role::users()->toString()])); - $this->assertEquals(true, Auth::isPrivilegedUser([Auth::USER_ROLE_ADMIN])); - $this->assertEquals(true, Auth::isPrivilegedUser([Auth::USER_ROLE_DEVELOPER])); - $this->assertEquals(true, Auth::isPrivilegedUser([Auth::USER_ROLE_OWNER])); - $this->assertEquals(false, Auth::isPrivilegedUser([Auth::USER_ROLE_APPS])); - $this->assertEquals(false, Auth::isPrivilegedUser([Auth::USER_ROLE_SYSTEM])); + $this->assertEquals(true, Auth::isPrivilegedUser([USER_ROLE_ADMIN])); + $this->assertEquals(true, Auth::isPrivilegedUser([USER_ROLE_DEVELOPER])); + $this->assertEquals(true, Auth::isPrivilegedUser([USER_ROLE_OWNER])); + $this->assertEquals(false, Auth::isPrivilegedUser([USER_ROLE_APPS])); + $this->assertEquals(false, Auth::isPrivilegedUser([USER_ROLE_SYSTEM])); - $this->assertEquals(false, Auth::isPrivilegedUser([Auth::USER_ROLE_APPS, Auth::USER_ROLE_APPS])); - $this->assertEquals(false, Auth::isPrivilegedUser([Auth::USER_ROLE_APPS, Role::guests()->toString()])); - $this->assertEquals(true, Auth::isPrivilegedUser([Auth::USER_ROLE_OWNER, Role::guests()->toString()])); - $this->assertEquals(true, Auth::isPrivilegedUser([Auth::USER_ROLE_OWNER, Auth::USER_ROLE_ADMIN, Auth::USER_ROLE_DEVELOPER])); + $this->assertEquals(false, Auth::isPrivilegedUser([USER_ROLE_APPS, USER_ROLE_APPS])); + $this->assertEquals(false, Auth::isPrivilegedUser([USER_ROLE_APPS, Role::guests()->toString()])); + $this->assertEquals(true, Auth::isPrivilegedUser([USER_ROLE_OWNER, Role::guests()->toString()])); + $this->assertEquals(true, Auth::isPrivilegedUser([USER_ROLE_OWNER, USER_ROLE_ADMIN, USER_ROLE_DEVELOPER])); } public function testIsAppUser(): void @@ -305,16 +306,16 @@ class AuthTest extends TestCase $this->assertEquals(false, Auth::isAppUser([])); $this->assertEquals(false, Auth::isAppUser([Role::guests()->toString()])); $this->assertEquals(false, Auth::isAppUser([Role::users()->toString()])); - $this->assertEquals(false, Auth::isAppUser([Auth::USER_ROLE_ADMIN])); - $this->assertEquals(false, Auth::isAppUser([Auth::USER_ROLE_DEVELOPER])); - $this->assertEquals(false, Auth::isAppUser([Auth::USER_ROLE_OWNER])); - $this->assertEquals(true, Auth::isAppUser([Auth::USER_ROLE_APPS])); - $this->assertEquals(false, Auth::isAppUser([Auth::USER_ROLE_SYSTEM])); + $this->assertEquals(false, Auth::isAppUser([USER_ROLE_ADMIN])); + $this->assertEquals(false, Auth::isAppUser([USER_ROLE_DEVELOPER])); + $this->assertEquals(false, Auth::isAppUser([USER_ROLE_OWNER])); + $this->assertEquals(true, Auth::isAppUser([USER_ROLE_APPS])); + $this->assertEquals(false, Auth::isAppUser([USER_ROLE_SYSTEM])); - $this->assertEquals(true, Auth::isAppUser([Auth::USER_ROLE_APPS, Auth::USER_ROLE_APPS])); - $this->assertEquals(true, Auth::isAppUser([Auth::USER_ROLE_APPS, Role::guests()->toString()])); - $this->assertEquals(false, Auth::isAppUser([Auth::USER_ROLE_OWNER, Role::guests()->toString()])); - $this->assertEquals(false, Auth::isAppUser([Auth::USER_ROLE_OWNER, Auth::USER_ROLE_ADMIN, Auth::USER_ROLE_DEVELOPER])); + $this->assertEquals(true, Auth::isAppUser([USER_ROLE_APPS, USER_ROLE_APPS])); + $this->assertEquals(true, Auth::isAppUser([USER_ROLE_APPS, Role::guests()->toString()])); + $this->assertEquals(false, Auth::isAppUser([USER_ROLE_OWNER, Role::guests()->toString()])); + $this->assertEquals(false, Auth::isAppUser([USER_ROLE_OWNER, USER_ROLE_ADMIN, USER_ROLE_DEVELOPER])); } public function testGuestRoles(): void @@ -394,7 +395,7 @@ class AuthTest extends TestCase public function testPrivilegedUserRoles(): void { - Authorization::setRole(Auth::USER_ROLE_OWNER); + Authorization::setRole(USER_ROLE_OWNER); $user = new Document([ '$id' => ID::custom('123'), 'emailVerification' => true, @@ -438,7 +439,7 @@ class AuthTest extends TestCase public function testAppUserRoles(): void { - Authorization::setRole(Auth::USER_ROLE_APPS); + Authorization::setRole(USER_ROLE_APPS); $user = new Document([ '$id' => ID::custom('123'), 'memberships' => [ diff --git a/tests/unit/Auth/KeyTest.php b/tests/unit/Auth/KeyTest.php index 8ae2114697..5ca6135dd0 100644 --- a/tests/unit/Auth/KeyTest.php +++ b/tests/unit/Auth/KeyTest.php @@ -21,7 +21,7 @@ class KeyTest extends TestCase 'collections.read', 'documents.read', ]; - $roleScopes = Config::getParam('roles', [])[Auth::USER_ROLE_APPS]['scopes']; + $roleScopes = Config::getParam('roles', [])[USER_ROLE_APPS]['scopes']; $key = static::generateKey($projectId, $usage, $scopes); $project = new Document(['$id' => $projectId,]); @@ -29,7 +29,7 @@ class KeyTest extends TestCase $this->assertEquals($projectId, $decoded->getProjectId()); $this->assertEquals(API_KEY_DYNAMIC, $decoded->getType()); - $this->assertEquals(Auth::USER_ROLE_APPS, $decoded->getRole()); + $this->assertEquals(USER_ROLE_APPS, $decoded->getRole()); $this->assertEquals(\array_merge($scopes, $roleScopes), $decoded->getScopes()); } diff --git a/tests/unit/Messaging/MessagingChannelsTest.php b/tests/unit/Messaging/MessagingChannelsTest.php index 8ba0374093..536228b504 100644 --- a/tests/unit/Messaging/MessagingChannelsTest.php +++ b/tests/unit/Messaging/MessagingChannelsTest.php @@ -59,7 +59,7 @@ class MessagingChannelsTest extends TestCase 'confirm' => true, 'roles' => [ empty($index % 2) - ? Auth::USER_ROLE_ADMIN + ? USER_ROLE_ADMIN : 'member', ] ] @@ -294,7 +294,7 @@ class MessagingChannelsTest extends TestCase } $role = empty($index % 2) - ? Auth::USER_ROLE_ADMIN + ? USER_ROLE_ADMIN : 'member'; $permissions = [ From 477add30229c9c145805f341fec58ae25dcc94b6 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 17 Mar 2025 21:49:10 +0100 Subject: [PATCH 015/245] Formatting --- app/config/collections/common.php | 1 - app/config/console.php | 1 - app/config/roles.php | 1 - app/controllers/api/projects.php | 1 - src/Appwrite/Migration/Version/V16.php | 1 - src/Appwrite/Migration/Version/V17.php | 1 - src/Appwrite/Migration/Version/V20.php | 1 - src/Appwrite/Platform/Workers/Audits.php | 1 - src/Appwrite/Platform/Workers/Deletes.php | 1 - tests/e2e/Services/Projects/ProjectsConsoleClientTest.php | 1 - tests/unit/Auth/KeyTest.php | 1 - 11 files changed, 11 deletions(-) diff --git a/app/config/collections/common.php b/app/config/collections/common.php index 7e7da1c94d..00ef59968d 100644 --- a/app/config/collections/common.php +++ b/app/config/collections/common.php @@ -1,6 +1,5 @@ Date: Mon, 17 Mar 2025 22:51:22 +0100 Subject: [PATCH 016/245] Improved errors --- app/controllers/general.php | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 3afe1d8a3d..b099031036 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -977,17 +977,29 @@ App::error() $trace = $error->getTrace(); if (php_sapi_name() === 'cli') { - Console::error('[Error] Timestamp: ' . date('c', time())); + $logLevel = $code >= 500 ? 'error' : 'warning'; + $logPrefix = $code >= 500 ? '[Error]' : '[Warning]'; + + Console::$logLevel($logPrefix . ' Timestamp: ' . date('c', time())); if ($route) { - Console::error('[Error] Method: ' . $route->getMethod()); - Console::error('[Error] URL: ' . $route->getPath()); + Console::$logLevel($logPrefix . ' Status Code: ' . $code); + Console::$logLevel($logPrefix . ' Method: ' . $route->getMethod()); + Console::$logLevel($logPrefix . ' URL: ' . $route->getPath()); + } + Console::$logLevel($logPrefix . ' Type: ' . get_class($error)); + Console::$logLevel($logPrefix . ' Message: ' . $message); + Console::$logLevel($logPrefix . ' File: ' . $file); + Console::$logLevel($logPrefix . ' Line: ' . $line); + Console::$logLevel($logPrefix . ' Trace:'); + foreach ($trace as $index => $entry) { + $file = $entry['file'] ?? 'unknown'; + $line = $entry['line'] ?? 0; + $function = $entry['function'] ?? 'unknown'; + $class = $entry['class'] ?? ''; + $type = $entry['type'] ?? ''; + Console::$logLevel(" #{$index} {$file}({$line}): {$class}{$type}{$function}()"); } - - Console::error('[Error] Type: ' . get_class($error)); - Console::error('[Error] Message: ' . $message); - Console::error('[Error] File: ' . $file); - Console::error('[Error] Line: ' . $line); } switch ($class) { From f537091eb23824e56aa5d93a044282572c542801 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 17 Mar 2025 23:57:44 +0100 Subject: [PATCH 017/245] Fixed tests --- app/controllers/api/account.php | 9 ++++++--- app/controllers/general.php | 18 +++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 812b454cac..4159b2eee0 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -902,7 +902,7 @@ App::post('/v1/account/sessions/email') Query::equal('email', [$email]), ]); - $userProofForPassword = ProofsPassword::createHash($profile->getAttribute('hash'), $profile->getAttribute('hashOptions')); + $userProofForPassword = ProofsPassword::createHash($profile->getAttribute('hash', $proofForPassword->getHash()->getName()), $profile->getAttribute('hashOptions', $proofForPassword->getHash()->getOptions())); if ($profile->isEmpty() || empty($profile->getAttribute('passwordUpdate')) || !$userProofForPassword->verify($password, $profile->getAttribute('password'))) { throw new Exception(Exception::USER_INVALID_CREDENTIALS); @@ -1457,7 +1457,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $name = ''; $nameOAuth = $oauth2->getUserName($accessToken); - $userParam = \json_decode($request->getParam('user'), true); + $userParam = \json_decode($request->getParam('user', '{}'), true); // only valid for Apple OAuth2 which returns a user param in the request if (!empty($nameOAuth)) { $name = $nameOAuth; } elseif (is_array($userParam)) { @@ -2472,7 +2472,8 @@ App::post('/v1/account/tokens/phone') ->inject('queueForStatsUsage') ->inject('plan') ->inject('store') - ->action(function (string $userId, string $phone, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, Store $store) { + ->inject('proofForPassword') + ->action(function (string $userId, string $phone, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, Store $store, ProofsPassword $proofForPassword) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -2510,6 +2511,8 @@ App::post('/v1/account/tokens/phone') 'status' => true, 'password' => null, 'passwordUpdate' => null, + 'hash' => $proofForPassword->getHash()->getName(), + 'hashOptions' => $proofForPassword->getHash()->getOptions(), 'registration' => DateTime::now(), 'reset' => false, 'prefs' => new \stdClass(), diff --git a/app/controllers/general.php b/app/controllers/general.php index b099031036..77da6189c7 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -978,14 +978,13 @@ App::error() if (php_sapi_name() === 'cli') { $logLevel = $code >= 500 ? 'error' : 'warning'; - $logPrefix = $code >= 500 ? '[Error]' : '[Warning]'; + $logPrefix = $code >= 500 || $code == 0 ? '[Error]' : '[Warning]'; Console::$logLevel($logPrefix . ' Timestamp: ' . date('c', time())); if ($route) { Console::$logLevel($logPrefix . ' Status Code: ' . $code); - Console::$logLevel($logPrefix . ' Method: ' . $route->getMethod()); - Console::$logLevel($logPrefix . ' URL: ' . $route->getPath()); + Console::$logLevel($logPrefix . ' URL: ' . $route->getMethod() . ' ' . $route->getPath()); } Console::$logLevel($logPrefix . ' Type: ' . get_class($error)); Console::$logLevel($logPrefix . ' Message: ' . $message); @@ -993,13 +992,14 @@ App::error() Console::$logLevel($logPrefix . ' Line: ' . $line); Console::$logLevel($logPrefix . ' Trace:'); foreach ($trace as $index => $entry) { - $file = $entry['file'] ?? 'unknown'; - $line = $entry['line'] ?? 0; - $function = $entry['function'] ?? 'unknown'; - $class = $entry['class'] ?? ''; - $type = $entry['type'] ?? ''; - Console::$logLevel(" #{$index} {$file}({$line}): {$class}{$type}{$function}()"); + $traceFile = $entry['file'] ?? 'unknown'; + $traceLine = $entry['line'] ?? 0; + $traceFunction = $entry['function'] ?? 'unknown'; + $traceClass = $entry['class'] ?? ''; + $traceType = $entry['type'] ?? ''; + Console::$logLevel(" #{$index} {$traceFile}({$traceLine}): {$traceClass}{$traceType}{$traceFunction}()"); } + Console::$logLevel(''); } switch ($class) { From 0180f72067939ce59c2489736be9253d12937b49 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 18 Mar 2025 00:30:31 +0100 Subject: [PATCH 018/245] Removed unsed methods and tests --- src/Appwrite/Auth/Auth.php | 12 --- tests/unit/Auth/AuthTest.php | 137 +---------------------------------- 2 files changed, 1 insertion(+), 148 deletions(-) diff --git a/src/Appwrite/Auth/Auth.php b/src/Appwrite/Auth/Auth.php index 18b863b15b..d644483b74 100644 --- a/src/Appwrite/Auth/Auth.php +++ b/src/Appwrite/Auth/Auth.php @@ -231,16 +231,4 @@ class Auth return $roles; } - - /** - * Check if user is anonymous. - * - * @param Document $user - * @return bool - */ - public static function isAnonymousUser(Document $user): bool - { - return is_null($user->getAttribute('email')) - && is_null($user->getAttribute('phone')); - } } diff --git a/tests/unit/Auth/AuthTest.php b/tests/unit/Auth/AuthTest.php index e8cf938ce9..02d433106f 100644 --- a/tests/unit/Auth/AuthTest.php +++ b/tests/unit/Auth/AuthTest.php @@ -28,142 +28,7 @@ class AuthTest extends TestCase $secret = 'secret'; $this->assertEquals(Auth::hash($secret), '2bb80d537b1da3e38bd30361aa855686bde0eacd7162fef6a25fe97bf527a25b'); } - - public function testPassword(): void - { - /* - General tests, using pre-defined hashes generated by online tools - */ - - // Bcrypt - Version Y - $plain = 'secret'; - $hash = '$2y$08$PDbMtV18J1KOBI9tIYabBuyUwBrtXPGhLxCy9pWP6xkldVOKLrLKy'; - $generatedHash = Password::createHash('bcrypt')->hash($plain); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'bcrypt')); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'bcrypt')); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'bcrypt')); - - // Bcrypt - Version A - $plain = 'test123'; - $hash = '$2a$12$3f2ZaARQ1AmhtQWx2nmQpuXcWfTj1YV2/Hl54e8uKxIzJe3IfwLiu'; - $generatedHash = Password::createHash('bcrypt')->hash($plain); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'bcrypt')); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'bcrypt')); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'bcrypt')); - - // Bcrypt - Cost 5 - $plain = 'hello-world'; - $hash = '$2a$05$IjrtSz6SN7UJ6Sh3l.b5jODEvEG2LMJTPAHIaLWRvlWx7if3VMkFO'; - $generatedHash = Password::createHash('bcrypt')->hash($plain); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'bcrypt')); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'bcrypt')); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'bcrypt')); - - // Bcrypt - Cost 15 - $plain = 'super-secret-password'; - $hash = '$2a$15$DS0ZzbsFZYumH/E4Qj5oeOHnBcM3nCCsCA2m4Goigat/0iMVQC4Na'; - $generatedHash = Password::createHash('bcrypt')->hash($plain); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'bcrypt')); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'bcrypt')); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'bcrypt')); - - // MD5 - Short - $plain = 'appwrite'; - $hash = '144fa7eaa4904e8ee120651997f70dcc'; - $generatedHash = Password::createHash('md5')->hash($plain); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'md5')); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'md5')); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'md5')); - - // MD5 - Long - $plain = 'AppwriteIsAwesomeBackendAsAServiceThatIsAlsoOpenSourced'; - $hash = '8410e96cf7ac64e0b84c3f8517a82616'; - $generatedHash = Password::createHash('md5')->hash($plain); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'md5')); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'md5')); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'md5')); - - // PHPass - $plain = 'pass123'; - $hash = '$P$BVKPmJBZuLch27D4oiMRTEykGLQ9tX0'; - $generatedHash = Password::createHash('phpass')->hash($plain); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'phpass')); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'phpass')); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'phpass')); - - // SHA - $plain = 'developersAreAwesome!'; - $hash = '2455118438cb125354b89bb5888346e9bd23355462c40df393fab514bf2220b5a08e4e2d7b85d7327595a450d0ac965cc6661152a46a157c66d681bed20a4735'; - $generatedHash = Password::createHash('sha')->hash($plain); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'sha')); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'sha')); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'sha')); - - // Argon2 - $plain = 'safe-argon-password'; - $hash = '$argon2id$v=19$m=2048,t=3,p=4$MWc5NWRmc2QxZzU2$41mp7rSgBZ49YxLbbxIac7aRaxfp5/e1G45ckwnK0g8'; - $generatedHash = Password::createHash('argon2')->hash($plain); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'argon2')); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'argon2')); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'argon2')); - - // Scrypt - $plain = 'some-scrypt-password'; - $hash = 'b448ad7ba88b653b5b56b8053a06806724932d0751988bc9cd0ef7ff059e8ba8a020e1913b7069a650d3f99a1559aba0221f2c277826919513a054e76e339028'; - $generatedHash = Password::createHash('scrypt')->setOptions([ 'salt' => 'some-salt', 'length' => 64, 'costCpu' => 16384, 'costMemory' => 12, 'costParallel' => 2])->hash($plain); - - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'scrypt', [ 'salt' => 'some-salt', 'length' => 64, 'costCpu' => 16384, 'costMemory' => 12, 'costParallel' => 2])); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'scrypt', [ 'salt' => 'some-salt', 'length' => 64, 'costCpu' => 16384, 'costMemory' => 12, 'costParallel' => 2])); - $this->assertEquals(false, Auth::passwordVerify($plain, $hash, 'scrypt', [ 'salt' => 'some-wrong-salt', 'length' => 64, 'costCpu' => 16384, 'costMemory' => 12, 'costParallel' => 2])); - $this->assertEquals(false, Auth::passwordVerify($plain, $hash, 'scrypt', [ 'salt' => 'some-salt', 'length' => 64, 'costCpu' => 16384, 'costMemory' => 10, 'costParallel' => 2])); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'scrypt', [ 'salt' => 'some-salt', 'length' => 64, 'costCpu' => 16384, 'costMemory' => 12, 'costParallel' => 2])); - - // ScryptModified tested are in provider-specific tests below - - /* - Provider-specific tests, ensuring functionality of specific use-cases - */ - - // Provider #1 (Database) - $plain = 'example-password'; - $hash = '$2a$10$3bIGRWUes86CICsuchGLj.e.BqdCdg2/1Ud9LvBhJr0j7Dze8PBdS'; - $generatedHash = Password::createHash('bcrypt')->hash($plain); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'bcrypt')); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'bcrypt')); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'bcrypt')); - - // Provider #2 (Blog) - $plain = 'your-password'; - $hash = '$P$BkiNDJTpAWXtpaMhEUhUdrv7M0I1g6.'; - $generatedHash = Password::createHash('phpass')->hash($plain); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'phpass')); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'phpass')); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'phpass')); - - // Provider #2 (Google) - $plain = 'users-password'; - $hash = 'EPKgfALpS9Tvgr/y1ki7ubY4AEGJeWL3teakrnmOacN4XGiyD00lkzEHgqCQ71wGxoi/zb7Y9a4orOtvMV3/Jw=='; - $salt = '56dFqW+kswqktw=='; - $saltSeparator = 'Bw=='; - $signerKey = 'XyEKE9RcTDeLEsL/RjwPDBv/RqDl8fb3gpYEOQaPihbxf1ZAtSOHCjuAAa7Q3oHpCYhXSN9tizHgVOwn6krflQ=='; - - $options = [ 'salt' => $salt, 'saltSeparator' => $saltSeparator, 'signerKey' => $signerKey ]; - $generatedHash = Password::createHash('scryptMod')->hash($plain, $options); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'scryptMod', $options)); - $this->assertEquals(true, Auth::passwordVerify($plain, $hash, 'scryptMod', $options)); - $this->assertEquals(false, Auth::passwordVerify('wrongPassword', $hash, 'scryptMod', $options)); - } - - public function testUnknownAlgo() - { - $this->expectExceptionMessage('Hashing algorithm \'md8\' is not supported.'); - - // Bcrypt - Cost 5 - $plain = 'whatIsMd8?!?'; - $generatedHash = Password::createHash('md8')->hash($plain); - $this->assertEquals(true, Auth::passwordVerify($plain, $generatedHash, 'md8')); - } - + public function testTokenGenerator(): void { $this->assertEquals(\strlen(Auth::tokenGenerator()), 256); From 84cec0e32cd7aa5379849b01e2bb55d01f54800a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 18 Mar 2025 10:18:29 +0100 Subject: [PATCH 019/245] format --- app/controllers/api/account.php | 4 ++-- app/controllers/api/users.php | 1 - src/Appwrite/Auth/MFA/Type.php | 1 - src/Appwrite/Platform/Tasks/Install.php | 1 - tests/unit/Auth/AuthTest.php | 1 - 5 files changed, 2 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 8f7d615ca7..0a157c37fe 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -38,9 +38,9 @@ use MaxMind\Db\Reader; use Utopia\Abuse\Abuse; use Utopia\App; use Utopia\Audit\Audit as EventAudit; +use Utopia\Auth\Proofs\Code as ProofsCode; use Utopia\Auth\Proofs\Password as ProofsPassword; use Utopia\Auth\Proofs\Token as ProofsToken; -use Utopia\Auth\Proofs\Code as ProofsCode; use Utopia\Auth\Store; use Utopia\Config\Config; use Utopia\Database\Database; @@ -3234,7 +3234,7 @@ App::post('/v1/account/recovery') ->inject('locale') ->inject('queueForMails') ->inject('queueForEvents') - ->inject('proofForToken') + ->inject('proofForToken') ->action(function (string $email, string $url, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Mail $queueForMails, Event $queueForEvents, ProofsToken $proofForToken) { if (empty(System::getEnv('_APP_SMTP_HOST'))) { diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 7421e1526a..bc02a9dd85 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1,7 +1,6 @@ Date: Tue, 18 Mar 2025 10:24:42 +0100 Subject: [PATCH 020/245] Fixed syntax error --- app/init/resources.php | 4 +++- src/Appwrite/Auth/Auth.php | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app/init/resources.php b/app/init/resources.php index 41c099ce05..41e70b4e34 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -870,7 +870,9 @@ App::setResource('proofForToken', function (): Token { }); App::setResource('proofForTokenCode', function (): Token { - return new Token()->setLength(6); + $token = new Token(); + $token->setLength(6); + return $token; }); App::setResource('proofForCode', function (): Code { diff --git a/src/Appwrite/Auth/Auth.php b/src/Appwrite/Auth/Auth.php index a6c6e87d4e..a838a9ce75 100644 --- a/src/Appwrite/Auth/Auth.php +++ b/src/Appwrite/Auth/Auth.php @@ -12,11 +12,18 @@ class Auth { /** * @var string + * + * @deprecated We plan to deprecate this class in the future. Use Utopia Auth when possible. */ public static $cookieNamePreview = 'a_jwt_console'; /** * Token type to session provider mapping. + * + * @deprecated We plan to deprecate this class in the future. Use Utopia Auth when possible. + * @param int $type + * + * @return string */ public static function getSessionProviderByTokenType(int $type): string { @@ -41,6 +48,7 @@ class Auth * * One-way encryption * + * @deprecated We plan to deprecate this class in the future. Use Utopia Auth when possible. * @param $string * * @return string @@ -55,6 +63,7 @@ class Auth * * Generate random password string * + * @deprecated We plan to deprecate this class in the future. Use Utopia Auth when possible. * @param int $length Length of returned token * * @return string @@ -74,6 +83,7 @@ class Auth /** * Verify token and check that its not expired. * + * @deprecated We plan to deprecate this class in the future. Use Utopia Auth when possible. * @param array $tokens * @param int $type Type of token to verify, if null will verify any type * @param string $secret @@ -101,6 +111,7 @@ class Auth /** * Verify session and check that its not expired. * + * @deprecated We plan to deprecate this class in the future. Use Utopia Auth when possible. * @param array $sessions * @param string $secret * @@ -125,6 +136,7 @@ class Auth /** * Is Privileged User? * + * @deprecated We plan to deprecate this class in the future. Use Utopia Auth when possible. * @param array $roles * * @return bool @@ -145,6 +157,7 @@ class Auth /** * Is App User? * + * @deprecated We plan to deprecate this class in the future. Use Utopia Auth when possible. * @param array $roles * * @return bool @@ -161,6 +174,7 @@ class Auth /** * Returns all roles for a user. * + * @deprecated We plan to deprecate this class in the future. Use Utopia Auth when possible. * @param Document $user * @return array */ From afb40218d73ccc53e922fd64ab25dd029b699053 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 18 Mar 2025 16:12:13 +0100 Subject: [PATCH 021/245] Fixed tests --- app/controllers/api/account.php | 92 ++++++++++--------- app/controllers/api/teams.php | 2 +- app/controllers/general.php | 2 +- app/init/resources.php | 36 +++++--- app/realtime.php | 4 +- src/Appwrite/Auth/Auth.php | 10 +- .../Functions/Http/Executions/Create.php | 7 +- .../Account/AccountConsoleClientTest.php | 2 +- .../Account/AccountCustomClientTest.php | 2 +- .../Account/AccountCustomServerTest.php | 2 +- tests/unit/Auth/AuthTest.php | 6 -- 11 files changed, 91 insertions(+), 74 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 0a157c37fe..703ba764ec 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -170,7 +170,7 @@ $createSession = function (string $userId, string $secret, Request $request, Res throw new Exception(Exception::USER_INVALID_TOKEN); } - $verifiedToken = Auth::tokenVerify($userFromRequest->getAttribute('tokens', []), null, $secret); + $verifiedToken = Auth::tokenVerify($userFromRequest->getAttribute('tokens', []), null, $secret, $proofForToken); if (!$verifiedToken) { throw new Exception(Exception::USER_INVALID_TOKEN); @@ -539,14 +539,15 @@ App::get('/v1/account/sessions') ->inject('user') ->inject('locale') ->inject('store') - ->action(function (Response $response, Document $user, Locale $locale, Store $store) { + ->inject('proofForToken') + ->action(function (Response $response, Document $user, Locale $locale, Store $store, ProofsToken $proofForToken) { $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); $isAppUser = Auth::isAppUser($roles); $sessions = $user->getAttribute('sessions', []); - $current = Auth::sessionVerify($sessions, $store->getProperty('secret', '')); + $current = Auth::sessionVerify($sessions, $store->getProperty('secret', ''), $proofForToken); foreach ($sessions as $key => $session) {/** @var Document $session */ $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); @@ -593,7 +594,8 @@ App::delete('/v1/account/sessions') ->inject('queueForEvents') ->inject('queueForDeletes') ->inject('store') - ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $queueForEvents, Delete $queueForDeletes, Store $store) { + ->inject('proofForToken') + ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $queueForEvents, Delete $queueForDeletes, Store $store, ProofsToken $proofForToken) { $protocol = $request->getProtocol(); $sessions = $user->getAttribute('sessions', []); @@ -609,7 +611,7 @@ App::delete('/v1/account/sessions') ->setAttribute('current', false) ->setAttribute('countryName', $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown'))); - if ($session->getAttribute('secret') == Auth::hash($store->getProperty('secret', ''))) { + if ($proofForToken->verify($session->getAttribute('secret'), $store->getProperty('secret', ''))) { $session->setAttribute('current', true); // If current session delete the cookies too @@ -659,7 +661,8 @@ App::get('/v1/account/sessions/:sessionId') ->inject('user') ->inject('locale') ->inject('store') - ->action(function (?string $sessionId, Response $response, Document $user, Locale $locale, Store $store) { + ->inject('proofForToken') + ->action(function (?string $sessionId, Response $response, Document $user, Locale $locale, Store $store, ProofsToken $proofForToken) { $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); @@ -667,7 +670,7 @@ App::get('/v1/account/sessions/:sessionId') $sessions = $user->getAttribute('sessions', []); $sessionId = ($sessionId === 'current') - ? Auth::sessionVerify($user->getAttribute('sessions'), $store->getProperty('secret', '')) + ? Auth::sessionVerify($user->getAttribute('sessions'), $store->getProperty('secret', ''), $proofForToken) : $sessionId; foreach ($sessions as $session) {/** @var Document $session */ @@ -675,7 +678,7 @@ App::get('/v1/account/sessions/:sessionId') $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session - ->setAttribute('current', ($session->getAttribute('secret') == Auth::hash($store->getProperty('secret', '')))) + ->setAttribute('current', ($proofForToken->verify($session->getAttribute('secret'), $store->getProperty('secret', '')))) ->setAttribute('countryName', $countryName) ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $session->getAttribute('secret', '') : '') ; @@ -718,11 +721,12 @@ App::delete('/v1/account/sessions/:sessionId') ->inject('queueForEvents') ->inject('queueForDeletes') ->inject('store') - ->action(function (?string $sessionId, ?\DateTime $requestTimestamp, Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $queueForEvents, Delete $queueForDeletes, Store $store) { + ->inject('proofForToken') + ->action(function (?string $sessionId, ?\DateTime $requestTimestamp, Request $request, Response $response, Document $user, Database $dbForProject, Locale $locale, Event $queueForEvents, Delete $queueForDeletes, Store $store, ProofsToken $proofForToken) { $protocol = $request->getProtocol(); $sessionId = ($sessionId === 'current') - ? Auth::sessionVerify($user->getAttribute('sessions'), $store->getProperty('secret', '')) + ? Auth::sessionVerify($user->getAttribute('sessions'), $store->getProperty('secret', ''), $proofForToken) : $sessionId; $sessions = $user->getAttribute('sessions', []); @@ -741,7 +745,7 @@ App::delete('/v1/account/sessions/:sessionId') $session->setAttribute('current', false); - if ($session->getAttribute('secret') == Auth::hash($store->getProperty('secret', ''))) { // If current session delete the cookies too + if ($proofForToken->verify($session->getAttribute('secret'), $store->getProperty('secret', ''))) { // If current session delete the cookies too $session ->setAttribute('current', true) ->setAttribute('countryName', $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown'))); @@ -803,10 +807,11 @@ App::patch('/v1/account/sessions/:sessionId') ->inject('project') ->inject('queueForEvents') ->inject('store') - ->action(function (?string $sessionId, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents, Store $store) { + ->inject('proofForToken') + ->action(function (?string $sessionId, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents, Store $store, ProofsToken $proofForToken) { $sessionId = ($sessionId === 'current') - ? Auth::sessionVerify($user->getAttribute('sessions'), $store->getProperty('secret', '')) + ? Auth::sessionVerify($user->getAttribute('sessions'), $store->getProperty('secret', ''), $proofForToken) : $sessionId; $sessions = $user->getAttribute('sessions', []); @@ -934,7 +939,7 @@ App::post('/v1/account/sessions/email') 'userInternalId' => $user->getInternalId(), 'provider' => SESSION_PROVIDER_EMAIL, 'providerUid' => $email, - 'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak + 'secret' => $proofForToken->hash($secret), // One way hash encryption to protect DB leak 'userAgent' => $request->getUserAgent('UNKNOWN'), 'ip' => $request->getIP(), 'factors' => ['password'], @@ -1193,7 +1198,7 @@ App::post('/v1/account/sessions/token') ->inject('queueForEvents') ->inject('queueForMails') ->inject('store') - ->inject('proofForTokenCode') + ->inject('proofForToken') ->action($createSession); App::get('/v1/account/sessions/oauth2/:provider') @@ -1498,7 +1503,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') } $sessions = $user->getAttribute('sessions', []); - $current = Auth::sessionVerify($sessions, $store->getProperty('secret', '')); + $current = Auth::sessionVerify($sessions, $store->getProperty('secret', ''), $proofForToken); if ($current) { // Delete current session of new one. $currentDocument = $dbForProject->getDocument('sessions', $current); @@ -2164,8 +2169,8 @@ App::post('/v1/account/tokens/email') ->inject('queueForEvents') ->inject('queueForMails') ->inject('proofForPassword') - ->inject('proofForTokenCode') - ->action(function (string $userId, string $email, bool $phrase, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Locale $locale, Event $queueForEvents, Mail $queueForMails, ProofsPassword $proofForPassword, ProofsToken $proofForTokenCode) { + ->inject('proofForCode') + ->action(function (string $userId, string $email, bool $phrase, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Locale $locale, Event $queueForEvents, Mail $queueForMails, ProofsPassword $proofForPassword, ProofsCode $proofForCode) { if (empty(System::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP disabled'); } @@ -2230,7 +2235,7 @@ App::post('/v1/account/tokens/email') Authorization::skip(fn () => $dbForProject->createDocument('users', $user)); } - $tokenSecret = $proofForTokenCode->generate(); + $tokenSecret = $proofForCode->generate(); $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_OTP)); $token = new Document([ @@ -2238,7 +2243,7 @@ App::post('/v1/account/tokens/email') 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), 'type' => TOKEN_TYPE_EMAIL, - 'secret' => Auth::hash($tokenSecret), // One way hash encryption to protect DB leak + 'secret' => $proofForCode->hash($tokenSecret), // One way hash encryption to protect DB leak 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), 'ip' => $request->getIP(), @@ -2402,7 +2407,7 @@ App::put('/v1/account/sessions/magic-url') ->inject('queueForEvents') ->inject('queueForMails') ->inject('store') - ->inject('proofForTokenCode') + ->inject('proofForToken') ->action($createSession); App::put('/v1/account/sessions/phone') @@ -2441,7 +2446,7 @@ App::put('/v1/account/sessions/phone') ->inject('queueForEvents') ->inject('queueForMails') ->inject('store') - ->inject('proofForTokenCode') + ->inject('proofForToken') ->action($createSession); App::post('/v1/account/tokens/phone') @@ -2689,19 +2694,15 @@ App::post('/v1/account/jwts') ->inject('response') ->inject('user') ->inject('store') - ->action(function (Response $response, Document $user, Store $store) { + ->inject('proofForToken') + ->action(function (Response $response, Document $user, Store $store, ProofsToken $proofForToken) { $sessions = $user->getAttribute('sessions', []); - $current = new Document(); - foreach ($sessions as $session) { /** @var Utopia\Database\Document $session */ - if ($session->getAttribute('secret') == Auth::hash($store->getProperty('secret', ''))) { // If current session delete the cookies too - $current = $session; - } - } + $sessionId = Auth::sessionVerify($sessions, $store->getProperty('secret', ''), $proofForToken); - if ($current->isEmpty()) { + if (!$sessionId) { throw new Exception(Exception::USER_SESSION_NOT_FOUND); } @@ -2711,7 +2712,7 @@ App::post('/v1/account/jwts') ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic(new Document(['jwt' => $jwt->encode([ 'userId' => $user->getId(), - 'sessionId' => $current->getId(), + 'sessionId' => $sessionId, ])]), Response::MODEL_JWT); }); @@ -3421,7 +3422,8 @@ App::put('/v1/account/recovery') ->inject('queueForEvents') ->inject('hooks') ->inject('proofForPassword') - ->action(function (string $userId, string $secret, string $password, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents, Hooks $hooks, ProofsPassword $proofForPassword) { + ->inject('proofForToken') + ->action(function (string $userId, string $secret, string $password, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents, Hooks $hooks, ProofsPassword $proofForPassword, ProofsToken $proofForToken) { $profile = $dbForProject->getDocument('users', $userId); if ($profile->isEmpty()) { @@ -3429,7 +3431,7 @@ App::put('/v1/account/recovery') } $tokens = $profile->getAttribute('tokens', []); - $verifiedToken = Auth::tokenVerify($tokens, TOKEN_TYPE_RECOVERY, $secret); + $verifiedToken = Auth::tokenVerify($tokens, TOKEN_TYPE_RECOVERY, $secret, $proofForToken); if (!$verifiedToken) { throw new Exception(Exception::USER_INVALID_TOKEN); @@ -3680,7 +3682,8 @@ App::put('/v1/account/verification') ->inject('user') ->inject('dbForProject') ->inject('queueForEvents') - ->action(function (string $userId, string $secret, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { + ->inject('proofForToken') + ->action(function (string $userId, string $secret, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, ProofsToken $proofForToken) { $profile = Authorization::skip(fn () => $dbForProject->getDocument('users', $userId)); @@ -3689,7 +3692,7 @@ App::put('/v1/account/verification') } $tokens = $profile->getAttribute('tokens', []); - $verifiedToken = Auth::tokenVerify($tokens, TOKEN_TYPE_VERIFICATION, $secret); + $verifiedToken = Auth::tokenVerify($tokens, TOKEN_TYPE_VERIFICATION, $secret, $proofForToken); if (!$verifiedToken) { throw new Exception(Exception::USER_INVALID_TOKEN); @@ -3751,8 +3754,8 @@ App::post('/v1/account/verification/phone') ->inject('timelimit') ->inject('queueForStatsUsage') ->inject('plan') - ->inject('proofForToken') - ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, ProofsToken $proofForToken) { + ->inject('proofForCode') + ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, ProofsCode $proofForCode) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -3781,7 +3784,7 @@ App::post('/v1/account/verification/phone') } } - $secret ??= $proofForToken->generate(); + $secret ??= $proofForCode->generate(); $expire = DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM); $verification = new Document([ @@ -3789,7 +3792,7 @@ App::post('/v1/account/verification/phone') 'userId' => $user->getId(), 'userInternalId' => $user->getInternalId(), 'type' => TOKEN_TYPE_PHONE, - 'secret' => $proofForToken->hash($secret), + 'secret' => $proofForCode->hash($secret), 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), 'ip' => $request->getIP(), @@ -3904,7 +3907,8 @@ App::put('/v1/account/verification/phone') ->inject('user') ->inject('dbForProject') ->inject('queueForEvents') - ->action(function (string $userId, string $secret, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) { + ->inject('proofForCode') + ->action(function (string $userId, string $secret, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, ProofsCode $proofForCode) { $profile = Authorization::skip(fn () => $dbForProject->getDocument('users', $userId)); @@ -3912,7 +3916,7 @@ App::put('/v1/account/verification/phone') throw new Exception(Exception::USER_NOT_FOUND); } - $verifiedToken = Auth::tokenVerify($user->getAttribute('tokens', []), TOKEN_TYPE_PHONE, $secret); + $verifiedToken = Auth::tokenVerify($user->getAttribute('tokens', []), TOKEN_TYPE_PHONE, $secret, $proofForCode); if (!$verifiedToken) { throw new Exception(Exception::USER_INVALID_TOKEN); @@ -4389,6 +4393,7 @@ App::post('/v1/account/mfa/challenge') ->action(function (string $factor, Response $response, Database $dbForProject, Document $user, Locale $locale, Document $project, Request $request, Event $queueForEvents, Messaging $queueForMessaging, Mail $queueForMails, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, ProofsToken $proofForToken, ProofsCode $proofForCode) { $expire = DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM); + $code = $proofForCode->generate(); $challenge = new Document([ 'userId' => $user->getId(), @@ -4692,7 +4697,8 @@ App::post('/v1/account/targets/push') ->inject('response') ->inject('dbForProject') ->inject('store') - ->action(function (string $targetId, string $identifier, string $providerId, Event $queueForEvents, Document $user, Request $request, Response $response, Database $dbForProject, Store $store) { + ->inject('proofForToken') + ->action(function (string $targetId, string $identifier, string $providerId, Event $queueForEvents, Document $user, Request $request, Response $response, Database $dbForProject, Store $store, ProofsToken $proofForToken) { $targetId = $targetId == 'unique()' ? ID::unique() : $targetId; $provider = Authorization::skip(fn () => $dbForProject->getDocument('providers', $providerId)); @@ -4708,7 +4714,7 @@ App::post('/v1/account/targets/push') $device = $detector->getDevice(); - $sessionId = Auth::sessionVerify($user->getAttribute('sessions', []), $store->getProperty('secret', '')); + $sessionId = Auth::sessionVerify($user->getAttribute('sessions', []), $store->getProperty('secret', ''), $proofForToken); $session = $dbForProject->getDocument('sessions', $sessionId); try { diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 72838fe001..0e80cbb398 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -624,7 +624,7 @@ App::post('/v1/teams/:teamId/memberships') Authorization::skip(fn () => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1)); } elseif ($membership->getAttribute('confirm') === false) { - $membership->setAttribute('secret', Auth::hash($secret)); + $membership->setAttribute('secret', $proofForToken->hash($secret)); $membership->setAttribute('invited', DateTime::now()); if ($isPrivilegedUser || $isAppUser) { diff --git a/app/controllers/general.php b/app/controllers/general.php index 77da6189c7..8926c5e6ce 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -977,7 +977,7 @@ App::error() $trace = $error->getTrace(); if (php_sapi_name() === 'cli') { - $logLevel = $code >= 500 ? 'error' : 'warning'; + $logLevel = $code >= 500 || $code == 0 ? 'error' : 'warning'; $logPrefix = $code >= 500 || $code == 0 ? '[Error]' : '[Warning]'; Console::$logLevel($logPrefix . ' Timestamp: ' . date('c', time())); diff --git a/app/init/resources.php b/app/init/resources.php index 41e70b4e34..84d2a65d3a 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -23,6 +23,8 @@ use Appwrite\Network\Validator\Origin; use Appwrite\Utopia\Request; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; +use Utopia\Auth\Hashes\Argon2; +use Utopia\Auth\Hashes\Sha; use Utopia\Auth\Proofs\Code; use Utopia\Auth\Proofs\Password; use Utopia\Auth\Proofs\Token; @@ -168,7 +170,7 @@ App::setResource('clients', function ($request, $console, $project) { return \array_unique($clients); }, ['request', 'console', 'project']); -App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForPlatform, Store $store) { +App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForPlatform, Store $store, Token $proofForToken) { /** @var Appwrite\Utopia\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ @@ -250,7 +252,7 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons if ( $user->isEmpty() // Check a document has been found in the DB - || !Auth::sessionVerify($user->getAttribute('sessions', []), $store->getProperty('secret', '')) + || !Auth::sessionVerify($user->getAttribute('sessions', []), $store->getProperty('secret', ''), $proofForToken) ) { // Validate user has valid login token $user = new Document([]); } @@ -291,7 +293,7 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons $dbForPlatform->setMetadata('user', $user->getId()); return $user; -}, ['mode', 'project', 'console', 'request', 'response', 'dbForProject', 'dbForPlatform', 'store']); +}, ['mode', 'project', 'console', 'request', 'response', 'dbForProject', 'dbForPlatform', 'store', 'proofForToken']); App::setResource('project', function ($dbForPlatform, $request, $console) { /** @var Appwrite\Utopia\Request $request */ @@ -309,13 +311,13 @@ App::setResource('project', function ($dbForPlatform, $request, $console) { return $project; }, ['dbForPlatform', 'request', 'console']); -App::setResource('session', function (Document $user, Store $store) { +App::setResource('session', function (Document $user, Store $store, Token $proofForToken) { if ($user->isEmpty()) { return; } $sessions = $user->getAttribute('sessions', []); - $sessionId = Auth::sessionVerify($user->getAttribute('sessions'), $store->getProperty('secret', '')); + $sessionId = Auth::sessionVerify($user->getAttribute('sessions'), $store->getProperty('secret', ''), $proofForToken); if (!$sessionId) { return; @@ -328,7 +330,7 @@ App::setResource('session', function (Document $user, Store $store) { } return; -}, ['user', 'store']); +}, ['user', 'store', 'proofForToken']); App::setResource('console', function () { return new Document(Config::getParam('console')); @@ -862,19 +864,27 @@ App::setResource('store', function (): Store { }); App::setResource('proofForPassword', function (): Password { - return new Password(); + $hash = new Argon2(); + $hash + ->setMemoryCost(2048) + ->setTimeCost(4) + ->setThreads(3); + + $password = new Password(); + $password + ->setHash($hash); + + return $password; }); App::setResource('proofForToken', function (): Token { - return new Token(); -}); - -App::setResource('proofForTokenCode', function (): Token { $token = new Token(); - $token->setLength(6); + $token->setHash(new Sha()); return $token; }); App::setResource('proofForCode', function (): Code { - return new Code(); + $code = new Code(); + $code->setHash(new Sha()); + return $code; }); diff --git a/app/realtime.php b/app/realtime.php index d65a2559b5..6d10fd7674 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -15,6 +15,7 @@ use Swoole\Timer; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; +use Utopia\Auth\Proofs\Token; use Utopia\Auth\Store; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; @@ -654,10 +655,11 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $store->decode($message['data']['session']); $user = $database->getDocument('users', $store->getProperty('id', '')); + $proofForToken = new Token(); if ( empty($user->getId()) // Check a document has been found in the DB - || !Auth::sessionVerify($user->getAttribute('sessions', []), $store->getProperty('secret', '')) // Validate user has valid login token + || !Auth::sessionVerify($user->getAttribute('sessions', []), $store->getProperty('secret', ''), $proofForToken) // Validate user has valid login token ) { // cookie not valid throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Session is not valid.'); diff --git a/src/Appwrite/Auth/Auth.php b/src/Appwrite/Auth/Auth.php index a838a9ce75..86d1e197bf 100644 --- a/src/Appwrite/Auth/Auth.php +++ b/src/Appwrite/Auth/Auth.php @@ -2,6 +2,8 @@ namespace Appwrite\Auth; +use Utopia\Auth\Proof; +use Utopia\Auth\Proofs\Token; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Helpers\Role; @@ -90,7 +92,7 @@ class Auth * * @return false|Document */ - public static function tokenVerify(array $tokens, int $type = null, string $secret): false|Document + public static function tokenVerify(array $tokens, int $type = null, string $secret, Proof $proofForToken): false|Document { foreach ($tokens as $token) { if ( @@ -98,7 +100,7 @@ class Auth $token->isSet('expire') && $token->isSet('type') && ($type === null || $token->getAttribute('type') === $type) && - $token->getAttribute('secret') === self::hash($secret) && + $proofForToken->verify($secret, $token->getAttribute('secret')) && DateTime::formatTz($token->getAttribute('expire')) >= DateTime::formatTz(DateTime::now()) ) { return $token; @@ -117,13 +119,13 @@ class Auth * * @return bool|string */ - public static function sessionVerify(array $sessions, string $secret) + public static function sessionVerify(array $sessions, string $secret, Token $proofForToken) { foreach ($sessions as $session) { if ( $session->isSet('secret') && $session->isSet('provider') && - $session->getAttribute('secret') === self::hash($secret) && + $proofForToken->verify($secret, $session->getAttribute('secret')) && DateTime::formatTz(DateTime::format(new \DateTime($session->getAttribute('expire')))) >= DateTime::formatTz(DateTime::now()) ) { return $session->getId(); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index 3a7030742e..af24e7c9f7 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -19,6 +19,7 @@ use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; use Executor\Executor; use MaxMind\Db\Reader; +use Utopia\Auth\Proofs\Token; use Utopia\Auth\Store; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -95,6 +96,7 @@ class Create extends Base ->inject('queueForFunctions') ->inject('geodb') ->inject('store') + ->inject('proofForToken') ->callback([$this, 'action']); } @@ -116,7 +118,8 @@ class Create extends Base StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, - Store $store + Store $store, + Token $proofForToken ) { $async = \strval($async) === 'true' || \strval($async) === '1'; @@ -193,7 +196,7 @@ class Create extends Base foreach ($sessions as $session) { /** @var Utopia\Database\Document $session */ - if ($session->getAttribute('secret') == Auth::hash($store->getProperty('secret', ''))) { // If current session delete the cookies too + if ($proofForToken->verify($store->getProperty('secret', ''), $session->getAttribute('secret'))) { // If current session delete the cookies too $current = $session; } } diff --git a/tests/e2e/Services/Account/AccountConsoleClientTest.php b/tests/e2e/Services/Account/AccountConsoleClientTest.php index 1df9ef6c18..51de5731bd 100644 --- a/tests/e2e/Services/Account/AccountConsoleClientTest.php +++ b/tests/e2e/Services/Account/AccountConsoleClientTest.php @@ -45,7 +45,6 @@ class AccountConsoleClientTest extends Scope $this->assertEquals($response['headers']['status-code'], 201); $session = $response['cookies']['a_session_' . $this->getProject()['$id']]; - // create team $team = $this->client->call(Client::METHOD_POST, '/teams', [ 'origin' => 'http://localhost', @@ -56,6 +55,7 @@ class AccountConsoleClientTest extends Scope 'teamId' => 'unique()', 'name' => 'myteam' ]); + $this->assertEquals($team['headers']['status-code'], 201); $teamId = $team['body']['$id']; diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index daa5bcbff8..683988f10e 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -2535,7 +2535,7 @@ class AccountCustomClientTest extends Scope $this->assertEquals($this->getProject()['name'] . ' Login', $lastEmail['subject']); $this->assertStringNotContainsStringIgnoringCase('security phrase', $lastEmail['text']); - $token = substr($lastEmail['text'], strpos($lastEmail['text'], '&secret=', 0) + 8, 64); + $token = substr($lastEmail['text'], strpos($lastEmail['text'], '&secret=', 0) + 8, 256); $expireTime = strpos($lastEmail['text'], 'expire=' . urlencode($response['body']['expire']), 0); diff --git a/tests/e2e/Services/Account/AccountCustomServerTest.php b/tests/e2e/Services/Account/AccountCustomServerTest.php index eb72a99913..e0a52c4007 100644 --- a/tests/e2e/Services/Account/AccountCustomServerTest.php +++ b/tests/e2e/Services/Account/AccountCustomServerTest.php @@ -218,7 +218,7 @@ class AccountCustomServerTest extends Scope $this->assertEquals($email, $lastEmail['to'][0]['address']); $this->assertEquals($this->getProject()['name'] . ' Login', $lastEmail['subject']); - $token = substr($lastEmail['text'], strpos($lastEmail['text'], '&secret=', 0) + 8, 64); + $token = substr($lastEmail['text'], strpos($lastEmail['text'], '&secret=', 0) + 8, 256); $expireTime = strpos($lastEmail['text'], 'expire=' . urlencode($response['body']['expire']), 0); diff --git a/tests/unit/Auth/AuthTest.php b/tests/unit/Auth/AuthTest.php index e12b5c36a6..84186ea222 100644 --- a/tests/unit/Auth/AuthTest.php +++ b/tests/unit/Auth/AuthTest.php @@ -22,12 +22,6 @@ class AuthTest extends TestCase Authorization::setRole(Role::any()->toString()); } - public function testHash(): void - { - $secret = 'secret'; - $this->assertEquals(Auth::hash($secret), '2bb80d537b1da3e38bd30361aa855686bde0eacd7162fef6a25fe97bf527a25b'); - } - public function testSessionVerify(): void { $expireTime1 = 60 * 60 * 24; From 4d5961c3ab9ed29aed2f30e4931fb4f13d9275ec Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 18 Mar 2025 16:20:37 +0100 Subject: [PATCH 022/245] Fixed test --- app/controllers/api/users.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index bc02a9dd85..d4e8c9cb48 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -2113,7 +2113,9 @@ App::post('/v1/users/:userId/tokens') throw new Exception(Exception::USER_NOT_FOUND); } - $secret = $proofForToken->generate(); + $secret = $proofForToken + ->setLength($length) + ->generate(); $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $expire)); $token = new Document([ From 24300cd3bded892cd09ea1ad7c7c441486c9c72b Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 18 Mar 2025 17:22:58 +0100 Subject: [PATCH 023/245] tests --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c3585dbb68..4af1370403 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -> [Get started with Appwrite](https://apwr.dev/appcloud) +> [Get started with Appwrite](https://apwr.dev/appcloud)

From 55e560bb413b40f10c7a36c77d752b72771ad395 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 18 Mar 2025 19:08:58 +0100 Subject: [PATCH 024/245] Fixed unit tests --- tests/unit/Auth/AuthTest.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/unit/Auth/AuthTest.php b/tests/unit/Auth/AuthTest.php index 84186ea222..22a84b4c3a 100644 --- a/tests/unit/Auth/AuthTest.php +++ b/tests/unit/Auth/AuthTest.php @@ -10,7 +10,7 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Role; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Roles; - +use Utopia\Auth\Proofs\Token; class AuthTest extends TestCase { /** @@ -64,10 +64,12 @@ class AuthTest extends TestCase ]), ]; - $this->assertEquals(Auth::sessionVerify($tokens1, $secret), 'token1'); - $this->assertEquals(Auth::sessionVerify($tokens1, 'false-secret'), false); - $this->assertEquals(Auth::sessionVerify($tokens2, $secret), false); - $this->assertEquals(Auth::sessionVerify($tokens2, 'false-secret'), false); + $proofForToken = new Token(); + + $this->assertEquals(Auth::sessionVerify($tokens1, $secret, $proofForToken), 'token1'); + $this->assertEquals(Auth::sessionVerify($tokens1, 'false-secret', $proofForToken), false); + $this->assertEquals(Auth::sessionVerify($tokens2, $secret, $proofForToken), false); + $this->assertEquals(Auth::sessionVerify($tokens2, 'false-secret', $proofForToken), false); } public function testTokenVerify(): void From 8cb85cafbd0b4d739e59faf5f56988fc64b2b760 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 18 Mar 2025 20:24:38 +0100 Subject: [PATCH 025/245] Fixes realtime tests --- app/realtime.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/realtime.php b/app/realtime.php index 6d10fd7674..95dabaafba 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -15,6 +15,7 @@ use Swoole\Timer; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; +use Utopia\Auth\Hashes\Sha; use Utopia\Auth\Proofs\Token; use Utopia\Auth\Store; use Utopia\Cache\Adapter\Sharding; @@ -652,10 +653,19 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re } $store = new Store(); + $store->decode($message['data']['session']); $user = $database->getDocument('users', $store->getProperty('id', '')); + + /** + * TODO: + * Moving forward, we should try to use our dependency injection container + * to inject the proof for token. + * This way we will have one source of truth for the proof for token. + */ $proofForToken = new Token(); + $proofForToken->setHash(new Sha()); if ( empty($user->getId()) // Check a document has been found in the DB From fed6579491c401f5c6f230679b374fc92a49de63 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 18 Mar 2025 21:52:41 +0100 Subject: [PATCH 026/245] Update teams tests --- composer.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer.lock b/composer.lock index 7e82876806..3cd44d69c2 100644 --- a/composer.lock +++ b/composer.lock @@ -3512,12 +3512,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/auth.git", - "reference": "ed49b9e481030ba5e589140b41a9f4be1486310f" + "reference": "dfdf614644237700e41935b51da7e39f6848a6e7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/auth/zipball/ed49b9e481030ba5e589140b41a9f4be1486310f", - "reference": "ed49b9e481030ba5e589140b41a9f4be1486310f", + "url": "https://api.github.com/repos/utopia-php/auth/zipball/dfdf614644237700e41935b51da7e39f6848a6e7", + "reference": "dfdf614644237700e41935b51da7e39f6848a6e7", "shasum": "" }, "require": { @@ -3559,7 +3559,7 @@ "issues": "https://github.com/utopia-php/auth/issues", "source": "https://github.com/utopia-php/auth/tree/dev" }, - "time": "2025-03-17T19:57:57+00:00" + "time": "2025-03-18T19:34:43+00:00" }, { "name": "utopia-php/cache", From b2b20c48b36f90168aafafdb8814ff810e16b71f Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 18 Mar 2025 22:12:03 +0100 Subject: [PATCH 027/245] Fixed tests --- app/controllers/api/account.php | 2 +- app/realtime.php | 4 ++-- src/Appwrite/Migration/Migration.php | 1 + src/Appwrite/Migration/Version/V23.php | 29 ++++++++++++++++++++++++++ tests/unit/Auth/AuthTest.php | 25 +++++++++++----------- tests/unit/Migration/MigrationTest.php | 2 +- 6 files changed, 47 insertions(+), 16 deletions(-) create mode 100644 src/Appwrite/Migration/Version/V23.php diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 703ba764ec..189982e45a 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -4393,7 +4393,7 @@ App::post('/v1/account/mfa/challenge') ->action(function (string $factor, Response $response, Database $dbForProject, Document $user, Locale $locale, Document $project, Request $request, Event $queueForEvents, Messaging $queueForMessaging, Mail $queueForMails, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, ProofsToken $proofForToken, ProofsCode $proofForCode) { $expire = DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM); - + $code = $proofForCode->generate(); $challenge = new Document([ 'userId' => $user->getId(), diff --git a/app/realtime.php b/app/realtime.php index 95dabaafba..d65a7cdb69 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -653,11 +653,11 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re } $store = new Store(); - + $store->decode($message['data']['session']); $user = $database->getDocument('users', $store->getProperty('id', '')); - + /** * TODO: * Moving forward, we should try to use our dependency injection container diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 56016f1057..17e93f43f5 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -93,6 +93,7 @@ abstract class Migration '1.6.0' => 'V21', '1.6.1' => 'V21', '1.6.2' => 'V22', + '1.7.0' => 'V23', ]; /** diff --git a/src/Appwrite/Migration/Version/V23.php b/src/Appwrite/Migration/Version/V23.php new file mode 100644 index 0000000000..f6f830436d --- /dev/null +++ b/src/Appwrite/Migration/Version/V23.php @@ -0,0 +1,29 @@ +hash($secret); $tokens1 = [ new Document([ '$id' => ID::custom('token1'), @@ -64,8 +66,6 @@ class AuthTest extends TestCase ]), ]; - $proofForToken = new Token(); - $this->assertEquals(Auth::sessionVerify($tokens1, $secret, $proofForToken), 'token1'); $this->assertEquals(Auth::sessionVerify($tokens1, 'false-secret', $proofForToken), false); $this->assertEquals(Auth::sessionVerify($tokens2, $secret, $proofForToken), false); @@ -74,8 +74,9 @@ class AuthTest extends TestCase public function testTokenVerify(): void { + $proofForToken = new Token(); $secret = 'secret1'; - $hash = Auth::hash($secret); + $hash = $proofForToken->hash($secret); $tokens1 = [ new Document([ '$id' => ID::custom('token1'), @@ -121,13 +122,13 @@ class AuthTest extends TestCase ]), ]; - $this->assertEquals(Auth::tokenVerify($tokens1, TOKEN_TYPE_RECOVERY, $secret), $tokens1[0]); - $this->assertEquals(Auth::tokenVerify($tokens1, null, $secret), $tokens1[0]); - $this->assertEquals(Auth::tokenVerify($tokens1, TOKEN_TYPE_RECOVERY, 'false-secret'), false); - $this->assertEquals(Auth::tokenVerify($tokens2, TOKEN_TYPE_RECOVERY, $secret), false); - $this->assertEquals(Auth::tokenVerify($tokens2, TOKEN_TYPE_RECOVERY, 'false-secret'), false); - $this->assertEquals(Auth::tokenVerify($tokens3, TOKEN_TYPE_RECOVERY, $secret), false); - $this->assertEquals(Auth::tokenVerify($tokens3, TOKEN_TYPE_RECOVERY, 'false-secret'), false); + $this->assertEquals(Auth::tokenVerify($tokens1, TOKEN_TYPE_RECOVERY, $secret, $proofForToken), $tokens1[0]); + $this->assertEquals(Auth::tokenVerify($tokens1, null, $secret, $proofForToken), $tokens1[0]); + $this->assertEquals(Auth::tokenVerify($tokens1, TOKEN_TYPE_RECOVERY, 'false-secret', $proofForToken), false); + $this->assertEquals(Auth::tokenVerify($tokens2, TOKEN_TYPE_RECOVERY, $secret, $proofForToken), false); + $this->assertEquals(Auth::tokenVerify($tokens2, TOKEN_TYPE_RECOVERY, 'false-secret', $proofForToken), false); + $this->assertEquals(Auth::tokenVerify($tokens3, TOKEN_TYPE_RECOVERY, $secret, $proofForToken), false); + $this->assertEquals(Auth::tokenVerify($tokens3, TOKEN_TYPE_RECOVERY, 'false-secret', $proofForToken), false); } public function testIsPrivilegedUser(): void diff --git a/tests/unit/Migration/MigrationTest.php b/tests/unit/Migration/MigrationTest.php index 536278d55b..8c619b76c2 100644 --- a/tests/unit/Migration/MigrationTest.php +++ b/tests/unit/Migration/MigrationTest.php @@ -7,7 +7,7 @@ use PHPUnit\Framework\TestCase; use ReflectionMethod; use Utopia\Database\Document; -abstract class MigrationTest extends TestCase +class MigrationTest extends TestCase { /** * @var Migration From 38cb95e94018d97de1d811d28f5bd029dac3bb63 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 18 Mar 2025 22:28:19 +0100 Subject: [PATCH 028/245] Fixed tests --- app/controllers/api/teams.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 0e80cbb398..489e801af3 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1153,7 +1153,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') throw new Exception(Exception::TEAM_MEMBERSHIP_MISMATCH); } - if ($proofForToken->verify($membership->getAttribute('secret'), $secret)) { + if (!$proofForToken->verify($secret, $membership->getAttribute('secret'))) { throw new Exception(Exception::TEAM_INVALID_SECRET); } From a7d7e39dfd96344c93c75bc51f1f84922babed88 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 18 Mar 2025 22:47:17 +0100 Subject: [PATCH 029/245] Fixed tests --- app/controllers/api/account.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 189982e45a..d029eff4f0 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -611,7 +611,7 @@ App::delete('/v1/account/sessions') ->setAttribute('current', false) ->setAttribute('countryName', $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown'))); - if ($proofForToken->verify($session->getAttribute('secret'), $store->getProperty('secret', ''))) { + if ($proofForToken->verify($store->getProperty('secret', ''), $session->getAttribute('secret'))) { $session->setAttribute('current', true); // If current session delete the cookies too @@ -678,7 +678,7 @@ App::get('/v1/account/sessions/:sessionId') $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session - ->setAttribute('current', ($proofForToken->verify($session->getAttribute('secret'), $store->getProperty('secret', '')))) + ->setAttribute('current', ($proofForToken->verify($store->getProperty('secret', ''), $session->getAttribute('secret')))) ->setAttribute('countryName', $countryName) ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $session->getAttribute('secret', '') : '') ; @@ -745,7 +745,7 @@ App::delete('/v1/account/sessions/:sessionId') $session->setAttribute('current', false); - if ($proofForToken->verify($session->getAttribute('secret'), $store->getProperty('secret', ''))) { // If current session delete the cookies too + if ($proofForToken->verify($store->getProperty('secret', ''), $session->getAttribute('secret'))) { // If current session delete the cookies too $session ->setAttribute('current', true) ->setAttribute('countryName', $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown'))); From 3d967e695f641a122459048b5b746e31aeeb2073 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Wed, 19 Mar 2025 07:57:28 +0100 Subject: [PATCH 030/245] Updated auth lib --- composer.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer.lock b/composer.lock index 3cd44d69c2..3751d5c3a1 100644 --- a/composer.lock +++ b/composer.lock @@ -3512,12 +3512,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/auth.git", - "reference": "dfdf614644237700e41935b51da7e39f6848a6e7" + "reference": "19fb580de44fac5928f9c0211fd0fdfd5022efdb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/auth/zipball/dfdf614644237700e41935b51da7e39f6848a6e7", - "reference": "dfdf614644237700e41935b51da7e39f6848a6e7", + "url": "https://api.github.com/repos/utopia-php/auth/zipball/19fb580de44fac5928f9c0211fd0fdfd5022efdb", + "reference": "19fb580de44fac5928f9c0211fd0fdfd5022efdb", "shasum": "" }, "require": { @@ -3559,7 +3559,7 @@ "issues": "https://github.com/utopia-php/auth/issues", "source": "https://github.com/utopia-php/auth/tree/dev" }, - "time": "2025-03-18T19:34:43+00:00" + "time": "2025-03-19T06:47:02+00:00" }, { "name": "utopia-php/cache", From 8c9123beaa7919f6466ecbc66e0ca26ced91c5fe Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Wed, 19 Mar 2025 13:54:32 +0100 Subject: [PATCH 031/245] Fixed tests --- app/controllers/api/account.php | 10 +++++++--- app/controllers/api/users.php | 4 +++- src/Appwrite/Auth/Validator/PasswordHistory.php | 14 +++++--------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index d029eff4f0..51a1c4f101 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -383,7 +383,7 @@ App::post('/v1/account') 'emailVerification' => false, 'status' => true, 'password' => $hash, - 'passwordHistory' => $passwordHistory > 0 ? [$password] : [], + 'passwordHistory' => $passwordHistory > 0 ? [$hash] : [], 'passwordUpdate' => DateTime::now(), 'hash' => $proof->getHash()->getName(), 'hashOptions' => $proof->getHash()->getOptions(), @@ -2894,9 +2894,11 @@ App::patch('/v1/account/password') $newPassword = $proofForPassword->hash($password); $historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0; + $hash = ProofsPassword::createHash($user->getAttribute('hash'), $user->getAttribute('hashOptions')); $history = $user->getAttribute('passwordHistory', []); + if ($historyLimit > 0) { - $validator = new PasswordHistory($history, $user->getAttribute('hash'), $user->getAttribute('hashOptions')); + $validator = new PasswordHistory($history, $hash); if (!$validator->isValid($password)) { throw new Exception(Exception::USER_PASSWORD_RECENTLY_USED); } @@ -3441,10 +3443,12 @@ App::put('/v1/account/recovery') $newPassword = $proofForPassword->hash($password); + $hash = ProofsPassword::createHash($profile->getAttribute('hash'), $profile->getAttribute('hashOptions')); $historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0; $history = $profile->getAttribute('passwordHistory', []); + if ($historyLimit > 0) { - $validator = new PasswordHistory($history, $profile->getAttribute('hash'), $profile->getAttribute('hashOptions')); + $validator = new PasswordHistory($history, $hash); if (!$validator->isValid($password)) { throw new Exception(Exception::USER_PASSWORD_RECENTLY_USED); } diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index d4e8c9cb48..65a35d616a 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1319,10 +1319,12 @@ App::patch('/v1/users/:userId/password') $newPassword = $hasher->hash($password); + $hash = ProofsPassword::createHash($user->getAttribute('hash'), $user->getAttribute('hashOptions')); $historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0; $history = $user->getAttribute('passwordHistory', []); + if ($historyLimit > 0) { - $validator = new PasswordHistory($history, $user->getAttribute('hash'), $user->getAttribute('hashOptions')); + $validator = new PasswordHistory($history, $hash); if (!$validator->isValid($password)) { throw new Exception(Exception::USER_PASSWORD_RECENTLY_USED); } diff --git a/src/Appwrite/Auth/Validator/PasswordHistory.php b/src/Appwrite/Auth/Validator/PasswordHistory.php index 7677deafc0..9b40b6a794 100644 --- a/src/Appwrite/Auth/Validator/PasswordHistory.php +++ b/src/Appwrite/Auth/Validator/PasswordHistory.php @@ -2,7 +2,7 @@ namespace Appwrite\Auth\Validator; -use Utopia\Auth\Proofs\Password as ProofsPassword; +use Utopia\Auth\Hash; /** * Password. @@ -12,16 +12,14 @@ use Utopia\Auth\Proofs\Password as ProofsPassword; class PasswordHistory extends Password { protected array $history; - protected string $algo; - protected array $algoOptions; + protected Hash $hash; - public function __construct(array $history, string $algo, array $algoOptions = []) + public function __construct(array $history, Hash $hash) { parent::__construct(); $this->history = $history; - $this->algo = $algo; - $this->algoOptions = $algoOptions; + $this->hash = $hash; } /** @@ -45,10 +43,8 @@ class PasswordHistory extends Password */ public function isValid($value): bool { - $proofForPassword = ProofsPassword::createHash($this->algo, $this->algoOptions); - foreach ($this->history as $hash) { - if (!empty($hash) && $proofForPassword->verify($value, $hash)) { + if (!empty($hash) && $this->hash->verify($value, $hash)) { return false; } } From d6bd72cfd32635d37ff144d54ee079bbcda72ad5 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Wed, 19 Mar 2025 14:10:56 +0100 Subject: [PATCH 032/245] formatting --- app/controllers/api/account.php | 2 +- app/controllers/api/users.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 51a1c4f101..74ad74b3db 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -3446,7 +3446,7 @@ App::put('/v1/account/recovery') $hash = ProofsPassword::createHash($profile->getAttribute('hash'), $profile->getAttribute('hashOptions')); $historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0; $history = $profile->getAttribute('passwordHistory', []); - + if ($historyLimit > 0) { $validator = new PasswordHistory($history, $hash); if (!$validator->isValid($password)) { diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 65a35d616a..0ab7f1a9d7 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1322,7 +1322,7 @@ App::patch('/v1/users/:userId/password') $hash = ProofsPassword::createHash($user->getAttribute('hash'), $user->getAttribute('hashOptions')); $historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0; $history = $user->getAttribute('passwordHistory', []); - + if ($historyLimit > 0) { $validator = new PasswordHistory($history, $hash); if (!$validator->isValid($password)) { From 17c555ab6c7f289046878f3582cddbb8c549c0ab Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 28 Mar 2025 02:13:17 +0100 Subject: [PATCH 033/245] Expanded the identities collection --- app/config/collections/common.php | 21 ++++ app/terminal.php | 194 ++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 app/terminal.php diff --git a/app/config/collections/common.php b/app/config/collections/common.php index 00ef59968d..14c3a7ea29 100644 --- a/app/config/collections/common.php +++ b/app/config/collections/common.php @@ -1100,6 +1100,27 @@ return [ 'array' => false, 'filters' => ['json', 'encrypt'], ], + [ + '$id' => ID::custom('scopes'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => true, + 'filters' => [], + ], + [ + '$id' => ID::custom('expire'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'required' => true, + 'signed' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], ], 'indexes' => [ [ diff --git a/app/terminal.php b/app/terminal.php new file mode 100644 index 0000000000..1d2f2f0b24 --- /dev/null +++ b/app/terminal.php @@ -0,0 +1,194 @@ +column('userId', Table::TYPE_STRING, 64); +$runtimes->column('runtimeHost', Table::TYPE_STRING, 255); +$runtimes->column('runtimePort', Table::TYPE_INT); +$runtimes->create(); + +// Store WebSocket client connections +$clients = []; + +const MAX_PACKAGE_LENGTH = 64000; +const MAX_RUNTIME_CONNECTIONS = 4096; + +$adapter = new Adapter\Swoole(port: System::getEnv('PORT', 80)); +$adapter + ->setPackageMaxLength(MAX_PACKAGE_LENGTH) + ->setWorkerNumber($workerNumber); + +$server = new Server($adapter); + +$server->onStart(function () use ($workerNumber) { + Console::success('Terminal WebSocket Proxy started successfully'); + Console::info('Listening on port: ' . System::getEnv('PORT', 80)); + Console::info('Worker processes: ' . $workerNumber); + Console::info('Max package length: ' . (MAX_PACKAGE_LENGTH / 1000) . 'KB'); + Console::info('Max runtime connections: ' . MAX_RUNTIME_CONNECTIONS); +}); + +$server->onOpen(function (int $connection, $request) use ($server, $runtimes, &$clients) { + try { + Console::info("New connection: {$connection}"); + + // Extract JWT from request + $token = $request->header['authorization'] ?? ''; + if (empty($token)) { + throw new Exception('Missing authentication token', 401); + } + + // Verify JWT and extract user information + $jwt = str_replace('Bearer ', '', $token); + $key = System::getEnv('_APP_OPENSSL_KEY_V1', ''); + $jwt = new JWT($key, 'HS256', 900, 0); + + try { + $payload = $jwt->decode($token); + $userId = $payload['userId'] ?? ''; + $sessionId = $payload['sessionId'] ?? ''; + + if (empty($userId) || empty($sessionId)) { + throw new Exception('Invalid JWT payload', 401); + } + } catch (\Exception $e) { + throw new Exception('Invalid JWT token', 401); + } + + // Get runtime details for user (this could come from your database/cache) + $runtimeHost = "runtime-{$userId}.internal"; // Example hostname + $runtimePort = 9000; + + // Create WebSocket connection to runtime + go(function () use ($server, $connection, &$clients, $runtimeHost, $runtimePort, $userId) { + try { + // $wsClient = new Client("ws://{$runtimeHost}:{$runtimePort}/", [ + // 'timeout' => 0, // Disable timeout for long-running connections + // 'filter' => ['text', 'binary', 'close'] // Only process these frame types + // ]); + + + $wsClient = new Client( + "ws://appwrite-traefik/v1/realtime", + [ + "headers" => [], + "timeout" => 30, + ] + ); + + // Store client connection + $clients[$connection] = [ + 'client' => $wsClient, + 'userId' => $userId + ]; + + // Forward messages from runtime back to client + while (true) { + try { + $message = $wsClient->receive(); + if ($message === null) { + // Connection closed normally + break; + } + $server->send([$connection], $message); + + // Yield to allow other coroutines to run + Swoole\Coroutine::yield(); + + } catch (\WebSocket\ConnectionException $e) { + Console::error("Runtime connection error for user {$userId}: " . $e->getMessage()); + break; + } + } + + // Cleanup on disconnect + $wsClient->close(); + unset($clients[$connection]); + $server->close($connection, CLOSE_NORMAL); + } catch (\WebSocket\ConnectionException $e) { + Console::error("Failed to connect to runtime for user {$userId}: " . $e->getMessage()); + $server->close($connection, CLOSE_SERVER_ERROR); + return; + } + }); + + // Send successful connection message + $server->send([$connection], json_encode([ + 'type' => 'connected', + 'data' => [ + 'userId' => $userId, + 'timestamp' => time() + ] + ])); + + } catch (Throwable $th) { + Console::error('Connection error: ' . $th->getMessage()); + + $server->send([$connection], json_encode([ + 'type' => 'error', + 'data' => [ + 'code' => $th->getCode(), + 'message' => $th->getMessage() + ] + ])); + + $server->close($connection, CLOSE_POLICY_VIOLATION); + } +}); + +$server->onMessage(function (int $connection, string $message) use ($server, &$clients) { + try { + if (!isset($clients[$connection])) { + throw new Exception('Client not connected to runtime', 1008); + } + + $wsClient = $clients[$connection]['client']; + try { + // Forward message to runtime + $wsClient->send($message); + } catch (\WebSocket\ConnectionException $e) { + throw new Exception('Runtime connection lost: ' . $e->getMessage(), 1008); + } + + } catch (Throwable $th) { + $server->send([$connection], json_encode([ + 'type' => 'error', + 'data' => [ + 'code' => $th->getCode(), + 'message' => $th->getMessage() + ] + ])); + + if ($th->getCode() === 1008) { + $server->close($connection, CLOSE_POLICY_VIOLATION); + } + } +}); + +$server->onClose(function (int $connection) use (&$clients) { + if (isset($clients[$connection])) { + $userId = $clients[$connection]['userId']; + $clients[$connection]['client']->close(); + unset($clients[$connection]); + Console::info("Closed connection for user {$userId}"); + } +}); + +$server->start(); \ No newline at end of file From 57edb4a38554cd82de79c81e8605120be653c541 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Fri, 28 Mar 2025 08:17:47 +0100 Subject: [PATCH 034/245] Removed leftovers --- README.md | 2 +- app/terminal.php | 194 --------------- src/Appwrite/Auth/Hash/Argon2.php | 47 ---- src/Appwrite/Auth/Hash/Bcrypt.php | 46 ---- src/Appwrite/Auth/Hash/Md5.php | 44 ---- src/Appwrite/Auth/Hash/Phpass.php | 290 ---------------------- src/Appwrite/Auth/Hash/Scrypt.php | 51 ---- src/Appwrite/Auth/Hash/Scryptmodified.php | 80 ------ src/Appwrite/Auth/Hash/Sha.php | 50 ---- 9 files changed, 1 insertion(+), 803 deletions(-) delete mode 100644 app/terminal.php delete mode 100644 src/Appwrite/Auth/Hash/Argon2.php delete mode 100644 src/Appwrite/Auth/Hash/Bcrypt.php delete mode 100644 src/Appwrite/Auth/Hash/Md5.php delete mode 100644 src/Appwrite/Auth/Hash/Phpass.php delete mode 100644 src/Appwrite/Auth/Hash/Scrypt.php delete mode 100644 src/Appwrite/Auth/Hash/Scryptmodified.php delete mode 100644 src/Appwrite/Auth/Hash/Sha.php diff --git a/README.md b/README.md index 4af1370403..c3585dbb68 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -> [Get started with Appwrite](https://apwr.dev/appcloud) +> [Get started with Appwrite](https://apwr.dev/appcloud)

diff --git a/app/terminal.php b/app/terminal.php deleted file mode 100644 index 1d2f2f0b24..0000000000 --- a/app/terminal.php +++ /dev/null @@ -1,194 +0,0 @@ -column('userId', Table::TYPE_STRING, 64); -$runtimes->column('runtimeHost', Table::TYPE_STRING, 255); -$runtimes->column('runtimePort', Table::TYPE_INT); -$runtimes->create(); - -// Store WebSocket client connections -$clients = []; - -const MAX_PACKAGE_LENGTH = 64000; -const MAX_RUNTIME_CONNECTIONS = 4096; - -$adapter = new Adapter\Swoole(port: System::getEnv('PORT', 80)); -$adapter - ->setPackageMaxLength(MAX_PACKAGE_LENGTH) - ->setWorkerNumber($workerNumber); - -$server = new Server($adapter); - -$server->onStart(function () use ($workerNumber) { - Console::success('Terminal WebSocket Proxy started successfully'); - Console::info('Listening on port: ' . System::getEnv('PORT', 80)); - Console::info('Worker processes: ' . $workerNumber); - Console::info('Max package length: ' . (MAX_PACKAGE_LENGTH / 1000) . 'KB'); - Console::info('Max runtime connections: ' . MAX_RUNTIME_CONNECTIONS); -}); - -$server->onOpen(function (int $connection, $request) use ($server, $runtimes, &$clients) { - try { - Console::info("New connection: {$connection}"); - - // Extract JWT from request - $token = $request->header['authorization'] ?? ''; - if (empty($token)) { - throw new Exception('Missing authentication token', 401); - } - - // Verify JWT and extract user information - $jwt = str_replace('Bearer ', '', $token); - $key = System::getEnv('_APP_OPENSSL_KEY_V1', ''); - $jwt = new JWT($key, 'HS256', 900, 0); - - try { - $payload = $jwt->decode($token); - $userId = $payload['userId'] ?? ''; - $sessionId = $payload['sessionId'] ?? ''; - - if (empty($userId) || empty($sessionId)) { - throw new Exception('Invalid JWT payload', 401); - } - } catch (\Exception $e) { - throw new Exception('Invalid JWT token', 401); - } - - // Get runtime details for user (this could come from your database/cache) - $runtimeHost = "runtime-{$userId}.internal"; // Example hostname - $runtimePort = 9000; - - // Create WebSocket connection to runtime - go(function () use ($server, $connection, &$clients, $runtimeHost, $runtimePort, $userId) { - try { - // $wsClient = new Client("ws://{$runtimeHost}:{$runtimePort}/", [ - // 'timeout' => 0, // Disable timeout for long-running connections - // 'filter' => ['text', 'binary', 'close'] // Only process these frame types - // ]); - - - $wsClient = new Client( - "ws://appwrite-traefik/v1/realtime", - [ - "headers" => [], - "timeout" => 30, - ] - ); - - // Store client connection - $clients[$connection] = [ - 'client' => $wsClient, - 'userId' => $userId - ]; - - // Forward messages from runtime back to client - while (true) { - try { - $message = $wsClient->receive(); - if ($message === null) { - // Connection closed normally - break; - } - $server->send([$connection], $message); - - // Yield to allow other coroutines to run - Swoole\Coroutine::yield(); - - } catch (\WebSocket\ConnectionException $e) { - Console::error("Runtime connection error for user {$userId}: " . $e->getMessage()); - break; - } - } - - // Cleanup on disconnect - $wsClient->close(); - unset($clients[$connection]); - $server->close($connection, CLOSE_NORMAL); - } catch (\WebSocket\ConnectionException $e) { - Console::error("Failed to connect to runtime for user {$userId}: " . $e->getMessage()); - $server->close($connection, CLOSE_SERVER_ERROR); - return; - } - }); - - // Send successful connection message - $server->send([$connection], json_encode([ - 'type' => 'connected', - 'data' => [ - 'userId' => $userId, - 'timestamp' => time() - ] - ])); - - } catch (Throwable $th) { - Console::error('Connection error: ' . $th->getMessage()); - - $server->send([$connection], json_encode([ - 'type' => 'error', - 'data' => [ - 'code' => $th->getCode(), - 'message' => $th->getMessage() - ] - ])); - - $server->close($connection, CLOSE_POLICY_VIOLATION); - } -}); - -$server->onMessage(function (int $connection, string $message) use ($server, &$clients) { - try { - if (!isset($clients[$connection])) { - throw new Exception('Client not connected to runtime', 1008); - } - - $wsClient = $clients[$connection]['client']; - try { - // Forward message to runtime - $wsClient->send($message); - } catch (\WebSocket\ConnectionException $e) { - throw new Exception('Runtime connection lost: ' . $e->getMessage(), 1008); - } - - } catch (Throwable $th) { - $server->send([$connection], json_encode([ - 'type' => 'error', - 'data' => [ - 'code' => $th->getCode(), - 'message' => $th->getMessage() - ] - ])); - - if ($th->getCode() === 1008) { - $server->close($connection, CLOSE_POLICY_VIOLATION); - } - } -}); - -$server->onClose(function (int $connection) use (&$clients) { - if (isset($clients[$connection])) { - $userId = $clients[$connection]['userId']; - $clients[$connection]['client']->close(); - unset($clients[$connection]); - Console::info("Closed connection for user {$userId}"); - } -}); - -$server->start(); \ No newline at end of file diff --git a/src/Appwrite/Auth/Hash/Argon2.php b/src/Appwrite/Auth/Hash/Argon2.php deleted file mode 100644 index c723b077b1..0000000000 --- a/src/Appwrite/Auth/Hash/Argon2.php +++ /dev/null @@ -1,47 +0,0 @@ -getOptions()); - } - - /** - * @param string $password Input password to validate - * @param string $hash Hash to verify password against - * - * @return boolean true if password matches hash - */ - public function verify(string $password, string $hash): bool - { - return \password_verify($password, $hash); - } - - /** - * Get default options for specific hashing algo - * - * @return array options named array - */ - public function getDefaultOptions(): array - { - return ['memory_cost' => 65536, 'time_cost' => 4, 'threads' => 3]; - } -} diff --git a/src/Appwrite/Auth/Hash/Bcrypt.php b/src/Appwrite/Auth/Hash/Bcrypt.php deleted file mode 100644 index 8b6177f33a..0000000000 --- a/src/Appwrite/Auth/Hash/Bcrypt.php +++ /dev/null @@ -1,46 +0,0 @@ -getOptions()); - } - - /** - * @param string $password Input password to validate - * @param string $hash Hash to verify password against - * - * @return boolean true if password matches hash - */ - public function verify(string $password, string $hash): bool - { - return \password_verify($password, $hash); - } - - /** - * Get default options for specific hashing algo - * - * @return array options named array - */ - public function getDefaultOptions(): array - { - return [ 'cost' => 8 ]; - } -} diff --git a/src/Appwrite/Auth/Hash/Md5.php b/src/Appwrite/Auth/Hash/Md5.php deleted file mode 100644 index 8ade3dd5e2..0000000000 --- a/src/Appwrite/Auth/Hash/Md5.php +++ /dev/null @@ -1,44 +0,0 @@ -hash($password) === $hash; - } - - /** - * Get default options for specific hashing algo - * - * @return array options named array - */ - public function getDefaultOptions(): array - { - return []; - } -} diff --git a/src/Appwrite/Auth/Hash/Phpass.php b/src/Appwrite/Auth/Hash/Phpass.php deleted file mode 100644 index 988c38cc8d..0000000000 --- a/src/Appwrite/Auth/Hash/Phpass.php +++ /dev/null @@ -1,290 +0,0 @@ - in 2004-2017 and placed in - * the public domain. Revised in subsequent years, still public domain. - * There's absolutely no warranty. - * The homepage URL for the source framework is: http://www.openwall.com/phpass/ - * Please be sure to update the Version line if you edit this file in any way. - * It is suggested that you leave the main version number intact, but indicate - * your project name (after the slash) and add your own revision information. - * Please do not change the "private" password hashing method implemented in - * here, thereby making your hashes incompatible. However, if you must, please - * change the hash type identifier (the "$P$") to something different. - * Obviously, since this code is in the public domain, the above are not - * requirements (there can be none), but merely suggestions. - * - * @author Solar Designer - * @copyright Copyright (C) 2017 All rights reserved. - * @license http://www.opensource.org/licenses/mit-license.html MIT License; see LICENSE.txt - */ - -namespace Appwrite\Auth\Hash; - -use Appwrite\Auth\Hash; - -/* - * PHPass accepted options: - * int iteration_count_log2; The Logarithmic cost value used when generating hash values indicating the number of rounds used to generate hashes - * string portable_hashes - * string random_state; The cached random state - * - * Reference: https://github.com/photodude/phpass -*/ -class Phpass extends Hash -{ - /** - * Alphabet used in itoa64 conversions. - * - * @var string - * @since 0.1.0 - */ - protected string $itoa64 = './0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'; - - /** - * Get default options for specific hashing algo - * - * @return array options named array - */ - public function getDefaultOptions(): array - { - $randomState = \microtime(); - if (\function_exists('getmypid')) { - $randomState .= getmypid(); - } - - return ['iteration_count_log2' => 8, 'portable_hashes' => false, 'random_state' => $randomState]; - } - - /** - * @param string $password Input password to hash - * - * @return string hash - */ - public function hash(string $password): string - { - $options = $this->getDefaultOptions(); - - $random = ''; - if (CRYPT_BLOWFISH === 1 && !$options['portable_hashes']) { - $random = $this->getRandomBytes(16, $options); - $hash = crypt($password, $this->gensaltBlowfish($random, $options)); - if (strlen($hash) === 60) { - return $hash; - } - } - if (strlen($random) < 6) { - $random = $this->getRandomBytes(6, $options); - } - $hash = $this->cryptPrivate($password, $this->gensaltPrivate($random, $options)); - if (strlen($hash) === 34) { - return $hash; - } - - /** - * Returning '*' on error is safe here, but would _not_ be safe - * in a crypt(3)-like function used _both_ for generating new - * hashes and for validating passwords against existing hashes. - */ - return '*'; - } - - /** - * @param string $password Input password to validate - * @param string $hash Hash to verify password against - * - * @return boolean true if password matches hash - */ - public function verify(string $password, string $hash): bool - { - $verificationHash = $this->cryptPrivate($password, $hash); - if ($verificationHash[0] === '*') { - $verificationHash = crypt($password, $hash); - } - - /** - * This is not constant-time. In order to keep the code simple, - * for timing safety we currently rely on the salts being - * unpredictable, which they are at least in the non-fallback - * cases (that is, when we use /dev/urandom and bcrypt). - */ - return $hash === $verificationHash; - } - - /** - * @param int $count - * - * @return String $output - * @since 0.1.0 - * @throws Exception Thows an Exception if the $count parameter is not a positive integer. - */ - protected function getRandomBytes(int $count, array $options): string - { - if (!is_int($count) || $count < 1) { - throw new \Exception('Argument count must be a positive integer'); - } - $output = ''; - if (@is_readable('/dev/urandom') && ($fh = @fopen('/dev/urandom', 'rb'))) { - $output = fread($fh, $count); - fclose($fh); - } - - if (strlen($output) < $count) { - $output = ''; - - for ($i = 0; $i < $count; $i += 16) { - $options['iteration_count_log2'] = md5(microtime() . $options['iteration_count_log2']); - $output .= md5($options['iteration_count_log2'], true); - } - - $output = substr($output, 0, $count); - } - - return $output; - } - - /** - * @param String $input - * @param int $count - * - * @return String $output - * @since 0.1.0 - * @throws Exception Thows an Exception if the $count parameter is not a positive integer. - */ - protected function encode64($input, $count) - { - if (!is_int($count) || $count < 1) { - throw new \Exception('Argument count must be a positive integer'); - } - $output = ''; - $i = 0; - do { - $value = ord($input[$i++]); - $output .= $this->itoa64[$value & 0x3f]; - if ($i < $count) { - $value |= ord($input[$i]) << 8; - } - $output .= $this->itoa64[($value >> 6) & 0x3f]; - if ($i++ >= $count) { - break; - } - if ($i < $count) { - $value |= ord($input[$i]) << 16; - } - $output .= $this->itoa64[($value >> 12) & 0x3f]; - if ($i++ >= $count) { - break; - } - $output .= $this->itoa64[($value >> 18) & 0x3f]; - } while ($i < $count); - - return $output; - } - - /** - * @param String $input - * - * @return String $output - * @since 0.1.0 - */ - private function gensaltPrivate($input, $options) - { - $output = '$P$'; - $output .= $this->itoa64[min($options['iteration_count_log2'] + ((PHP_VERSION >= '5') ? 5 : 3), 30)]; - $output .= $this->encode64($input, 6); - - return $output; - } - - /** - * @param String $password - * @param String $setting - * - * @return String $output - * @since 0.1.0 - */ - private function cryptPrivate($password, $setting) - { - $output = '*0'; - if (substr($setting, 0, 2) === $output) { - $output = '*1'; - } - $id = substr($setting, 0, 3); - // We use "$P$", phpBB3 uses "$H$" for the same thing - if ($id !== '$P$' && $id !== '$H$') { - return $output; - } - $count_log2 = strpos($this->itoa64, $setting[3]); - if ($count_log2 < 7 || $count_log2 > 30) { - return $output; - } - $count = 1 << $count_log2; - $salt = substr($setting, 4, 8); - if (strlen($salt) !== 8) { - return $output; - } - /** - * We were kind of forced to use MD5 here since it's the only - * cryptographic primitive that was available in all versions of PHP - * in use. To implement our own low-level crypto in PHP - * would have result in much worse performance and - * consequently in lower iteration counts and hashes that are - * quicker to crack (by non-PHP code). - */ - $hash = md5($salt . $password, true); - do { - $hash = md5($hash . $password, true); - } while (--$count); - $output = substr($setting, 0, 12); - $output .= $this->encode64($hash, 16); - - return $output; - } - - /** - * @param String $input - * - * @return String $output - * @since 0.1.0 - */ - private function gensaltBlowfish($input, $options) - { - /** - * This one needs to use a different order of characters and a - * different encoding scheme from the one in encode64() above. - * We care because the last character in our encoded string will - * only represent 2 bits. While two known implementations of - * bcrypt will happily accept and correct a salt string which - * has the 4 unused bits set to non-zero, we do not want to take - * chances and we also do not want to waste an additional byte - * of entropy. - */ - $itoa64 = './ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; - $output = '$2a$'; - $output .= chr(ord('0') + intval($options['iteration_count_log2'] / 10)); - $output .= chr(ord('0') + $options['iteration_count_log2'] % 10); - $output .= '$'; - $i = 0; - do { - $c1 = ord($input[$i++]); - $output .= $itoa64[$c1 >> 2]; - $c1 = ($c1 & 0x03) << 4; - if ($i >= 16) { - $output .= $itoa64[$c1]; - break; - } - $c2 = ord($input[$i++]); - $c1 |= $c2 >> 4; - $output .= $itoa64[$c1]; - $c1 = ($c2 & 0x0f) << 2; - $c2 = ord($input[$i++]); - $c1 |= $c2 >> 6; - $output .= $itoa64[$c1]; - $output .= $itoa64[$c2 & 0x3f]; - } while (1); - - return $output; - } -} diff --git a/src/Appwrite/Auth/Hash/Scrypt.php b/src/Appwrite/Auth/Hash/Scrypt.php deleted file mode 100644 index 821b1fba69..0000000000 --- a/src/Appwrite/Auth/Hash/Scrypt.php +++ /dev/null @@ -1,51 +0,0 @@ -getOptions(); - - return \scrypt($password, $options['salt'], $options['costCpu'], $options['costMemory'], $options['costParallel'], $options['length']); - } - - /** - * @param string $password Input password to validate - * @param string $hash Hash to verify password against - * - * @return boolean true if password matches hash - */ - public function verify(string $password, string $hash): bool - { - return $hash === $this->hash($password); - } - - /** - * Get default options for specific hashing algo - * - * @return array options named array - */ - public function getDefaultOptions(): array - { - return [ 'costCpu' => 8, 'costMemory' => 14, 'costParallel' => 1, 'length' => 64 ]; - } -} diff --git a/src/Appwrite/Auth/Hash/Scryptmodified.php b/src/Appwrite/Auth/Hash/Scryptmodified.php deleted file mode 100644 index 7717f324e5..0000000000 --- a/src/Appwrite/Auth/Hash/Scryptmodified.php +++ /dev/null @@ -1,80 +0,0 @@ -getOptions(); - - $derivedKeyBytes = $this->generateDerivedKey($password); - $signerKeyBytes = \base64_decode($options['signerKey']); - - $hashedPassword = $this->hashKeys($signerKeyBytes, $derivedKeyBytes); - - return \base64_encode($hashedPassword); - } - - /** - * @param string $password Input password to validate - * @param string $hash Hash to verify password against - * - * @return boolean true if password matches hash - */ - public function verify(string $password, string $hash): bool - { - return $this->hash($password) === $hash; - } - - /** - * Get default options for specific hashing algo - * - * @return array options named array - */ - public function getDefaultOptions(): array - { - return [ ]; - } - - private function generateDerivedKey(string $password) - { - $options = $this->getOptions(); - - $saltBytes = \base64_decode($options['salt']); - $saltSeparatorBytes = \base64_decode($options['saltSeparator']); - - $password = mb_convert_encoding($password, 'UTF-8'); - $derivedKey = \scrypt($password, $saltBytes . $saltSeparatorBytes, 16384, 8, 1, 64); - $derivedKey = \hex2bin($derivedKey); - - return $derivedKey; - } - - private function hashKeys($signerKeyBytes, $derivedKeyBytes): string - { - $key = \substr($derivedKeyBytes, 0, 32); - - $iv = "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"; - - $hash = \openssl_encrypt($signerKeyBytes, 'aes-256-ctr', $key, OPENSSL_RAW_DATA, $iv); - - return $hash; - } -} diff --git a/src/Appwrite/Auth/Hash/Sha.php b/src/Appwrite/Auth/Hash/Sha.php deleted file mode 100644 index c2ae3b52c1..0000000000 --- a/src/Appwrite/Auth/Hash/Sha.php +++ /dev/null @@ -1,50 +0,0 @@ -getOptions()['version']; - - return \hash($algo, $password); - } - - /** - * @param string $password Input password to validate - * @param string $hash Hash to verify password against - * - * @return boolean true if password matches hash - */ - public function verify(string $password, string $hash): bool - { - return $this->hash($password) === $hash; - } - - /** - * Get default options for specific hashing algo - * - * @return array options named array - */ - public function getDefaultOptions(): array - { - return [ 'version' => 'sha3-512' ]; - } -} From 0483c7efb5502ce51a1bd464f6db92a9d347b320 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 29 Apr 2025 20:44:05 +0200 Subject: [PATCH 035/245] Merge fixes --- app/controllers/api/account.php | 46 ++---------------- app/init/resources.php | 3 -- composer.json | 2 +- composer.lock | 48 ++++++++----------- src/Appwrite/Migration/Version/V23.php | 11 ----- .../Functions/Http/Executions/Create.php | 8 +--- 6 files changed, 27 insertions(+), 91 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 463bd19890..7939e21818 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -158,15 +158,7 @@ function sendSessionAlert(Locale $locale, Document $user, Document $project, Doc ->trigger(); }; - -<<<<<<< HEAD $createSession = function (string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Mail $queueForMails, Store $store, ProofsToken $proofForToken) { - $roles = Authorization::getRoles(); - $isPrivilegedUser = Auth::isPrivilegedUser($roles); - $isAppUser = Auth::isAppUser($roles); -======= -$createSession = function (string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Mail $queueForMails) { ->>>>>>> origin/1.7.x /** @var Utopia\Database\Document $user */ $userFromRequest = Authorization::skip(fn () => $dbForProject->getDocument('users', $userId)); @@ -287,11 +279,7 @@ $createSession = function (string $userId, string $secret, Request $request, Res ->setAttribute('current', true) ->setAttribute('countryName', $countryName) ->setAttribute('expire', $expire) -<<<<<<< HEAD - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $encoded : '') -======= - ->setAttribute('secret', Auth::encodeSession($user->getId(), $sessionSecret)) ->>>>>>> origin/1.7.x + ->setAttribute('secret', $encoded) ; $response->dynamic($session, Response::MODEL_SESSION); @@ -999,11 +987,7 @@ App::post('/v1/account/sessions/email') $session ->setAttribute('current', true) ->setAttribute('countryName', $countryName) -<<<<<<< HEAD - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $encoded : '') -======= - ->setAttribute('secret', Auth::encodeSession($user->getId(), $secret)) ->>>>>>> origin/1.7.x + ->setAttribute('secret', $encoded) ; $queueForEvents @@ -1166,11 +1150,7 @@ App::post('/v1/account/sessions/anonymous') $session ->setAttribute('current', true) ->setAttribute('countryName', $countryName) -<<<<<<< HEAD - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $encoded : '') -======= - ->setAttribute('secret', Auth::encodeSession($user->getId(), $secret)) ->>>>>>> origin/1.7.x + ->setAttribute('secret', $encoded) ; $response->dynamic($session, Response::MODEL_SESSION); @@ -2654,18 +2634,8 @@ App::post('/v1/account/tokens/phone') $queueForEvents ->setPayload($response->output($token, Response::MODEL_TOKEN), sensitive: ['secret']); -<<<<<<< HEAD - $encoded = $store - ->setProperty('id', $user->getId()) - ->setProperty('secret', $secret) - ->encode(); - - // Hide secret for clients - $token->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $encoded : ''); -======= // Encode secret for clients - $token->setAttribute('secret', Auth::encodeSession($user->getId(), $secret)); ->>>>>>> origin/1.7.x + $token->setAttribute('secret', $encoded); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -3532,16 +3502,8 @@ App::post('/v1/account/verification') throw new Exception(Exception::USER_EMAIL_ALREADY_VERIFIED); } -<<<<<<< HEAD - $roles = Authorization::getRoles(); - $isPrivilegedUser = Auth::isPrivilegedUser($roles); - $isAppUser = Auth::isAppUser($roles); $verificationSecret = $proofForToken->generate(); $expire = DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM); -======= - $verificationSecret = Auth::tokenGenerator(Auth::TOKEN_LENGTH_VERIFICATION); - $expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_CONFIRM); ->>>>>>> origin/1.7.x $verification = new Document([ '$id' => ID::unique(), diff --git a/app/init/resources.php b/app/init/resources.php index c90c72b3d7..00fde2daf2 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -926,7 +926,6 @@ App::setResource('apiKey', function (Request $request, Document $project): ?Key return Key::decode($project, $key); }, ['request', 'project']); -<<<<<<< HEAD App::setResource('store', function (): Store { return new Store(); @@ -957,7 +956,6 @@ App::setResource('proofForCode', function (): Code { $code->setHash(new Sha()); return $code; }); -======= App::setResource('executor', fn () => new Executor(fn (string $projectId, string $deploymentId) => System::getEnv('_APP_EXECUTOR_HOST'))); App::setResource('resourceToken', function ($project, $dbForProject, $request) { @@ -1002,4 +1000,3 @@ App::setResource('resourceToken', function ($project, $dbForProject, $request) { } return new Document([]); }, ['project', 'dbForProject', 'request']); ->>>>>>> origin/1.7.x diff --git a/composer.json b/composer.json index 3bf2a8c759..2991742d47 100644 --- a/composer.json +++ b/composer.json @@ -46,7 +46,7 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.19.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/auth": "dev-dev", + "utopia-php/auth": "0.3.0", "utopia-php/abuse": "0.52.*", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.55.*", diff --git a/composer.lock b/composer.lock index 3daaf6e3e7..dbba4679fe 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "51959289a3f882160f5a9eeb605d41d7", + "content-hash": "2ed8b411e74c4f6e7d514749f28e933a", "packages": [ { "name": "adhocore/jwt", @@ -3300,16 +3300,16 @@ }, { "name": "utopia-php/auth", - "version": "dev-dev", + "version": "0.3.0", "source": { "type": "git", "url": "https://github.com/utopia-php/auth.git", - "reference": "19fb580de44fac5928f9c0211fd0fdfd5022efdb" + "reference": "231e1e0bb97e79438399ffe4de3079063b5dc0b1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/auth/zipball/19fb580de44fac5928f9c0211fd0fdfd5022efdb", - "reference": "19fb580de44fac5928f9c0211fd0fdfd5022efdb", + "url": "https://api.github.com/repos/utopia-php/auth/zipball/231e1e0bb97e79438399ffe4de3079063b5dc0b1", + "reference": "231e1e0bb97e79438399ffe4de3079063b5dc0b1", "shasum": "" }, "require": { @@ -3349,9 +3349,9 @@ ], "support": { "issues": "https://github.com/utopia-php/auth/issues", - "source": "https://github.com/utopia-php/auth/tree/dev" + "source": "https://github.com/utopia-php/auth/tree/0.3.0" }, - "time": "2025-03-19T06:47:02+00:00" + "time": "2025-03-09T21:44:43+00:00" }, { "name": "utopia-php/cache", @@ -3760,16 +3760,16 @@ }, { "name": "utopia-php/fetch", - "version": "0.4.1", + "version": "0.4.2", "source": { "type": "git", "url": "https://github.com/utopia-php/fetch.git", - "reference": "65095dac14037db0c822fb5e209e5bd3187a0303" + "reference": "83986d1be75a2fae4e684107fe70dd78a8e19b77" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/fetch/zipball/65095dac14037db0c822fb5e209e5bd3187a0303", - "reference": "65095dac14037db0c822fb5e209e5bd3187a0303", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/83986d1be75a2fae4e684107fe70dd78a8e19b77", + "reference": "83986d1be75a2fae4e684107fe70dd78a8e19b77", "shasum": "" }, "require": { @@ -3793,9 +3793,9 @@ "description": "A simple library that provides an interface for making HTTP Requests.", "support": { "issues": "https://github.com/utopia-php/fetch/issues", - "source": "https://github.com/utopia-php/fetch/tree/0.4.1" + "source": "https://github.com/utopia-php/fetch/tree/0.4.2" }, - "time": "2025-04-14T07:34:27+00:00" + "time": "2025-04-25T13:48:02+00:00" }, { "name": "utopia-php/framework", @@ -5330,16 +5330,16 @@ }, { "name": "myclabs/deep-copy", - "version": "1.13.0", + "version": "1.13.1", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "024473a478be9df5fdaca2c793f2232fe788e414" + "reference": "1720ddd719e16cf0db4eb1c6eca108031636d46c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/024473a478be9df5fdaca2c793f2232fe788e414", - "reference": "024473a478be9df5fdaca2c793f2232fe788e414", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/1720ddd719e16cf0db4eb1c6eca108031636d46c", + "reference": "1720ddd719e16cf0db4eb1c6eca108031636d46c", "shasum": "" }, "require": { @@ -5378,7 +5378,7 @@ ], "support": { "issues": "https://github.com/myclabs/DeepCopy/issues", - "source": "https://github.com/myclabs/DeepCopy/tree/1.13.0" + "source": "https://github.com/myclabs/DeepCopy/tree/1.13.1" }, "funding": [ { @@ -5386,7 +5386,7 @@ "type": "tidelift" } ], - "time": "2025-02-12T12:17:51+00:00" + "time": "2025-04-29T12:36:36+00:00" }, { "name": "nikic/php-parser", @@ -8284,13 +8284,7 @@ ], "aliases": [], "minimum-stability": "stable", -<<<<<<< HEAD - "stability-flags": { - "utopia-php/auth": 20 - }, -======= - "stability-flags": [], ->>>>>>> origin/1.7.x + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -8314,5 +8308,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } diff --git a/src/Appwrite/Migration/Version/V23.php b/src/Appwrite/Migration/Version/V23.php index f06947f95c..dec7e8e9d3 100644 --- a/src/Appwrite/Migration/Version/V23.php +++ b/src/Appwrite/Migration/Version/V23.php @@ -5,11 +5,8 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; use Exception; use Throwable; -<<<<<<< HEAD -======= use Utopia\CLI\Console; use Utopia\Database\Database; ->>>>>>> origin/1.7.x class V23 extends Migration { @@ -18,9 +15,6 @@ class V23 extends Migration */ public function execute(): void { -<<<<<<< HEAD - // TBD -======= /** * Disable SubQueries for Performance. */ @@ -34,7 +28,6 @@ class V23 extends Migration Console::info('Migrating Collections'); $this->migrateCollections(); ->>>>>>> origin/1.7.x } /** @@ -45,9 +38,6 @@ class V23 extends Migration */ private function migrateCollections(): void { -<<<<<<< HEAD - // TBD -======= $internalProjectId = $this->project->getInternalId(); $collectionType = match ($internalProjectId) { 'console' => 'console', @@ -75,6 +65,5 @@ class V23 extends Migration usleep(50000); } ->>>>>>> origin/1.7.x } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index ec6f214b12..9298cb066b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -96,12 +96,9 @@ class Create extends Base ->inject('queueForStatsUsage') ->inject('queueForFunctions') ->inject('geodb') -<<<<<<< HEAD ->inject('store') ->inject('proofForToken') -======= ->inject('executor') ->>>>>>> origin/1.7.x ->callback([$this, 'action']); } @@ -123,12 +120,9 @@ class Create extends Base StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, -<<<<<<< HEAD Store $store, - Token $proofForToken -======= + Token $proofForToken, Executor $executor ->>>>>>> origin/1.7.x ) { $async = \strval($async) === 'true' || \strval($async) === '1'; From 6f861a91ee22d269db0117982caea88c61f69073 Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Tue, 29 Apr 2025 21:33:59 +0200 Subject: [PATCH 036/245] Updated auth lib --- composer.json | 2 +- composer.lock | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index 2991742d47..907625f679 100644 --- a/composer.json +++ b/composer.json @@ -46,7 +46,7 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.19.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/auth": "0.3.0", + "utopia-php/auth": "0.4.0", "utopia-php/abuse": "0.52.*", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.55.*", diff --git a/composer.lock b/composer.lock index dbba4679fe..b1aa0690d3 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2ed8b411e74c4f6e7d514749f28e933a", + "content-hash": "f86338f8299f81c9fe28fb41bf59a00b", "packages": [ { "name": "adhocore/jwt", @@ -3300,16 +3300,16 @@ }, { "name": "utopia-php/auth", - "version": "0.3.0", + "version": "0.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/auth.git", - "reference": "231e1e0bb97e79438399ffe4de3079063b5dc0b1" + "reference": "02415e1a89cdbc14e3e16a7856ecf7f868869449" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/auth/zipball/231e1e0bb97e79438399ffe4de3079063b5dc0b1", - "reference": "231e1e0bb97e79438399ffe4de3079063b5dc0b1", + "url": "https://api.github.com/repos/utopia-php/auth/zipball/02415e1a89cdbc14e3e16a7856ecf7f868869449", + "reference": "02415e1a89cdbc14e3e16a7856ecf7f868869449", "shasum": "" }, "require": { @@ -3349,9 +3349,9 @@ ], "support": { "issues": "https://github.com/utopia-php/auth/issues", - "source": "https://github.com/utopia-php/auth/tree/0.3.0" + "source": "https://github.com/utopia-php/auth/tree/0.4.0" }, - "time": "2025-03-09T21:44:43+00:00" + "time": "2025-04-29T19:29:28+00:00" }, { "name": "utopia-php/cache", From c70915306d6cbf6aaca7c9b1e20a7f3a0920b448 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 5 Aug 2025 17:04:31 +1200 Subject: [PATCH 037/245] WIP export --- app/controllers/api/migrations.php | 74 +++++++++++++++++++- src/Appwrite/Platform/Workers/Migrations.php | 7 ++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 85751811ba..51f305d16e 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -306,7 +306,7 @@ App::post('/v1/migrations/nhost') ->dynamic($migration, Response::MODEL_MIGRATION); }); -App::post('/v1/migrations/csv') +App::post('/v1/migrations/csv/imports') ->groups(['api', 'migrations']) ->desc('Import documents from a CSV') ->label('scope', 'migrations.write') @@ -315,8 +315,8 @@ App::post('/v1/migrations/csv') ->label('sdk', new Method( namespace: 'migrations', group: null, - name: 'createCsvMigration', - description: '/docs/references/migrations/migration-csv.md', + name: 'createCsvImportMigration', + description: '/docs/references/migrations/migration-csv-import.md', auth: [AuthType::ADMIN], responses: [ new SDKResponse( @@ -431,6 +431,74 @@ App::post('/v1/migrations/csv') ->dynamic($migration, Response::MODEL_MIGRATION); }); +App::post('/v1/migrations/csv/exports') + ->groups(['api', 'migrations']) + ->desc('Export documents to CSV') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.[migrationId].create') + ->label('audits.event', 'migration.create') + ->label('sdk', new Method( + namespace: 'migrations', + group: null, + name: 'createCsvExportMigration', + description: '/docs/references/migrations/migration-csv-export.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_ACCEPTED, + model: Response::MODEL_MIGRATION, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.') + ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('queueForEvents') + ->inject('queueForMigrations') + ->action(function (string $bucketId, string $resourceId, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Migration $queueForMigrations) { + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); + $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $migrationId = ID::unique(); + $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); + + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => $migrationId, + 'status' => 'pending', + 'stage' => 'init', + 'source' => Appwrite::getName(), + 'destination' => CSV::getName(), + 'resources' => $resources, + 'resourceId' => $resourceId, + 'resourceType' => Resource::TYPE_DATABASE, + 'statusCounters' => '{}', + 'resourceData' => '{}', + 'errors' => [], + 'options' => [ + 'bucketId' => $bucketId, + ], + ])); + + $queueForEvents->setParam('migrationId', $migration->getId()); + + $queueForMigrations + ->setMigration($migration) + ->setProject($project) + ->trigger(); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($migration, Response::MODEL_MIGRATION); + }); + App::get('/v1/migrations') ->groups(['api', 'migrations']) ->desc('List migrations') diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 807cf5ec9d..f878853ed6 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -172,6 +172,13 @@ class Migrations extends Action $this->dbForProject, Config::getParam('collections', [])['databases']['collections'], ), + DestinationCSV::getName() => new DestinationCSV( + $this->project, + $this->dbForProject, + $migration->getAttribute('resourceId'), + $migration->getAttribute('options', []), + $this->deviceForImports + ), default => throw new \Exception('Invalid destination type'), }; } From bf1af094c139f7de04c26d6ea4ecf3f400e63b41 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 6 Aug 2025 00:40:39 +1200 Subject: [PATCH 038/245] Add specific column selection --- app/controllers/api/migrations.php | 17 ++++++++++------- app/init/resources.php | 2 +- app/worker.php | 2 +- composer.lock | 12 ++++++------ src/Appwrite/Platform/Workers/Migrations.php | 18 ++++++++---------- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 51f305d16e..f6dc56a5eb 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -307,6 +307,7 @@ App::post('/v1/migrations/nhost') }); App::post('/v1/migrations/csv/imports') + ->alias('/v1/migrations/csv') ->groups(['api', 'migrations']) ->desc('Import documents from a CSV') ->label('scope', 'migrations.write') @@ -332,10 +333,10 @@ App::post('/v1/migrations/csv/imports') ->inject('dbForProject') ->inject('project') ->inject('deviceForFiles') - ->inject('deviceForImports') + ->inject('deviceForMigrations') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForImports, Event $queueForEvents, Migration $queueForMigrations) { + ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForMigrations, Event $queueForEvents, Migration $queueForMigrations) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -361,7 +362,7 @@ App::post('/v1/migrations/csv/imports') $hasCompression = $compression !== Compression::NONE; $migrationId = ID::unique(); - $newPath = $deviceForImports->getPath($migrationId . '_' . $fileId . '.csv'); + $newPath = $deviceForMigrations->getPath($migrationId . '_' . $fileId . '.csv'); if ($hasEncryption || $hasCompression) { $source = $deviceForFiles->read($path); @@ -391,14 +392,14 @@ App::post('/v1/migrations/csv/imports') } // manual write after decryption and/or decompression - if (! $deviceForImports->write($newPath, $source, 'text/csv')) { + if (! $deviceForMigrations->write($newPath, $source, 'text/csv')) { throw new \Exception("Unable to copy file"); } - } elseif (! $deviceForFiles->transfer($path, $newPath, $deviceForImports)) { + } elseif (! $deviceForFiles->transfer($path, $newPath, $deviceForMigrations)) { throw new \Exception("Unable to copy file"); } - $fileSize = $deviceForImports->getFileSize($newPath); + $fileSize = $deviceForMigrations->getFileSize($newPath); $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); $migration = $dbForProject->createDocument('migrations', new Document([ @@ -452,12 +453,13 @@ App::post('/v1/migrations/csv/exports') )) ->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.') ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') + ->param('columns', [], new ArrayList(new Text(255)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.') ->inject('response') ->inject('dbForProject') ->inject('project') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $resourceId, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Migration $queueForMigrations) { + ->action(function (string $bucketId, string $resourceId, array $columns, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Migration $queueForMigrations) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -484,6 +486,7 @@ App::post('/v1/migrations/csv/exports') 'errors' => [], 'options' => [ 'bucketId' => $bucketId, + 'columns' => $columns, ], ])); diff --git a/app/init/resources.php b/app/init/resources.php index 162eab1973..8b7ec941f8 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -514,7 +514,7 @@ App::setResource('deviceForFiles', function ($project, Telemetry $telemetry) { App::setResource('deviceForSites', function ($project, Telemetry $telemetry) { return new Device\Telemetry($telemetry, getDevice(APP_STORAGE_SITES . '/app-' . $project->getId())); }, ['project', 'telemetry']); -App::setResource('deviceForImports', function ($project, Telemetry $telemetry) { +App::setResource('deviceForMigrations', function ($project, Telemetry $telemetry) { return new Device\Telemetry($telemetry, getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId())); }, ['project', 'telemetry']); App::setResource('deviceForFunctions', function ($project, Telemetry $telemetry) { diff --git a/app/worker.php b/app/worker.php index 90f3368fe7..9429cb853f 100644 --- a/app/worker.php +++ b/app/worker.php @@ -341,7 +341,7 @@ Server::setResource('deviceForSites', function (Document $project, Telemetry $te return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_SITES . '/app-' . $project->getId())); }, ['project', 'telemetry']); -Server::setResource('deviceForImports', function (Document $project, Telemetry $telemetry) { +Server::setResource('deviceForMigrations', function (Document $project, Telemetry $telemetry) { return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId())); }, ['project', 'telemetry']); diff --git a/composer.lock b/composer.lock index e3b3d2208f..2cd331ff6d 100644 --- a/composer.lock +++ b/composer.lock @@ -3542,16 +3542,16 @@ }, { "name": "utopia-php/database", - "version": "0.71.11", + "version": "0.71.12", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "644ed827aace63cbdf8c6c64a3998c11b43e3383" + "reference": "72c2a9c185f0f606e4792913a071f744cca21d42" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/644ed827aace63cbdf8c6c64a3998c11b43e3383", - "reference": "644ed827aace63cbdf8c6c64a3998c11b43e3383", + "url": "https://api.github.com/repos/utopia-php/database/zipball/72c2a9c185f0f606e4792913a071f744cca21d42", + "reference": "72c2a9c185f0f606e4792913a071f744cca21d42", "shasum": "" }, "require": { @@ -3592,9 +3592,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.71.11" + "source": "https://github.com/utopia-php/database/tree/0.71.12" }, - "time": "2025-08-05T08:35:29+00:00" + "time": "2025-08-05T09:38:25+00:00" }, { "name": "utopia-php/detector", diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index f878853ed6..4e20c966b1 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Realtime; +use Appwrite\Migration\CSV as DestinationCSV; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -34,7 +35,7 @@ class Migrations extends Action protected Database $dbForPlatform; - protected Device $deviceForImports; + protected Device $deviceForMigrations; protected Document $project; @@ -68,17 +69,17 @@ class Migrations extends Action ->inject('dbForPlatform') ->inject('logError') ->inject('queueForRealtime') - ->inject('deviceForImports') + ->inject('deviceForMigrations') ->callback($this->action(...)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForMigrations): void { $payload = $message->getPayload() ?? []; - $this->deviceForImports = $deviceForImports; + $this->deviceForMigrations = $deviceForMigrations; if (empty($payload)) { throw new Exception('Missing payload'); @@ -146,7 +147,7 @@ class Migrations extends Action CSV::getName() => new CSV( $resourceId, $migrationOptions['path'], - $this->deviceForImports, + $this->deviceForMigrations, $this->dbForProject ), default => throw new \Exception('Invalid source type'), @@ -173,11 +174,9 @@ class Migrations extends Action Config::getParam('collections', [])['databases']['collections'], ), DestinationCSV::getName() => new DestinationCSV( - $this->project, - $this->dbForProject, + $this->deviceForMigrations, $migration->getAttribute('resourceId'), - $migration->getAttribute('options', []), - $this->deviceForImports + $migration->getAttribute('options', [])['columns'] ?? [], ), default => throw new \Exception('Invalid destination type'), }; @@ -211,7 +210,6 @@ class Migrations extends Action // set the errors back without trace $clonedMigrationDocument->setAttribute('errors', $errorMessages); - /** Trigger Realtime Events */ $queueForRealtime ->setProject($project) From 892cfe01b2ebb34e144caccb95f729ee4d607377 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 7 Aug 2025 00:47:11 +1200 Subject: [PATCH 039/245] Optional columns --- app/controllers/api/migrations.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index f6dc56a5eb..36a920a56e 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -453,7 +453,7 @@ App::post('/v1/migrations/csv/exports') )) ->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.') ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') - ->param('columns', [], new ArrayList(new Text(255)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.') + ->param('columns', [], new ArrayList(new Text(255)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true) ->inject('response') ->inject('dbForProject') ->inject('project') From 57a46b98ecd92e630b70d81986da92db917f97eb Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 22 Aug 2025 18:01:38 +1200 Subject: [PATCH 040/245] Use utopia adapter --- composer.json | 2 +- composer.lock | 27 +++++++++++++------- src/Appwrite/Platform/Workers/Migrations.php | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/composer.json b/composer.json index 0c662c775f..fcc0acb213 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/locale": "0.8.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.18.*", - "utopia-php/migration": "1.*", + "utopia-php/migration": "dev-feat-csv-export as 1.0.0", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.8.*", diff --git a/composer.lock b/composer.lock index b7e9a76088..0dffee1bde 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "0da713ee5642eba1d30bc51c1a04a723", + "content-hash": "954529a36566209d6687df9f41a0f2e6", "packages": [ { "name": "adhocore/jwt", @@ -4109,16 +4109,16 @@ }, { "name": "utopia-php/migration", - "version": "1.0.0", + "version": "dev-feat-csv-export", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "0e4499d9dd2c90c2be188cc5fb7a32d9a892b569" + "reference": "8435f1db0db4854ca27cb4c9cf275b905fcb3b41" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/0e4499d9dd2c90c2be188cc5fb7a32d9a892b569", - "reference": "0e4499d9dd2c90c2be188cc5fb7a32d9a892b569", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/8435f1db0db4854ca27cb4c9cf275b905fcb3b41", + "reference": "8435f1db0db4854ca27cb4c9cf275b905fcb3b41", "shasum": "" }, "require": { @@ -4159,9 +4159,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.0.0" + "source": "https://github.com/utopia-php/migration/tree/feat-csv-export" }, - "time": "2025-08-13T09:15:53+00:00" + "time": "2025-08-21T12:56:18+00:00" }, { "name": "utopia-php/orchestration", @@ -8425,9 +8425,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/migration", + "version": "dev-feat-csv-export", + "alias": "1.0.0", + "alias_normalized": "1.0.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/migration": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 23b8cf7ba3..33af785ab5 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,7 +4,7 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Realtime; -use Appwrite\Migration\CSV as DestinationCSV; +use Utopia\Migration\Destinations\CSV as DestinationCSV; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; From 1722e9e416ad998fed889c098526a01f76493848 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 22 Aug 2025 18:03:10 +1200 Subject: [PATCH 041/245] Fix DB read source --- src/Appwrite/Platform/Workers/Migrations.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 33af785ab5..ce9b7e2881 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -18,6 +18,7 @@ use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Source; +use Utopia\Migration\Sources\Appwrite; use Utopia\Migration\Sources\Appwrite as SourceAppwrite; use Utopia\Migration\Sources\CSV; use Utopia\Migration\Sources\Firebase; @@ -113,9 +114,17 @@ class Migrations extends Action protected function processSource(Document $migration): Source { $source = $migration->getAttribute('source'); + $destination = $migration->getAttribute('destination'); $resourceId = $migration->getAttribute('resourceId'); $credentials = $migration->getAttribute('credentials'); $migrationOptions = $migration->getAttribute('options'); + $dataSource = Appwrite::SOURCE_API; + $database = null; + + if ($source === Appwrite::getName() && $destination === DestinationCSV::getName()) { + $dataSource = Appwrite::SOURCE_DATABASE; + $database = $this->dbForProject; + } $migrationSource = match ($source) { Firebase::getName() => new Firebase( @@ -143,6 +152,8 @@ class Migrations extends Action $credentials['projectId'], $credentials['endpoint'] === 'http://localhost/v1' ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey'], + $dataSource, + $database ), CSV::getName() => new CSV( $resourceId, @@ -251,7 +262,9 @@ class Migrations extends Action 'functions.write', 'databases.read', 'collections.read', + 'collections.write', 'tables.read', + 'tables.write', 'documents.read', 'documents.write', 'rows.read', From 4478edc211bbbf8540ee83472611c9badc0b7ff5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 10 Sep 2025 04:25:21 +0000 Subject: [PATCH 042/245] fix linter --- app/config/console.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/config/console.php b/app/config/console.php index 2ee0712691..6ba9533728 100644 --- a/app/config/console.php +++ b/app/config/console.php @@ -4,8 +4,6 @@ * Initializes console project document. */ -use Appwrite\Network\Validator\Origin; -use Appwrite\Auth\Auth; use Appwrite\Network\Platform; use Utopia\Database\Helpers\ID; use Utopia\System\System; From 631c00023e418d7e99414bbdcd7ca6ead686ec42 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 10 Sep 2025 04:38:14 +0000 Subject: [PATCH 043/245] update lock --- composer.lock | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/composer.lock b/composer.lock index d2cb0af89f..70c79d2f52 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7553e976312b0423cc31544abb91caec", + "content-hash": "6e00e41bd4002c54cae87f68e5cb3742", "packages": [ { "name": "adhocore/jwt", @@ -3693,16 +3693,16 @@ }, { "name": "utopia-php/database", - "version": "1.4.1", + "version": "1.4.4", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "b5ea4d133a1a4e747b7522e61e072289129a06f4" + "reference": "16f96e5d9784dae87d4f6b864e87da8e3be15507" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/b5ea4d133a1a4e747b7522e61e072289129a06f4", - "reference": "b5ea4d133a1a4e747b7522e61e072289129a06f4", + "url": "https://api.github.com/repos/utopia-php/database/zipball/16f96e5d9784dae87d4f6b864e87da8e3be15507", + "reference": "16f96e5d9784dae87d4f6b864e87da8e3be15507", "shasum": "" }, "require": { @@ -3743,9 +3743,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/1.4.1" + "source": "https://github.com/utopia-php/database/tree/1.4.4" }, - "time": "2025-09-05T13:23:52+00:00" + "time": "2025-09-10T00:50:05+00:00" }, { "name": "utopia-php/detector", @@ -5062,16 +5062,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.3.2", + "version": "1.3.4", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "375a6c9b168db6fdf58fbe0d49d2261d80700b4a" + "reference": "d3b420dced42f1eec1f6d0aa98b7bbf8de4042ac" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/375a6c9b168db6fdf58fbe0d49d2261d80700b4a", - "reference": "375a6c9b168db6fdf58fbe0d49d2261d80700b4a", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/d3b420dced42f1eec1f6d0aa98b7bbf8de4042ac", + "reference": "d3b420dced42f1eec1f6d0aa98b7bbf8de4042ac", "shasum": "" }, "require": { @@ -5107,9 +5107,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.3.2" + "source": "https://github.com/appwrite/sdk-generator/tree/1.3.4" }, - "time": "2025-09-05T15:50:35+00:00" + "time": "2025-09-08T11:56:04+00:00" }, { "name": "doctrine/annotations", @@ -8567,7 +8567,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -8591,5 +8591,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } From 41dbe1a384ee8762d69c261226a7d45a2b8034c7 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 10 Sep 2025 07:03:11 +0000 Subject: [PATCH 044/245] Fix internal ID refactor --- app/controllers/api/account.php | 16 ++++++++-------- app/controllers/api/teams.php | 2 +- app/controllers/api/users.php | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index fb36829e2c..5d39c2728c 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1110,7 +1110,7 @@ App::post('/v1/account/sessions/anonymous') [ '$id' => ID::unique(), 'userId' => $user->getId(), - 'userInternalId' => $user->getInternalId(), + 'userInternalId' => $user->getSequence(), 'provider' => SESSION_PROVIDER_ANONYMOUS, 'secret' => $proofForToken->hash($secret), // One way hash encryption to protect DB leak 'userAgent' => $request->getUserAgent('UNKNOWN'), @@ -1726,7 +1726,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $token = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), - 'userInternalId' => $user->getInternalId(), + 'userInternalId' => $user->getSequence(), 'type' => TOKEN_TYPE_OAUTH2, 'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak 'expire' => $expire, @@ -2039,7 +2039,7 @@ App::post('/v1/account/tokens/magic-url') $token = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), - 'userInternalId' => $user->getInternalId(), + 'userInternalId' => $user->getSequence(), 'type' => TOKEN_TYPE_MAGIC_URL, 'secret' => $proofForToken->hash($tokenSecret), // One way hash encryption to protect DB leak 'expire' => $expire, @@ -2312,7 +2312,7 @@ App::post('/v1/account/tokens/email') $token = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), - 'userInternalId' => $user->getInternalId(), + 'userInternalId' => $user->getSequence(), 'type' => TOKEN_TYPE_EMAIL, 'secret' => $proofForCode->hash($tokenSecret), // One way hash encryption to protect DB leak 'expire' => $expire, @@ -2652,7 +2652,7 @@ App::post('/v1/account/tokens/phone') $token = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), - 'userInternalId' => $user->getInternalId(), + 'userInternalId' => $user->getSequence(), 'type' => TOKEN_TYPE_PHONE, 'secret' => $proofForToken->hash($secret), 'expire' => $expire, @@ -3354,7 +3354,7 @@ App::post('/v1/account/recovery') $recovery = new Document([ '$id' => ID::unique(), 'userId' => $profile->getId(), - 'userInternalId' => $profile->getInternalId(), + 'userInternalId' => $profile->getSequence(), 'type' => TOKEN_TYPE_RECOVERY, 'secret' => $proofForToken->hash($secret), // One way hash encryption to protect DB leak 'expire' => $expire, @@ -3617,7 +3617,7 @@ App::post('/v1/account/verification') $verification = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), - 'userInternalId' => $user->getInternalId(), + 'userInternalId' => $user->getSequence(), 'type' => TOKEN_TYPE_VERIFICATION, 'secret' => $proofForToken->hash($verificationSecret), // One way hash encryption to protect DB leak 'expire' => $expire, @@ -3869,7 +3869,7 @@ App::post('/v1/account/verification/phone') $verification = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), - 'userInternalId' => $user->getInternalId(), + 'userInternalId' => $user->getSequence(), 'type' => TOKEN_TYPE_PHONE, 'secret' => $proofForCode->hash($secret), 'expire' => $expire, diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 0f2672af79..4732c6c810 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1257,7 +1257,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') Permission::delete(Role::user($user->getId())), ], 'userId' => $user->getId(), - 'userInternalId' => $user->getInternalId(), + 'userInternalId' => $user->getSequence(), 'provider' => SESSION_PROVIDER_EMAIL, 'providerUid' => $user->getAttribute('email'), 'secret' => $proofForToken->hash($secret), // One way hash encryption to protect DB leak diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index dbd33f0903..95cf41ee09 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -2239,7 +2239,7 @@ App::post('/v1/users/:userId/sessions') [ '$id' => ID::unique(), 'userId' => $user->getId(), - 'userInternalId' => $user->getInternalId(), + 'userInternalId' => $user->getSequence(), 'provider' => SESSION_PROVIDER_SERVER, 'secret' => $proofForToken->hash($secret), // One way hash encryption to protect DB leak 'userAgent' => $request->getUserAgent('UNKNOWN'), @@ -2327,7 +2327,7 @@ App::post('/v1/users/:userId/tokens') $token = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), - 'userInternalId' => $user->getInternalId(), + 'userInternalId' => $user->getSequence(), 'type' => TOKEN_TYPE_GENERIC, 'secret' => $proofForToken->hash($secret), 'expire' => $expire, From 487663f8922ee3fe1cef4002d59cfd1a878a26fa Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 10 Sep 2025 07:30:22 +0000 Subject: [PATCH 045/245] fixed attribute --- app/config/collections/common.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/collections/common.php b/app/config/collections/common.php index 2dd9a763e1..a291f7b19d 100644 --- a/app/config/collections/common.php +++ b/app/config/collections/common.php @@ -1115,7 +1115,7 @@ return [ '$id' => ID::custom('expire'), 'type' => Database::VAR_DATETIME, 'size' => 0, - 'required' => true, + 'required' => false, 'format' => '', 'signed' => false, 'default' => null, From 6099313cb42dd66df7aac8a94fc90a560a0b5b26 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 10 Sep 2025 07:57:45 +0000 Subject: [PATCH 046/245] Fxi teams tests --- app/controllers/api/account.php | 8 ++++++-- tests/e2e/Services/Teams/TeamsBaseClient.php | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 5d39c2728c..aae55c52e3 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2954,8 +2954,9 @@ App::patch('/v1/account/password') ->inject('dbForProject') ->inject('queueForEvents') ->inject('hooks') + ->inject('store') ->inject('proofForPassword') - ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks, ProofsPassword $proofForPassword) { + ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks, Store $store, ProofsPassword $proofForPassword) { $userProofForPassword = ProofsPassword::createHash($user->getAttribute('hash'), $user->getAttribute('hashOptions')); // Check old password only if its an existing user. @@ -2995,7 +2996,10 @@ App::patch('/v1/account/password') ->setAttribute('hashOptions', $proofForPassword->getHash()->getOptions()); $sessions = $user->getAttribute('sessions', []); - $current = Auth::sessionVerify($sessions, Auth::$secret); + + $proofsToken = new ProofsToken(); + + $current = Auth::sessionVerify($sessions, $store->getProperty('secret', ''), $proofsToken); $invalidate = $project->getAttribute('auths', default: [])['invalidateSessions'] ?? false; if ($invalidate && !empty($current)) { foreach ($sessions as $session) { diff --git a/tests/e2e/Services/Teams/TeamsBaseClient.php b/tests/e2e/Services/Teams/TeamsBaseClient.php index 5c8e94feb1..74ffe704b5 100644 --- a/tests/e2e/Services/Teams/TeamsBaseClient.php +++ b/tests/e2e/Services/Teams/TeamsBaseClient.php @@ -718,7 +718,7 @@ trait TeamsBaseClient ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(4, $response['body']['total']); + $this->assertEquals(3, $response['body']['total']); $ownerMembershipUid = $response['body']['memberships'][0]['$id']; @@ -773,7 +773,7 @@ trait TeamsBaseClient ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(3, $response['body']['total']); + $this->assertEquals(2, $response['body']['total']); /** * Test for when the owner tries to delete their membership From 4578819a582c75b3e78e99e21a2a0fd4a0dbdd81 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 10 Sep 2025 07:59:00 +0000 Subject: [PATCH 047/245] revert test update --- tests/e2e/Services/Teams/TeamsBaseClient.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Services/Teams/TeamsBaseClient.php b/tests/e2e/Services/Teams/TeamsBaseClient.php index 74ffe704b5..5c8e94feb1 100644 --- a/tests/e2e/Services/Teams/TeamsBaseClient.php +++ b/tests/e2e/Services/Teams/TeamsBaseClient.php @@ -718,7 +718,7 @@ trait TeamsBaseClient ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(3, $response['body']['total']); + $this->assertEquals(4, $response['body']['total']); $ownerMembershipUid = $response['body']['memberships'][0]['$id']; @@ -773,7 +773,7 @@ trait TeamsBaseClient ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(2, $response['body']['total']); + $this->assertEquals(3, $response['body']['total']); /** * Test for when the owner tries to delete their membership From 356fbed325a521d72191b570620adea9fd60c7f6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 10 Sep 2025 09:09:12 +0000 Subject: [PATCH 048/245] fix password update --- app/controllers/api/account.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index aae55c52e3..eafff19073 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2956,8 +2956,9 @@ App::patch('/v1/account/password') ->inject('hooks') ->inject('store') ->inject('proofForPassword') - ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks, Store $store, ProofsPassword $proofForPassword) { - + ->inject('proofForToken') + ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks, Store $store, ProofsPassword $proofForPassword, ProofsToken $proofForToken) { + var_dump('updating password ' . $oldPassword . ':' . $password); $userProofForPassword = ProofsPassword::createHash($user->getAttribute('hash'), $user->getAttribute('hashOptions')); // Check old password only if its an existing user. if (!empty($user->getAttribute('passwordUpdate')) && !$userProofForPassword->verify($oldPassword, $user->getAttribute('password'))) { // Double check user password @@ -2997,9 +2998,8 @@ App::patch('/v1/account/password') $sessions = $user->getAttribute('sessions', []); - $proofsToken = new ProofsToken(); + $current = Auth::sessionVerify($sessions, $store->getProperty('secret', ''), $proofForToken); - $current = Auth::sessionVerify($sessions, $store->getProperty('secret', ''), $proofsToken); $invalidate = $project->getAttribute('auths', default: [])['invalidateSessions'] ?? false; if ($invalidate && !empty($current)) { foreach ($sessions as $session) { From 55bebd92f3e90f52c04359627899771640ede713 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 11 Sep 2025 06:27:58 +0000 Subject: [PATCH 049/245] Fix date format --- app/controllers/api/account.php | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index eafff19073..f6a2760cde 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -213,7 +213,7 @@ $createSession = function (string $userId, string $secret, Request $request, Res 'ip' => $request->getIP(), 'factors' => [$factor], 'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--', - 'expire' => DateTime::addSeconds(new \DateTime(), $duration) + 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)) ], $detector->getOS(), $detector->getClient(), @@ -838,7 +838,7 @@ App::patch('/v1/account/sessions/:sessionId') // Extend session $authDuration = $project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; - $session->setAttribute('expire', DateTime::addSeconds(new \DateTime(), $authDuration)); + $session->setAttribute('expire', DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $authDuration))); // Refresh OAuth access token $provider = $session->getAttribute('provider', ''); @@ -950,7 +950,7 @@ App::post('/v1/account/sessions/email') 'ip' => $request->getIP(), 'factors' => ['password'], 'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--', - 'expire' => DateTime::addSeconds(new \DateTime(), $duration) + 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)) ], $detector->getOS(), $detector->getClient(), @@ -1117,7 +1117,7 @@ App::post('/v1/account/sessions/anonymous') 'ip' => $request->getIP(), 'factors' => ['anonymous'], 'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--', - 'expire' => DateTime::addSeconds(new \DateTime(), $duration) + 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)) ], $detector->getOS(), $detector->getClient(), @@ -1772,7 +1772,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') 'ip' => $request->getIP(), 'factors' => [TYPE::EMAIL, 'oauth2'], // include a special oauth2 factor to bypass MFA checks 'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--', - 'expire' => DateTime::addSeconds(new \DateTime(), $duration) + 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)) ], $detector->getOS(), $detector->getClient(), $detector->getDevice())); $session = $dbForProject->createDocument('sessions', $session->setAttribute('$permissions', [ @@ -3352,7 +3352,7 @@ App::post('/v1/account/recovery') throw new Exception(Exception::USER_BLOCKED); } - $expire = DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_RECOVERY); + $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_RECOVERY)); $secret = $proofForToken->generate(); $recovery = new Document([ @@ -3616,7 +3616,7 @@ App::post('/v1/account/verification') } $verificationSecret = $proofForToken->generate(); - $expire = DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM); + $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM)); $verification = new Document([ '$id' => ID::unique(), @@ -3868,7 +3868,7 @@ App::post('/v1/account/verification/phone') } $secret ??= $proofForCode->generate(); - $expire = DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM); + $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM)); $verification = new Document([ '$id' => ID::unique(), @@ -4639,7 +4639,7 @@ App::post('/v1/account/mfa/challenge') ->inject('proofForCode') ->action(function (string $factor, Response $response, Database $dbForProject, Document $user, Locale $locale, Document $project, Request $request, Event $queueForEvents, Messaging $queueForMessaging, Mail $queueForMails, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, ProofsToken $proofForToken, ProofsCode $proofForCode) { - $expire = DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM); + $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM)); $code = $proofForCode->generate(); $challenge = new Document([ From 6d19d76bac375a428e7e8c7bdecbb33a97b93d2e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 11 Sep 2025 06:51:01 +0000 Subject: [PATCH 050/245] Fix scope check --- app/controllers/shared/api.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index e84699274f..4f7e351084 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -434,9 +434,9 @@ App::init() } // Step 9: Validate scope permissions - $scope = $route->getLabel('scope', 'none'); - if (!\in_array($scope, $scopes)) { - throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, $user->getAttribute('email', 'User') . ' (role: ' . \strtolower($roles[$role]['label']) . ') missing scope (' . $scope . ')'); + $allowed = (array)$route->getLabel('scope', 'none'); + if (empty(\array_intersect($allowed, $scopes))) { + throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, $user->getAttribute('email', 'User') . ' (role: ' . \strtolower($roles[$role]['label']) . ') missing scopes (' . \json_encode($allowed) . ')'); } // Step 10: Check if user is blocked From e4d70a4d4f87435108cca8c09c5b9e2731c6624c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 11 Sep 2025 07:55:52 +0000 Subject: [PATCH 051/245] Fix: default options --- app/config/collections/common.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/config/collections/common.php b/app/config/collections/common.php index a291f7b19d..804929fcfd 100644 --- a/app/config/collections/common.php +++ b/app/config/collections/common.php @@ -1,5 +1,6 @@ 256, 'signed' => true, 'required' => false, - 'default' => '', + 'default' => (new Argon2())->getName(), 'array' => false, 'filters' => [], ], @@ -183,7 +184,7 @@ return [ 'size' => 65535, 'signed' => true, 'required' => false, - 'default' => new \stdClass(), + 'default' => (new Argon2())->getOptions(), 'array' => false, 'filters' => ['json'], ], From 72186f18826cda51bee83a64e8a71a653e8a6f19 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 11 Sep 2025 08:46:38 +0000 Subject: [PATCH 052/245] reset det change --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index f6a2760cde..02b8c92125 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -213,7 +213,7 @@ $createSession = function (string $userId, string $secret, Request $request, Res 'ip' => $request->getIP(), 'factors' => [$factor], 'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--', - 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)) + 'expire' => DateTime::addSeconds(new \DateTime(), $duration) ], $detector->getOS(), $detector->getClient(), From bcec5c0922475d3b621b9667e691a30fca16866c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 11 Sep 2025 08:57:27 +0000 Subject: [PATCH 053/245] revert date format --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 02b8c92125..ccf67c5095 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -838,7 +838,7 @@ App::patch('/v1/account/sessions/:sessionId') // Extend session $authDuration = $project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; - $session->setAttribute('expire', DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $authDuration))); + $session->setAttribute('expire', DateTime::addSeconds(new \DateTime(), $authDuration)); // Refresh OAuth access token $provider = $session->getAttribute('provider', ''); From e36de6b72ba4f3df9a3f088f9788bf8d19a2527f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 11 Sep 2025 08:59:27 +0000 Subject: [PATCH 054/245] revert date format --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index ccf67c5095..c6b8131680 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -950,7 +950,7 @@ App::post('/v1/account/sessions/email') 'ip' => $request->getIP(), 'factors' => ['password'], 'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--', - 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)) + 'expire' => DateTime::addSeconds(new \DateTime(), $duration) ], $detector->getOS(), $detector->getClient(), From 72f5793928fcc5d7c28a6ec6d6f399c4df51d2b0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 11 Sep 2025 09:18:23 +0000 Subject: [PATCH 055/245] revert date format --- app/controllers/api/account.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index c6b8131680..059dce5fc1 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1117,7 +1117,7 @@ App::post('/v1/account/sessions/anonymous') 'ip' => $request->getIP(), 'factors' => ['anonymous'], 'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--', - 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)) + 'expire' => DateTime::addSeconds(new \DateTime(), $duration) ], $detector->getOS(), $detector->getClient(), @@ -1772,7 +1772,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') 'ip' => $request->getIP(), 'factors' => [TYPE::EMAIL, 'oauth2'], // include a special oauth2 factor to bypass MFA checks 'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--', - 'expire' => DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)) + 'expire' => DateTime::addSeconds(new \DateTime(), $duration) ], $detector->getOS(), $detector->getClient(), $detector->getDevice())); $session = $dbForProject->createDocument('sessions', $session->setAttribute('$permissions', [ From c903caabcc4553d1dd3ff7f5dbb1369fce4b99a3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 14 Sep 2025 04:50:13 +0000 Subject: [PATCH 056/245] remove dump --- app/controllers/api/account.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 059dce5fc1..e979fc6255 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2958,7 +2958,6 @@ App::patch('/v1/account/password') ->inject('proofForPassword') ->inject('proofForToken') ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks, Store $store, ProofsPassword $proofForPassword, ProofsToken $proofForToken) { - var_dump('updating password ' . $oldPassword . ':' . $password); $userProofForPassword = ProofsPassword::createHash($user->getAttribute('hash'), $user->getAttribute('hashOptions')); // Check old password only if its an existing user. if (!empty($user->getAttribute('passwordUpdate')) && !$userProofForPassword->verify($oldPassword, $user->getAttribute('password'))) { // Double check user password From 1c726d046616dd64c17a232cbdaf591fa55c58d4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 14 Sep 2025 04:53:49 +0000 Subject: [PATCH 057/245] fix: encoded not defined --- app/controllers/api/teams.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 4732c6c810..1c96aa0116 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -1272,12 +1272,12 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') Authorization::setRole(Role::user($userId)->toString()); - if (!Config::getParam('domainVerification')) { - $encoded = $store - ->setProperty('id', $user->getId()) - ->setProperty('secret', $secret) - ->encode(); + $encoded = $store + ->setProperty('id', $user->getId()) + ->setProperty('secret', $secret) + ->encode(); + if (!Config::getParam('domainVerification')) { $response->addHeader('X-Fallback-Cookies', \json_encode([$store->getKey() => $encoded])); } From 365aaca56df32dc8bbe8c990d432a82175dc5ba5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 14 Sep 2025 04:58:34 +0000 Subject: [PATCH 058/245] fix: remove legacy token generator use --- app/controllers/api/account.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index e979fc6255..ed2f839b5b 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1720,15 +1720,16 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $duration = $project->getAttribute('auths', [])['duration'] ?? TOKEN_EXPIRATION_LOGIN_LONG; $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); + $proofsForTokenOAuth2 = new ProofsToken(TOKEN_LENGTH_OAUTH2); // If the `token` param is set, we will return the token in the query string if ($state['token']) { - $secret = Auth::tokenGenerator(TOKEN_LENGTH_OAUTH2); + $secret = $proofsForTokenOAuth2->generate(); $token = new Document([ '$id' => ID::unique(), 'userId' => $user->getId(), 'userInternalId' => $user->getSequence(), 'type' => TOKEN_TYPE_OAUTH2, - 'secret' => Auth::hash($secret), // One way hash encryption to protect DB leak + 'secret' => $proofsForTokenOAuth2->hash($secret), // One way hash encryption to protect DB leak 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), 'ip' => $request->getIP(), From 32b290a23150c73933b0e4856cd14cc89da3756e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 14 Sep 2025 10:44:46 +0545 Subject: [PATCH 059/245] Update src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matej Bačo --- .../Platform/Modules/Functions/Http/Executions/Create.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index bed59d96bf..4d14efee3d 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -205,7 +205,7 @@ class Create extends Base foreach ($sessions as $session) { /** @var Utopia\Database\Document $session */ - if ($proofForToken->verify($store->getProperty('secret', ''), $session->getAttribute('secret'))) { // If current session delete the cookies too + if ($proofForToken->verify($store->getProperty('secret', ''), $session->getAttribute('secret'))) { // Find most recent active session for user ID and JWT headers $current = $session; } } From 3b668f78064f6b9d75fd3cf85b7a35ef5bfa2c40 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 14 Sep 2025 10:45:53 +0545 Subject: [PATCH 060/245] Update composer.json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matej Bačo --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index a38dc080ed..e456ec1385 100644 --- a/composer.json +++ b/composer.json @@ -46,7 +46,7 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.19.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/auth": "0.4.0", + "utopia-php/auth": "0.4.*", "utopia-php/abuse": "1.*", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "1.*", From d42841d80aa8eccd14dfde312bd8d00328de6d83 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 14 Sep 2025 05:02:50 +0000 Subject: [PATCH 061/245] fix: remove hardcode, reuse hasher --- app/controllers/api/users.php | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 95cf41ee09..b8874bf076 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1402,12 +1402,8 @@ App::patch('/v1/users/:userId/password') ->setAttribute('password', $newPassword) ->setAttribute('passwordHistory', $history) ->setAttribute('passwordUpdate', DateTime::now()) - ->setAttribute('hash', 'argon2') - ->setAttribute('hashOptions', [ - 'memoryCost' => 65536, - 'timeCost' => 4, - 'threads' => 3 - ]); + ->setAttribute('hash', $hasher->getName()) + ->setAttribute('hashOptions', $hasher->getOptions()); $user = $dbForProject->updateDocument('users', $user->getId(), $user); From cfa453079793ac889e87c1ca1cc7dbee42deb513 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 14 Sep 2025 05:03:20 +0000 Subject: [PATCH 062/245] composer update --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index 063541a5ba..948b18c211 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6e00e41bd4002c54cae87f68e5cb3742", + "content-hash": "883816d2ccfa5372c8c3bb0504dde205", "packages": [ { "name": "adhocore/jwt", From 800db0b99debdbfb4e93e264e3aa8b05147ab4f9 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 14 Sep 2025 05:23:30 +0000 Subject: [PATCH 063/245] Fix magic URL token length --- app/controllers/api/account.php | 5 +++-- tests/e2e/Services/Account/AccountCustomClientTest.php | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index ed2f839b5b..3059f3e815 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2034,7 +2034,8 @@ App::post('/v1/account/tokens/magic-url') Authorization::skip(fn () => $dbForProject->createDocument('users', $user)); } - $tokenSecret = $proofForToken->generate(); + $proofsForTokenMagicUrl = new ProofsToken(TOKEN_LENGTH_MAGIC_URL); + $tokenSecret = $proofsForTokenMagicUrl->generate(); $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM)); $token = new Document([ @@ -2042,7 +2043,7 @@ App::post('/v1/account/tokens/magic-url') 'userId' => $user->getId(), 'userInternalId' => $user->getSequence(), 'type' => TOKEN_TYPE_MAGIC_URL, - 'secret' => $proofForToken->hash($tokenSecret), // One way hash encryption to protect DB leak + 'secret' => $proofsForTokenMagicUrl->hash($tokenSecret), // One way hash encryption to protect DB leak 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), 'ip' => $request->getIP(), diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index a8035ff234..bd3fec8439 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -2698,7 +2698,7 @@ class AccountCustomClientTest extends Scope $this->assertStringContainsStringIgnoringCase('Sign in to '. $this->getProject()['name'] . ' with your secure link. Expires in 1 hour.', $lastEmail['text']); $this->assertStringNotContainsStringIgnoringCase('security phrase', $lastEmail['text']); - $token = substr($lastEmail['text'], strpos($lastEmail['text'], '&secret=', 0) + 8, 256); + $token = substr($lastEmail['text'], strpos($lastEmail['text'], '&secret=', 0) + 8, 64); $expireTime = strpos($lastEmail['text'], 'expire=' . urlencode($response['body']['expire']), 0); From 518389d32c854f8bb556e7c716e8d0b2b7b32c6d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 14 Sep 2025 05:37:06 +0000 Subject: [PATCH 064/245] fix length --- app/controllers/api/account.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 3059f3e815..38c505f07c 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2034,8 +2034,9 @@ App::post('/v1/account/tokens/magic-url') Authorization::skip(fn () => $dbForProject->createDocument('users', $user)); } - $proofsForTokenMagicUrl = new ProofsToken(TOKEN_LENGTH_MAGIC_URL); - $tokenSecret = $proofsForTokenMagicUrl->generate(); + $proofForToken->setLength(TOKEN_LENGTH_MAGIC_URL); + + $tokenSecret = $proofForToken->generate(); $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM)); $token = new Document([ @@ -2043,7 +2044,7 @@ App::post('/v1/account/tokens/magic-url') 'userId' => $user->getId(), 'userInternalId' => $user->getSequence(), 'type' => TOKEN_TYPE_MAGIC_URL, - 'secret' => $proofsForTokenMagicUrl->hash($tokenSecret), // One way hash encryption to protect DB leak + 'secret' => $proofForToken->hash($tokenSecret), // One way hash encryption to protect DB leak 'expire' => $expire, 'userAgent' => $request->getUserAgent('UNKNOWN'), 'ip' => $request->getIP(), From 13b98409aecfb7d1c0a9ad32c490cab9f2c8bb24 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 14 Sep 2025 05:56:56 +0000 Subject: [PATCH 065/245] reset length --- tests/e2e/Services/Account/AccountCustomServerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Account/AccountCustomServerTest.php b/tests/e2e/Services/Account/AccountCustomServerTest.php index e0a52c4007..eb72a99913 100644 --- a/tests/e2e/Services/Account/AccountCustomServerTest.php +++ b/tests/e2e/Services/Account/AccountCustomServerTest.php @@ -218,7 +218,7 @@ class AccountCustomServerTest extends Scope $this->assertEquals($email, $lastEmail['to'][0]['address']); $this->assertEquals($this->getProject()['name'] . ' Login', $lastEmail['subject']); - $token = substr($lastEmail['text'], strpos($lastEmail['text'], '&secret=', 0) + 8, 256); + $token = substr($lastEmail['text'], strpos($lastEmail['text'], '&secret=', 0) + 8, 64); $expireTime = strpos($lastEmail['text'], 'expire=' . urlencode($response['body']['expire']), 0); From 19cf94bd7ed1aee53ea1e5a2c586dbc805008a98 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 15 Sep 2025 10:22:45 +0000 Subject: [PATCH 066/245] Fix oauth2 changes --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 38c505f07c..941a28efae 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1501,7 +1501,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $name = ''; $nameOAuth = $oauth2->getUserName($accessToken); - $userParam = \json_decode($request->getParam('user', '{}'), true); // only valid for Apple OAuth2 which returns a user param in the request + $userParam = $request->getParam('user'); if (!empty($nameOAuth)) { $name = $nameOAuth; } elseif ($userParam !== null) { From 4579e41ace3ada4336c3f98f2e811d417ba16658 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 15 Sep 2025 10:33:21 +0000 Subject: [PATCH 067/245] update check --- app/init/resources.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init/resources.php b/app/init/resources.php index a1de0826d5..8e19d26beb 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -265,7 +265,7 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons } $fallback = $request->getHeader('x-fallback-cookies', ''); $fallback = \json_decode($fallback, true); - $store->decode(((isset($fallback[$store->getKey()])) ? $fallback[$store->getKey()] : '')); + $store->decode(((is_array($fallback) && isset($fallback[$store->getKey()])) ? $fallback[$store->getKey()] : '')); } if (APP_MODE_ADMIN !== $mode) { From 780799f87a51160d5d18fa2a9ded1f07aa559683 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 15 Sep 2025 10:41:29 +0000 Subject: [PATCH 068/245] update to initialize first --- src/Appwrite/Platform/Tasks/Install.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Install.php b/src/Appwrite/Platform/Tasks/Install.php index 246c2fb3cc..c70ced33ee 100644 --- a/src/Appwrite/Platform/Tasks/Install.php +++ b/src/Appwrite/Platform/Tasks/Install.php @@ -158,12 +158,14 @@ class Install extends Action } if ($var['filter'] === 'token') { - $input[$var['name']] = (new Token())->generate(); + $token = new Token(); + $input[$var['name']] = $token->generate(); continue; } if ($var['filter'] === 'password') { - $input[$var['name']] = (new Password())->generate(); + $password = new Password(); + $input[$var['name']] = $password->generate(); continue; } } From b5ca4c116632951f3919a17ab14443431ed1a0c3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 15 Sep 2025 10:43:40 +0000 Subject: [PATCH 069/245] update suggestion --- src/Appwrite/Migration/Version/V17.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index e8d7129f8f..79e2a8377d 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -3,6 +3,7 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; +use Utopia\Auth\Proofs\Password; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document; @@ -269,7 +270,7 @@ class V17 extends Migration * Set hashOptions type */ $document->setAttribute('hashOptions', array_merge($document->getAttribute('hashOptions', []), [ - 'type' => $document->getAttribute('hash', 'argon2') + 'type' => $document->getAttribute('hash', (new Password())->getHash()->getName()) ])); break; } From cfd82d97095d438d09a336dc4b45b950c6f7121f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 15 Sep 2025 10:57:31 +0000 Subject: [PATCH 070/245] use newer syntax --- app/controllers/general.php | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 0bd56d6867..599a4af95a 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -1188,26 +1188,26 @@ App::error() $logLevel = $code >= 500 || $code == 0 ? 'error' : 'warning'; $logPrefix = $code >= 500 || $code == 0 ? '[Error]' : '[Warning]'; - Console::$logLevel($logPrefix . ' Timestamp: ' . date('c', time())); + Console::{$logLevel}($logPrefix . ' Timestamp: ' . date('c', time())); if ($route) { - Console::$logLevel($logPrefix . ' Status Code: ' . $code); - Console::$logLevel($logPrefix . ' URL: ' . $route->getMethod() . ' ' . $route->getPath()); + Console::{$logLevel}($logPrefix . ' Status Code: ' . $code); + Console::{$logLevel}($logPrefix . ' URL: ' . $route->getMethod() . ' ' . $route->getPath()); } - Console::$logLevel($logPrefix . ' Type: ' . get_class($error)); - Console::$logLevel($logPrefix . ' Message: ' . $message); - Console::$logLevel($logPrefix . ' File: ' . $file); - Console::$logLevel($logPrefix . ' Line: ' . $line); - Console::$logLevel($logPrefix . ' Trace:'); + Console::{$logLevel}($logPrefix . ' Type: ' . get_class($error)); + Console::{$logLevel}($logPrefix . ' Message: ' . $message); + Console::{$logLevel}($logPrefix . ' File: ' . $file); + Console::{$logLevel}($logPrefix . ' Line: ' . $line); + Console::{$logLevel}($logPrefix . ' Trace:'); foreach ($trace as $index => $entry) { $traceFile = $entry['file'] ?? 'unknown'; $traceLine = $entry['line'] ?? 0; $traceFunction = $entry['function'] ?? 'unknown'; $traceClass = $entry['class'] ?? ''; $traceType = $entry['type'] ?? ''; - Console::$logLevel(" #{$index} {$traceFile}({$traceLine}): {$traceClass}{$traceType}{$traceFunction}()"); + Console::{$logLevel}(" #{$index} {$traceFile}({$traceLine}): {$traceClass}{$traceType}{$traceFunction}()"); } - Console::$logLevel(''); + Console::{$logLevel}(''); } switch ($class) { From 679de3574f23cbd091405f5c99150332bac72b89 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 15 Sep 2025 19:11:33 +0000 Subject: [PATCH 071/245] Initial plan From fea3544d4bd7e89f4e3fd5dddccbc662a6ad5c43 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 15 Sep 2025 19:18:07 +0000 Subject: [PATCH 072/245] Implement dynamic specification defaults for Functions and Sites Co-authored-by: stnguyen90 <1477010+stnguyen90@users.noreply.github.com> --- .../Platform/Modules/Compute/Base.php | 44 +++++++++++++++++++ .../Functions/Http/Functions/Create.php | 2 +- .../Functions/Http/Functions/Update.php | 2 +- .../Modules/Sites/Http/Sites/Create.php | 2 +- .../Modules/Sites/Http/Sites/Update.php | 2 +- 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index 92805fbaf8..7115747fb8 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -5,6 +5,7 @@ namespace Appwrite\Platform\Modules\Compute; use Appwrite\Event\Build; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; +use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception\Duplicate; @@ -19,6 +20,49 @@ use Utopia\VCS\Exception\RepositoryNotFound; class Base extends Action { + /** + * Get default specification based on plan and available specifications. + * + * @param array $plan The billing plan configuration + * @return string The appropriate default specification + */ + protected function getDefaultSpecification(array $plan): string + { + $specifications = Config::getParam('specifications', []); + + if (empty($specifications)) { + return APP_COMPUTE_SPECIFICATION_DEFAULT; + } + + // If there's a plan with runtime specifications, use the highest one from the plan + if (!empty($plan) && array_key_exists('runtimeSpecifications', $plan) && !empty($plan['runtimeSpecifications'])) { + $planSpecifications = $plan['runtimeSpecifications']; + // Find the highest specification in the plan + foreach (array_reverse(array_keys($specifications)) as $specKey) { + if (in_array($specKey, $planSpecifications)) { + return $specKey; + } + } + } + + // If no plan or plan-based specification, use the highest available specification + $maxCpus = (float) System::getEnv('_APP_COMPUTE_CPUS', 0); + $maxMemory = (int) System::getEnv('_APP_COMPUTE_MEMORY', 0); + + $highestSpec = APP_COMPUTE_SPECIFICATION_DEFAULT; + foreach (array_reverse(array_keys($specifications)) as $specKey) { + $spec = $specifications[$specKey]; + $withinCpuLimit = empty($maxCpus) || $spec['cpus'] <= $maxCpus; + $withinMemoryLimit = empty($maxMemory) || $spec['memory'] <= $maxMemory; + + if ($withinCpuLimit && $withinMemoryLimit) { + $highestSpec = $specKey; + break; + } + } + + return $highestSpec; + } public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, Build $queueForBuilds, Document $template, GitHub $github, bool $activate, string $referenceType = 'branch', string $reference = ''): Document { $deploymentId = ID::unique(); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php index 21a74f9a81..8e3e0c3772 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php @@ -92,7 +92,7 @@ class Create extends Base ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the function.', true) ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the function? In silent mode, comments will not be made on commits and pull requests.', true) ->param('providerRootDirectory', '', new Text(128, 0), 'Path to function code in the linked repo.', true) - ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new Specification( + ->param('specification', fn (array $plan) => $this->getDefaultSpecification($plan), fn (array $plan) => new Specification( $plan, Config::getParam('specifications', []), System::getEnv('_APP_COMPUTE_CPUS', 0), diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php index aaff953af0..318c2a2032 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php @@ -89,7 +89,7 @@ class Update extends Base ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the function', true) ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the function? In silent mode, comments will not be made on commits and pull requests.', true) ->param('providerRootDirectory', '', new Text(128, 0), 'Path to function code in the linked repo.', true) - ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new Specification( + ->param('specification', fn (array $plan) => $this->getDefaultSpecification($plan), fn (array $plan) => new Specification( $plan, Config::getParam('specifications', []), System::getEnv('_APP_COMPUTE_CPUS', 0), diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php index 9be95441cb..a1633b8eba 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php @@ -78,7 +78,7 @@ class Create extends Base ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the site.', true) ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the site? In silent mode, comments will not be made on commits and pull requests.', true) ->param('providerRootDirectory', '', new Text(128, 0), 'Path to site code in the linked repo.', true) - ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new Specification( + ->param('specification', fn (array $plan) => $this->getDefaultSpecification($plan), fn (array $plan) => new Specification( $plan, Config::getParam('specifications', []), System::getEnv('_APP_COMPUTE_CPUS', 0), diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php index 80354d5067..72ec04a2a5 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php @@ -82,7 +82,7 @@ class Update extends Base ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the site.', true) ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the site? In silent mode, comments will not be made on commits and pull requests.', true) ->param('providerRootDirectory', '', new Text(128, 0), 'Path to site code in the linked repo.', true) - ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new Specification( + ->param('specification', fn (array $plan) => $this->getDefaultSpecification($plan), fn (array $plan) => new Specification( $plan, Config::getParam('specifications', []), System::getEnv('_APP_COMPUTE_CPUS', 0), From 62014fda26633c1c77c4129ccf8f28ade1d85b93 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Mon, 15 Sep 2025 21:29:44 +0000 Subject: [PATCH 073/245] fix: implement dynamic specification defaults for Functions and Sites Prior to this, self-hosted instances would defualt to the lowest specification. Now, if there is no plan (ie. on self-hosted), the highest specification is used. If there is a plan, the lowest specification available in the plan is used. --- .../Platform/Modules/Compute/Base.php | 43 +++++++------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index 7115747fb8..a538eb1497 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -5,6 +5,7 @@ namespace Appwrite\Platform\Modules\Compute; use Appwrite\Event\Build; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; +use Appwrite\Platform\Modules\Compute\Validator\Specification as SpecificationValidator; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; @@ -29,40 +30,28 @@ class Base extends Action protected function getDefaultSpecification(array $plan): string { $specifications = Config::getParam('specifications', []); - + if (empty($specifications)) { return APP_COMPUTE_SPECIFICATION_DEFAULT; } - // If there's a plan with runtime specifications, use the highest one from the plan - if (!empty($plan) && array_key_exists('runtimeSpecifications', $plan) && !empty($plan['runtimeSpecifications'])) { - $planSpecifications = $plan['runtimeSpecifications']; - // Find the highest specification in the plan - foreach (array_reverse(array_keys($specifications)) as $specKey) { - if (in_array($specKey, $planSpecifications)) { - return $specKey; - } - } + $specificationValidator = new SpecificationValidator( + $plan, + $specifications, + System::getEnv('_APP_COMPUTE_CPUS', 0), + System::getEnv('_APP_COMPUTE_MEMORY', 0) + ); + $allowedSpecifications = $specificationValidator->getAllowedSpecifications(); + + // If there is no plan use the highest specification + if (empty($plan)) { + return end($allowedSpecifications) ?? APP_COMPUTE_SPECIFICATION_DEFAULT; } - // If no plan or plan-based specification, use the highest available specification - $maxCpus = (float) System::getEnv('_APP_COMPUTE_CPUS', 0); - $maxMemory = (int) System::getEnv('_APP_COMPUTE_MEMORY', 0); - - $highestSpec = APP_COMPUTE_SPECIFICATION_DEFAULT; - foreach (array_reverse(array_keys($specifications)) as $specKey) { - $spec = $specifications[$specKey]; - $withinCpuLimit = empty($maxCpus) || $spec['cpus'] <= $maxCpus; - $withinMemoryLimit = empty($maxMemory) || $spec['memory'] <= $maxMemory; - - if ($withinCpuLimit && $withinMemoryLimit) { - $highestSpec = $specKey; - break; - } - } - - return $highestSpec; + // Otherwise, use the lowest specification available in the plan + return $allowedSpecifications[0] ?? APP_COMPUTE_SPECIFICATION_DEFAULT; } + public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, Build $queueForBuilds, Document $template, GitHub $github, bool $activate, string $referenceType = 'branch', string $reference = ''): Document { $deploymentId = ID::unique(); From 536cff9cc65a8836d0b5821c1bec231b44913845 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Mon, 15 Sep 2025 21:30:18 +0000 Subject: [PATCH 074/245] chore: add specification validator test --- .../Compute/Validator/SpecificationTest.php | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/unit/Platform/Modules/Compute/Validator/SpecificationTest.php diff --git a/tests/unit/Platform/Modules/Compute/Validator/SpecificationTest.php b/tests/unit/Platform/Modules/Compute/Validator/SpecificationTest.php new file mode 100644 index 0000000000..0505494e17 --- /dev/null +++ b/tests/unit/Platform/Modules/Compute/Validator/SpecificationTest.php @@ -0,0 +1,75 @@ +specifications = Config::getParam('specifications', []); + } + + public function testGetAllowedSpecificationsNoLimits(): void + { + $validator = new Specification( + plan: [], + specifications: $this->specifications, + maxCpus: 0, + maxMemory: 0 + ); + + $allowed = $validator->getAllowedSpecifications(); + $this->assertCount(count($this->specifications), $allowed); + $this->assertEquals( + $this->specifications[array_key_last($this->specifications)]['slug'], + $allowed[array_key_last($allowed)] + ); + } + + public function testGetAllowedSpecificationsWithMaxCpusAndMemory(): void + { + $validator = new Specification( + plan: [], + specifications: $this->specifications, + maxCpus: 2, + maxMemory: 2048 + ); + + $allowed = $validator->getAllowedSpecifications(); + $this->assertCount(4, $allowed); + $this->assertEquals( + SpecificationConstants::S_2VCPU_2GB, + $allowed[array_key_last($allowed)] + ); + } + + public function testGetAllowedSpecificationsWithPlanLimits(): void + { + $plan = [ + 'runtimeSpecifications' => [ + SpecificationConstants::S_05VCPU_512MB, + SpecificationConstants::S_1VCPU_512MB + ] + ]; + $validator = new Specification( + plan: $plan, + specifications: $this->specifications, + maxCpus: 0, + maxMemory: 0 + ); + + $allowed = $validator->getAllowedSpecifications(); + $this->assertCount(2, $allowed); + $this->assertContains(SpecificationConstants::S_05VCPU_512MB, $allowed); + $this->assertContains(SpecificationConstants::S_1VCPU_512MB, $allowed); + } +} From 1157d6fd100981f86fdb3a1ccdb84e7a2d3d834f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 16 Sep 2025 04:27:00 +0000 Subject: [PATCH 075/245] Fix re-hashing --- app/controllers/api/account.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 941a28efae..665d32ba11 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -961,10 +961,11 @@ App::post('/v1/account/sessions/email') // Re-hash if not using recommended algo if ($user->getAttribute('hash') !== $proofForPassword->getHash()->getName()) { + $proofForPasswordUpdated = new ProofsPassword(); $user - ->setAttribute('password', (new ProofsPassword())->hash($password)) - ->setAttribute('hash', $proofForPassword->getHash()->getName()) - ->setAttribute('hashOptions', $proofForPassword->getHash()->getOptions()); + ->setAttribute('password', $proofForPasswordUpdated->hash($password)) + ->setAttribute('hash', $proofForPasswordUpdated->getHash()->getName()) + ->setAttribute('hashOptions', $proofForPasswordUpdated->getHash()->getOptions()); $dbForProject->updateDocument('users', $user->getId(), $user); } From 73e7c98131dcbf56442cb0c69b128561f2809bea Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 16 Sep 2025 04:36:30 +0000 Subject: [PATCH 076/245] Fix token length update --- app/controllers/api/account.php | 5 ++--- app/controllers/api/users.php | 8 +++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 665d32ba11..a2c031451a 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1969,8 +1969,7 @@ App::post('/v1/account/tokens/magic-url') ->inject('queueForEvents') ->inject('queueForMails') ->inject('proofForPassword') - ->inject('proofForToken') - ->action(function (string $userId, string $email, string $url, bool $phrase, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Locale $locale, Event $queueForEvents, Mail $queueForMails, ProofsPassword $proofForPassword, ProofsToken $proofForToken) { + ->action(function (string $userId, string $email, string $url, bool $phrase, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Locale $locale, Event $queueForEvents, Mail $queueForMails, ProofsPassword $proofForPassword) { if (empty(System::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP disabled'); } @@ -2035,7 +2034,7 @@ App::post('/v1/account/tokens/magic-url') Authorization::skip(fn () => $dbForProject->createDocument('users', $user)); } - $proofForToken->setLength(TOKEN_LENGTH_MAGIC_URL); + $proofForToken = new ProofsToken(TOKEN_LENGTH_MAGIC_URL); $tokenSecret = $proofForToken->generate(); $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM)); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index b8874bf076..8b4967144d 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -2307,17 +2307,15 @@ App::post('/v1/users/:userId/tokens') ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('proofForToken') - ->action(function (string $userId, int $length, int $expire, Request $request, Response $response, Database $dbForProject, Event $queueForEvents, Token $proofForToken) { + ->action(function (string $userId, int $length, int $expire, Request $request, Response $response, Database $dbForProject, Event $queueForEvents) { $user = $dbForProject->getDocument('users', $userId); if ($user->isEmpty()) { throw new Exception(Exception::USER_NOT_FOUND); } - $secret = $proofForToken - ->setLength($length) - ->generate(); + $proofForToken = new Token($length); + $secret = $proofForToken->generate(); $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $expire)); $token = new Document([ From 33f7056e7a5f057f24e73fbca80cb70f179e9ff5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 16 Sep 2025 04:38:52 +0000 Subject: [PATCH 077/245] reest callback --- app/controllers/api/account.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index a2c031451a..cfe31d0b04 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1401,8 +1401,15 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') ->inject('proofForPassword') ->inject('proofForToken') ->action(function (string $provider, string $code, string $state, string $error, string $error_description, Request $request, Response $response, Document $project, array $platforms, Document $devKey, Document $user, Database $dbForProject, Reader $geodb, Event $queueForEvents, Store $store, ProofsPassword $proofForPassword, ProofsToken $proofForToken) use ($oauthDefaultSuccess) { - $protocol = $request->getProtocol(); - $callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId(); + $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; + $port = $request->getPort(); + $callbackBase = $protocol . '://' . $request->getHostname(); + if ($protocol === 'https' && $port !== '443') { + $callbackBase .= ':' . $port; + } elseif ($protocol === 'http' && $port !== '80') { + $callbackBase .= ':' . $port; + } + $callback = $callbackBase . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId(); $defaultState = ['success' => $project->getAttribute('url', ''), 'failure' => '']; $redirect = new Redirect($platforms); $appId = $project->getAttribute('oAuthProviders', [])[$provider . 'Appid'] ?? ''; From 74f181d7a834c4874f231c4c5dd727ae25ef1c8a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 17 Sep 2025 10:18:49 +0000 Subject: [PATCH 078/245] fix token length --- app/controllers/api/account.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index cfe31d0b04..64e90e9b03 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2488,8 +2488,7 @@ App::put('/v1/account/sessions/magic-url') ->inject('queueForEvents') ->inject('queueForMails') ->inject('store') - ->inject('proofForToken') - ->action($createSession); + ->action(fn ($userId, $secret, $request, $response, $user, $dbForProject, $project, $locale, $geodb, $queueForEvents, $queueForMails, $store) => $createSession($userId, $secret, $request, $response, $user, $dbForProject, $project, $locale, $geodb, $queueForEvents, $queueForMails, $store, new ProofsToken(TOKEN_LENGTH_MAGIC_URL))); App::put('/v1/account/sessions/phone') ->desc('Update phone session') From 3065f53d83bdfc0a926027cce01955661ac6f610 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 18 Sep 2025 01:14:14 +0000 Subject: [PATCH 079/245] Fix token hash --- app/controllers/api/account.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 64e90e9b03..e354a19b9a 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -40,6 +40,7 @@ use MaxMind\Db\Reader; use Utopia\Abuse\Abuse; use Utopia\App; use Utopia\Audit\Audit as EventAudit; +use Utopia\Auth\Hashes\Sha; use Utopia\Auth\Proofs\Code as ProofsCode; use Utopia\Auth\Proofs\Password as ProofsPassword; use Utopia\Auth\Proofs\Token as ProofsToken; @@ -2042,6 +2043,7 @@ App::post('/v1/account/tokens/magic-url') } $proofForToken = new ProofsToken(TOKEN_LENGTH_MAGIC_URL); + $proofForToken->setHash(new Sha()); $tokenSecret = $proofForToken->generate(); $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), TOKEN_EXPIRATION_CONFIRM)); From 4540362f42da32d29df59cb3498cd2b6aa7c75bd Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 18 Sep 2025 01:37:19 +0000 Subject: [PATCH 080/245] Fix: token hash magic url session --- app/controllers/api/account.php | 6 +++++- app/controllers/api/users.php | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index e354a19b9a..18e2aed277 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2490,7 +2490,11 @@ App::put('/v1/account/sessions/magic-url') ->inject('queueForEvents') ->inject('queueForMails') ->inject('store') - ->action(fn ($userId, $secret, $request, $response, $user, $dbForProject, $project, $locale, $geodb, $queueForEvents, $queueForMails, $store) => $createSession($userId, $secret, $request, $response, $user, $dbForProject, $project, $locale, $geodb, $queueForEvents, $queueForMails, $store, new ProofsToken(TOKEN_LENGTH_MAGIC_URL))); + ->action(function ($userId, $secret, $request, $response, $user, $dbForProject, $project, $locale, $geodb, $queueForEvents, $queueForMails, $store) use ($createSession) { + $proofForToken = new ProofsToken(TOKEN_LENGTH_MAGIC_URL); + $proofForToken->setHash(new Sha()); + $createSession($userId, $secret, $request, $response, $user, $dbForProject, $project, $locale, $geodb, $queueForEvents, $queueForMails, $store, $proofForToken); + }); App::put('/v1/account/sessions/phone') ->desc('Update phone session') diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 8b4967144d..536adcf128 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -2315,6 +2315,7 @@ App::post('/v1/users/:userId/tokens') } $proofForToken = new Token($length); + $proofForToken->setHash(new Sha()); $secret = $proofForToken->generate(); $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $expire)); From 5d5c1d8f43286c5baec29183ed77f2583c5d9bcb Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 22 Sep 2025 20:29:25 +0530 Subject: [PATCH 081/245] Fix Author URL in template deployments --- app/init/constants.php | 1 + src/Appwrite/Platform/Modules/Functions/Workers/Builds.php | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/init/constants.php b/app/init/constants.php index 28cf8a4052..6d723e9182 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -85,6 +85,7 @@ const APP_PLATFORM_CLIENT = 'client'; const APP_PLATFORM_CONSOLE = 'console'; const APP_VCS_GITHUB_USERNAME = 'Appwrite'; const APP_VCS_GITHUB_EMAIL = 'team@appwrite.io'; +const APP_VCS_GITHUB_URL = 'https://github.com/TeamAppwrite'; // Database Reconnect const DATABASE_RECONNECT_SLEEP = 2; diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 9547a752ef..c0d55aa7fb 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -467,11 +467,10 @@ class Builds extends Action } $providerCommitHash = \trim($stdout); - $authorUrl = "https://github.com/$cloneOwner"; $deployment->setAttribute('providerCommitHash', $providerCommitHash ?? ''); - $deployment->setAttribute('providerCommitAuthorUrl', $authorUrl); - $deployment->setAttribute('providerCommitAuthor', 'Appwrite'); + $deployment->setAttribute('providerCommitAuthorUrl', APP_VCS_GITHUB_URL); + $deployment->setAttribute('providerCommitAuthor', APP_VCS_GITHUB_USERNAME); $deployment->setAttribute('providerCommitMessage', "Create '" . $resource->getAttribute('name', '') . "' function"); $deployment->setAttribute('providerCommitUrl', "https://github.com/$cloneOwner/$cloneRepository/commit/$providerCommitHash"); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); From c8993d7f713a073ead1bfca7c54ee130863ea30c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:37:12 +1200 Subject: [PATCH 082/245] Add additional export params --- app/controllers/api/migrations.php | 101 +++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 27 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index d5bcfc2fd7..6e0ecd218d 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -339,12 +339,20 @@ App::post('/v1/migrations/csv/imports') ->inject('deviceForMigrations') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $fileId, string $resourceId, bool $internalFile, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForMigrations, Event $queueForEvents, Migration $queueForMigrations) { - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); - $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); - if ($internalFile && !$isPrivilegedUser) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } + ->action(function ( + string $bucketId, + string $fileId, + string $resourceId, + bool $internalFile, + Response $response, + Database $dbForProject, + Database $dbForPlatform, + Document $project, + Device $deviceForFiles, + Device $deviceForMigrations, + Event $queueForEvents, + Migration $queueForMigrations + ) { $bucket = Authorization::skip(function () use ($internalFile, $dbForPlatform, $dbForProject, $bucketId) { if ($internalFile) { return $dbForPlatform->getDocument('buckets', 'default'); @@ -352,7 +360,7 @@ App::post('/v1/migrations/csv/imports') return $dbForProject->getDocument('buckets', $bucketId); }); - if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { + if ($bucket->isEmpty()) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } @@ -366,7 +374,7 @@ App::post('/v1/migrations/csv/imports') throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); } - // no encryption, compression on files above 20MB. + // No encryption or compression on files above 20MB. $hasEncryption = !empty($file->getAttribute('openSSLCipher')); $compression = $file->getAttribute('algorithm', Compression::NONE); $hasCompression = $compression !== Compression::NONE; @@ -377,7 +385,6 @@ App::post('/v1/migrations/csv/imports') if ($hasEncryption || $hasCompression) { $source = $deviceForFiles->read($path); - // 1. decrypt if ($hasEncryption) { $source = OpenSSL::decrypt( $source, @@ -389,7 +396,6 @@ App::post('/v1/migrations/csv/imports') ); } - // 2. decompress if ($hasCompression) { switch ($compression) { case Compression::ZSTD: @@ -401,12 +407,12 @@ App::post('/v1/migrations/csv/imports') } } - // manual write after decryption and/or decompression - if (! $deviceForMigrations->write($newPath, $source, 'text/csv')) { - throw new \Exception("Unable to copy file"); + // Manual write after decryption and/or decompression + if (!$deviceForMigrations->write($newPath, $source, 'text/csv')) { + throw new \Exception('Unable to copy file'); } - } elseif (! $deviceForFiles->transfer($path, $newPath, $deviceForMigrations)) { - throw new \Exception("Unable to copy file"); + } elseif (!$deviceForFiles->transfer($path, $newPath, $deviceForMigrations)) { + throw new \Exception('Unable to copy file'); } $fileSize = $deviceForMigrations->getFileSize($newPath); @@ -461,34 +467,68 @@ App::post('/v1/migrations/csv/exports') ) ] )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.') ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') - ->param('columns', [], new ArrayList(new Text(255)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.') + ->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .csv extension.') + ->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true) + ->param('delimiter', ',', new Text(1), 'The character that separates each column value. Default is comma ",".', true) + ->param('enclosure', '"', new Text(1), 'The character that encloses each column value. Default is double quotes \'"\'.', true) + ->param('escape', '\\', new Text(1), 'The escape character for the enclosure character. Default is backslash "\\".', true) + ->param('header', true, new Boolean(), 'Whether to include the header row with column names. Default is true.', true) + ->param('notify', true, new Boolean(), 'Set to true to receive an email when the export is complete. Default is true.', true) + ->inject('user') ->inject('response') ->inject('dbForProject') ->inject('project') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $resourceId, array $columns, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Migration $queueForMigrations) { - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); - $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); - + ->action(function ( + string $resourceId, + string $bucketId, + string $filename, + array $columns, + string $delimiter, + string $enclosure, + string $escape, + bool $header, + bool $notify, + Document $user, + Response $response, + Database $dbForProject, + Document $project, + Event $queueForEvents, + Migration $queueForMigrations + ) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { + if ($bucket->isEmpty()) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } - $migrationId = ID::unique(); - $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); + [$databaseId, $collectionId] = \explode(':', $resourceId, 2); + if (empty($databaseId)) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + if (empty($collectionId)) { + throw new Exception(Exception::COLLECTION_NOT_FOUND); + } + + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + if ($database->isEmpty()) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId)); + if ($collection->isEmpty()) { + throw new Exception(Exception::COLLECTION_NOT_FOUND); + } $migration = $dbForProject->createDocument('migrations', new Document([ - '$id' => $migrationId, + '$id' => ID::unique(), 'status' => 'pending', 'stage' => 'init', 'source' => Appwrite::getName(), 'destination' => CSV::getName(), - 'resources' => $resources, + 'resources' => Transfer::extractServices([Transfer::GROUP_DATABASES]), 'resourceId' => $resourceId, 'resourceType' => Resource::TYPE_DATABASE, 'statusCounters' => '{}', @@ -496,7 +536,14 @@ App::post('/v1/migrations/csv/exports') 'errors' => [], 'options' => [ 'bucketId' => $bucketId, + 'filename' => $filename, 'columns' => $columns, + 'delimiter' => $delimiter, + 'enclosure' => $enclosure, + 'escape' => $escape, + 'header' => $header, + 'notify' => $notify, + 'userInternalId' => $user->getSequence(), ], ])); From 5e8951fbe02dce5ac3f7841b45ff2328ca868794 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:37:49 +1200 Subject: [PATCH 083/245] Save export to bucket on complete --- src/Appwrite/Platform/Workers/Migrations.php | 224 +++++++++++++------ 1 file changed, 153 insertions(+), 71 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index ce9b7e2881..bbdf537157 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -3,19 +3,23 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; +use Appwrite\Event\Mail; use Appwrite\Event\Realtime; -use Utopia\Migration\Destinations\CSV as DestinationCSV; +use Appwrite\Template\Template; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Locale\Locale; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; +use Utopia\Database\Query; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; +use Utopia\Migration\Destinations\CSV as DestinationCSV; use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite; @@ -27,6 +31,7 @@ use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Storage\Compression\Compression; use Utopia\Storage\Device; use Utopia\System\System; @@ -37,6 +42,7 @@ class Migrations extends Action protected Database $dbForPlatform; protected Device $deviceForMigrations; + protected Device $deviceForFiles; protected Document $project; @@ -71,16 +77,28 @@ class Migrations extends Action ->inject('logError') ->inject('queueForRealtime') ->inject('deviceForMigrations') + ->inject('deviceForFiles') + ->inject('queueForMails') ->callback($this->action(...)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForMigrations): void - { + public function action( + Message $message, + Document $project, + Database $dbForProject, + Database $dbForPlatform, + callable $logError, + Realtime $queueForRealtime, + Device $deviceForMigrations, + Device $deviceForFiles, + Mail $queueForMails, + ): void { $payload = $message->getPayload() ?? []; $this->deviceForMigrations = $deviceForMigrations; + $this->deviceForFiles = $deviceForFiles; if (empty($payload)) { throw new Exception('Missing payload'); @@ -105,7 +123,7 @@ class Migrations extends Action return; } - $this->processMigration($migration, $queueForRealtime); + $this->processMigration($migration, $queueForRealtime, $queueForMails); } /** @@ -175,6 +193,7 @@ class Migrations extends Action protected function processDestination(Document $migration, string $apiKey): Destination { $destination = $migration->getAttribute('destination'); + $options = $migration->getAttribute('options', []); return match ($destination) { DestinationAppwrite::getName() => new DestinationAppwrite( @@ -185,14 +204,32 @@ class Migrations extends Action Config::getParam('collections', [])['databases']['collections'], ), DestinationCSV::getName() => new DestinationCSV( - $this->deviceForMigrations, + $this->deviceForFiles, $migration->getAttribute('resourceId'), - $migration->getAttribute('options', [])['columns'] ?? [], + $options['bucketId'], + $options['filename'], + $options['columns'], + $options['delimiter'], + $options['enclosure'], + $options['escape'], + $options['header'], ), default => throw new \Exception('Invalid destination type'), }; } + /** + * Sanitize a filename to make it filesystem-safe + */ + protected function sanitizeFilename(string $filename): string + { + // Replace problematic characters with underscores + $sanitized = \preg_replace('/[:\/<>"|*?]/', '_', $filename); + $sanitized = \preg_replace('/[^\x20-\x7E]/', '_', $sanitized); + $sanitized = \trim($sanitized); + return empty($sanitized) ? 'export' : $sanitized; + } + /** * @throws Authorization * @throws Structure @@ -202,24 +239,18 @@ class Migrations extends Action */ protected function updateMigrationDocument(Document $migration, Document $project, Realtime $queueForRealtime): Document { - $errorMessages = []; - $clonedMigrationDocument = clone $migration; - - // we cannot use #sensitive because - // `errors` is nested which requires an override. - $errors = $clonedMigrationDocument->getAttribute('errors', []); + $messages = []; + $errors = $migration->getAttribute('errors', []); foreach ($errors as $error) { - $decoded = json_decode($error, true); - - if (is_array($decoded) && isset($decoded['trace'])) { + $decoded = \json_decode($error, true); + if (\is_array($decoded) && isset($decoded['trace'])) { unset($decoded['trace']); - $errorMessages[] = json_encode($decoded); + $messages[] = json_encode($decoded); } } - // set the errors back without trace - $clonedMigrationDocument->setAttribute('errors', $errorMessages); + $migration->setAttribute('errors', $messages); /** Trigger Realtime Events */ $queueForRealtime @@ -227,10 +258,14 @@ class Migrations extends Action ->setSubscribers(['console', $project->getId()]) ->setEvent('migrations.[migrationId].update') ->setParam('migrationId', $migration->getId()) - ->setPayload($clonedMigrationDocument->getArrayCopy(), ['options', 'credentials']) + ->setPayload($migration->getArrayCopy(), sensitive: ['options', 'credentials']) ->trigger(); - return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration); + return $this->dbForProject->updateDocument( + 'migrations', + $migration->getId(), + $migration + ); } /** @@ -285,11 +320,13 @@ class Migrations extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function processMigration(Document $migration, Realtime $queueForRealtime): void - { - $project = $this->project; - $projectDocument = $this->dbForPlatform->getDocument('projects', $project->getId()); - $tempAPIKey = $this->generateAPIKey($projectDocument); + protected function processMigration( + Document $migration, + Realtime $queueForRealtime, + Mail $queueForMails, + ): void { + $project = $this->dbForPlatform->getDocument('projects', $this->project->getId()); + $tempAPIKey = $this->generateAPIKey($project); $transfer = $source = $destination = null; @@ -299,17 +336,15 @@ class Migrations extends Action empty($migration->getAttribute('credentials', [])) ) { $credentials = $migration->getAttribute('credentials', []); - - $credentials['projectId'] = $credentials['projectId'] ?? $projectDocument->getId(); + $credentials['projectId'] = $credentials['projectId'] ?? $project->getId(); $credentials['endpoint'] = $credentials['endpoint'] ?? 'http://appwrite/v1'; $credentials['apiKey'] = $credentials['apiKey'] ?? $tempAPIKey; - $migration->setAttribute('credentials', $credentials); } $migration->setAttribute('stage', 'processing'); $migration->setAttribute('status', 'processing'); - $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); + $this->updateMigrationDocument($migration, $project, $queueForRealtime); $source = $this->processSource($migration); $destination = $this->processDestination($migration, $tempAPIKey); @@ -322,40 +357,44 @@ class Migrations extends Action /** Start Transfer */ if (empty($source->getErrors())) { $migration->setAttribute('stage', 'migrating'); - $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); + $this->updateMigrationDocument($migration, $project, $queueForRealtime); $transfer->run( $migration->getAttribute('resources'), - function () use ($migration, $transfer, $projectDocument, $queueForRealtime) { + function () use ($migration, $transfer, $project, $queueForRealtime) { $migration->setAttribute('resourceData', json_encode($transfer->getCache())); $migration->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); - $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); + $this->updateMigrationDocument($migration, $project, $queueForRealtime); }, $migration->getAttribute('resourceId'), $migration->getAttribute('resourceType') ); } - $destination->shutDown(); - $source->shutDown(); + // Debug logging for CSV exports before shutdown + if ($migration->getAttribute('destination') === DestinationCSV::getName()) { + $statusCounters = $transfer->getStatusCounters(); + Console::info('CSV export transfer completed. Status counters: ' . json_encode($statusCounters)); + Console::info('CSV export options: ' . json_encode($migration->getAttribute('options'))); + Console::info('CSV export errors: ' . json_encode($destination->getErrors())); + } + + $destination->shutdown(); + $source->shutdown(); $sourceErrors = $source->getErrors(); $destinationErrors = $destination->getErrors(); - if (! empty($sourceErrors) || ! empty($destinationErrors)) { + if (!empty($sourceErrors) || ! empty($destinationErrors)) { $migration->setAttribute('status', 'failed'); $migration->setAttribute('stage', 'finished'); - $errorMessages = []; - foreach ($sourceErrors as $error) { - $errorMessages[] = json_encode($error); - } - foreach ($destinationErrors as $error) { - $errorMessages[] = json_encode($error); + $errors = []; + foreach ([...$sourceErrors, ...$destinationErrors] as $error) { + $errors[] = \json_encode($error); } - $migration->setAttribute('errors', $errorMessages); - + $migration->setAttribute('errors', $errors); return; } @@ -382,57 +421,100 @@ class Migrations extends Action $sourceErrors = $source->getErrors(); $destinationErrors = $destination->getErrors(); - $errorMessages = []; - foreach ($sourceErrors as $error) { - $errorMessages[] = json_encode($error); - } - foreach ($destinationErrors as $error) { - $errorMessages[] = json_encode($error); + $errors = []; + foreach ([...$sourceErrors, ...$destinationErrors] as $error) { + $errors[] = \json_encode($error); } - $migration->setAttribute('errors', $errorMessages); + $migration->setAttribute('errors', $errors); } } finally { - $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); + $this->updateMigrationDocument($migration, $project, $queueForRealtime); if ($migration->getAttribute('status', '') === 'failed') { Console::error('Migration('.$migration->getSequence().':'.$migration->getId().') failed, Project('.$this->project->getSequence().':'.$this->project->getId().')'); - if ($destination) { - $destination->error(); + $sourceErrors = $source?->getErrors() ?? []; + $destinationErrors = $destination?->getErrors() ?? []; - foreach ($destination->getErrors() as $error) { - /** @var MigrationException $error */ - call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ + foreach ([...$sourceErrors, ...$destinationErrors] as $error) { + /** @var MigrationException $error */ + if ($error->getCode() === 0 || $error->getCode() >= 500) { + ($this->logError)($error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ 'migrationId' => $migration->getId(), 'source' => $migration->getAttribute('source') ?? '', 'destination' => $migration->getAttribute('destination') ?? '', 'resourceName' => $error->getResourceName(), - 'resourceGroup' => $error->getResourceGroup() + 'resourceGroup' => $error->getResourceGroup(), ]); } } - if ($source) { - $source->error(); - - foreach ($source->getErrors() as $error) { - /** @var MigrationException $error */ - call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ - 'migrationId' => $migration->getId(), - 'source' => $migration->getAttribute('source') ?? '', - 'destination' => $migration->getAttribute('destination') ?? '', - 'resourceName' => $error->getResourceName(), - 'resourceGroup' => $error->getResourceGroup() - ]); - } - } + $source?->error(); + $destination?->error(); } if ($migration->getAttribute('status', '') === 'completed') { $destination?->success(); $source?->success(); + + if ($migration->getAttribute('destination') === DestinationCSV::getName()) { + $this->handleCSVExportComplete($project, $migration, $queueForMails); + } } } } + + protected function handleCSVExportComplete(Document $project, Document $migration, Mail $queueForMails): void + { + $options = $migration->getAttribute('options', []); + $bucketId = $options['bucketId'] ?? null; + $filename = $options['filename'] ?? 'export.csv'; + $userInternalId = $options['userInternalId'] ?? ''; + $resourceId = $migration->getAttribute('resourceId'); + + // Save file to bucket + $bucket = $this->dbForProject->getDocument('buckets', $bucketId); + if ($bucket->isEmpty()) { + throw new \Exception("Bucket not found: $bucketId"); + } + + $path = $this->deviceForFiles->getPath($bucketId . '/' . $this->sanitizeFilename($filename) . '.csv'); + $size = $this->deviceForFiles->getFileSize($path); + $mime = $this->deviceForFiles->getFileMimeType($path); + $hash = $this->deviceForFiles->getFileHash($path); + $algorithm = Compression::NONE; + $fileId = \md5($resourceId); + + $this->dbForProject->createDocument('bucket_' . $bucket->getSequence(), new Document([ + '$id' => $fileId, + '$permissions' => [], + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $filename, + 'path' => $path, + 'signature' => $hash, + 'mimeType' => $mime, + 'sizeOriginal' => $size, + 'sizeActual' => $size, + 'algorithm' => $algorithm, + 'comment' => '', + 'chunksTotal' => 1, + 'chunksUploaded' => 1, + 'openSSLVersion' => null, + 'openSSLCipher' => null, + 'openSSLTag' => null, + 'openSSLIV' => null, + 'search' => \implode(' ', [$fileId, $filename]), + 'metadata' => ['content_type' => $mime] + ])); + + Console::info("Created file document in bucket: $fileId"); + + // No notification required, skip email sending + if (!($options['notify'] ?? false)) { + return; + } + + } } From 4970aa7426925e3c4d0567e5d8d3bc4c37a9ce33 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:38:12 +1200 Subject: [PATCH 084/245] Send email if notify on complete is set --- app/config/locale/translations/en.json | 8 +++ src/Appwrite/Platform/Workers/Migrations.php | 66 ++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/app/config/locale/translations/en.json b/app/config/locale/translations/en.json index e2ee20b2d7..c194819744 100644 --- a/app/config/locale/translations/en.json +++ b/app/config/locale/translations/en.json @@ -54,6 +54,14 @@ "emails.recovery.thanks": "Thanks,", "emails.recovery.buttonText": "Reset password", "emails.recovery.signature": "{{project}} team", + "emails.csvExport.subject": "Your CSV export is ready", + "emails.csvExport.preview": "Your data export has been completed successfully.", + "emails.csvExport.hello": "Hello {{user}},", + "emails.csvExport.body": "Your CSV export is ready for download. Click the link below to download your data export.", + "emails.csvExport.footer": "This download link will expire in 1 hour.", + "emails.csvExport.thanks": "Thanks,", + "emails.csvExport.buttonText": "Download CSV", + "emails.csvExport.signature": "{{project}} team", "emails.invitation.subject": "Invitation to {{team}} Team at {{project}}", "emails.invitation.preview": "{{owner}} invited you to join {{team}} at {{project}}", "emails.invitation.hello": "Hello {{user}},", diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index bbdf537157..ab8c7091cb 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -516,5 +516,71 @@ class Migrations extends Action return; } + $user = $this->dbForPlatform->findOne('users', [ + Query::equal('$sequence', [$userInternalId]) + ]); + + // Set up locale + $locale = new Locale(System::getEnv('_APP_LOCALE', 'en')); + $locale->setFallback(System::getEnv('_APP_LOCALE', 'en')); + + // Generate JWT + $expiry = (new \DateTime())->add(new \DateInterval('PT1H'))->format('U'); + $encoder = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', \intval($expiry), 0); + $jwt = $encoder->encode([ + 'bucketId' => $bucketId, + 'fileId' => $fileId, + 'projectId' => $project->getId(), + ]); + + // Generate download URL with JWT + $endpoint = System::getEnv('_APP_DOMAIN', ''); + $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS', 'disabled') === 'enabled' ? 'https' : 'http'; + $downloadUrl = "{$protocol}://{$endpoint}/v1/storage/buckets/{$bucketId}/files/{$fileId}/push?project={$project->getId()}&jwt={$jwt}"; + + // Get localized email content + $subject = $locale->getText('emails.csvExport.subject'); + $preview = $locale->getText('emails.csvExport.preview'); + $hello = $locale->getText('emails.csvExport.hello'); + $body = $locale->getText('emails.csvExport.body'); + $footer = $locale->getText('emails.csvExport.footer'); + $thanks = $locale->getText('emails.csvExport.thanks'); + $buttonText = $locale->getText('emails.csvExport.buttonText'); + $signature = $locale->getText('emails.csvExport.signature'); + + // Build email body using inner template + $message = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-inner-base.tpl'); + $message + ->setParam('{{body}}', $body, escapeHtml: false) + ->setParam('{{hello}}', $hello) + ->setParam('{{footer}}', $footer) + ->setParam('{{thanks}}', $thanks) + ->setParam('{{buttonText}}', $buttonText) + ->setParam('{{signature}}', $signature) + ->setParam('{{direction}}', $locale->getText('settings.direction')) + ->setParam('{{project}}', $project->getAttribute('name')) + ->setParam('{{user}}', $user->getAttribute('name', $user->getAttribute('email'))) + ->setParam('{{redirect}}', $downloadUrl); + + $emailBody = $message->render(); + + $emailVariables = [ + 'direction' => $locale->getText('settings.direction'), + 'project' => $project->getAttribute('name'), + 'user' => $user->getAttribute('name', $user->getAttribute('email')), + 'redirect' => $downloadUrl, + ]; + + $queueForMails + ->setSubject($subject) + ->setPreview($preview) + ->setBody($emailBody) + ->setBodyTemplate(__DIR__ . '/../../../../app/config/locale/templates/email-base-styled.tpl') + ->setVariables($emailVariables) + ->setName($user->getAttribute('name', $user->getAttribute('email'))) + ->setRecipient($user->getAttribute('email')) + ->trigger(); + + Console::info('CSV export notification email sent to ' . $user->getAttribute('email')); } } From 61d9db8c67500fadda6bf65805ae47168a5e265e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:38:35 +1200 Subject: [PATCH 085/245] Cover export + notify in tests --- .../Services/Migrations/MigrationsBase.php | 231 +++++++++++++++++- 1 file changed, 230 insertions(+), 1 deletion(-) diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 7a57b7f8f9..4d27cf2828 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -900,7 +900,7 @@ trait MigrationsBase /** * Import documents from a CSV file. */ - public function testCreateCsvMigration(): void + public function testCreateCSVImport(): void { // Make a database $response = $this->client->call(Client::METHOD_POST, '/databases', [ @@ -1194,4 +1194,233 @@ trait MigrationsBase 'x-appwrite-project' => $this->getProject()['$id'], ], $body); } + + /** + * Test CSV export with email notification + */ + public function testCreateCSVExport(): void + { + // Create a database + $database = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Export Database' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create a collection + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Test Export Collection', + 'permissions' => [] + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create a simple attribute like the basic test + $name = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'name', + 'size' => 255, + 'required' => true, + ]); + + $this->assertEquals(202, $name['headers']['status-code']); + + // Create a simple attribute like the basic test + $email = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'email', + 'size' => 255, + 'required' => false, + ]); + + $this->assertEquals(202, $email['headers']['status-code']); + + \sleep(3); + + // Create sample documents + for ($i = 1; $i <= 10; $i++) { + $doc = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'name' => 'Test User ' . $i, + 'email' => 'user' . $i . '@appwrite.io' + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code'], 'Failed to create document ' . $i); + } + + // Verify documents were created + $docs = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $docs['headers']['status-code']); + $this->assertEquals(10, $docs['body']['total'], 'Expected 10 documents but got ' . $docs['body']['total']); + + // Create a storage bucket for the export + $bucketIdUnique = ID::unique(); + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'bucketId' => $bucketIdUnique, + 'name' => 'Test Export Bucket', + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'fileSecurity' => false, + 'enabled' => true, + 'maximumFileSize' => 10485760, // 10MB + 'allowedFileExtensions' => ['csv'], + 'compression' => 'none', + 'encryption' => false, + 'antivirus' => false + ]); + + $this->assertEquals(201, $bucket['headers']['status-code']); + $bucketId = $bucket['body']['$id']; + + // Perform CSV export with notification enabled + $migration = $this->client->call(Client::METHOD_POST, '/migrations/csv/exports', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'bucketId' => $bucketId, + 'resourceId' => $databaseId . ':' . $collectionId, + 'filename' => 'test-export', + 'columns' => [], + 'delimiter' => ',', + 'enclosure' => '"', + 'escape' => '\\', + 'header' => true, + 'notify' => true + ]); + + $this->assertEquals(202, $migration['headers']['status-code']); + $this->assertNotEmpty($migration['body']['$id']); + $migrationId = $migration['body']['$id']; + + $this->assertEventually(function () use ($migrationId) { + $response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('finished', $response['body']['stage']); + $this->assertEquals('completed', $response['body']['status']); + $this->assertEquals('Appwrite', $response['body']['source']); + $this->assertEquals('CSV', $response['body']['destination']); + + return true; + }, 30000, 500); + + // Check that the file was created in the bucket + // File ID is MD5 of the resourceId (not the filename) + $fileId = \md5($databaseId . ':' . $collectionId); + + $file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $file['headers']['status-code']); + $this->assertEquals($fileId, $file['body']['$id']); + $this->assertEquals($bucketId, $file['body']['bucketId']); + $this->assertEquals('test-export', $file['body']['name']); + $this->assertEquals('text/csv', $file['body']['mimeType']); + $this->assertGreaterThan(0, $file['body']['sizeOriginal']); + + // Download and verify CSV content + $download = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download', \array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $download['headers']['status-code']); + + $csvContent = $download['body']; + $lines = explode("\n", trim($csvContent)); + $this->assertCount(11, $lines); + $this->assertStringContainsString('$id', $lines[0]); + $this->assertStringContainsString('$permissions', $lines[0]); + $this->assertStringContainsString('$createdAt', $lines[0]); + $this->assertStringContainsString('$updatedAt', $lines[0]); + $this->assertStringContainsString('name', $lines[0]); + $this->assertStringContainsString('email', $lines[0]); + + $this->assertStringContainsString('Test User 1', $lines[1]); + $this->assertStringContainsString('user1@appwrite.io', $lines[1]); + + // Check that email was sent with download link + $lastEmail = $this->getLastEmail(); + $this->assertNotEmpty($lastEmail); + $this->assertEquals('Your CSV export is ready', $lastEmail['subject']); + $this->assertStringContainsStringIgnoringCase('Your data export has been completed successfully', $lastEmail['text']); + + // Extract download URL from email HTML + \preg_match('/href="([^"]*\/storage\/buckets\/[^"]*\/push[^"]*)"/', $lastEmail['html'], $matches); + $this->assertNotEmpty($matches[1], 'Download URL not found in email'); + $downloadUrl = html_entity_decode($matches[1]); + + // Parse the URL to extract components + $components = \parse_url($downloadUrl); + $this->assertNotEmpty($components); + \parse_str($components['query'] ?? '', $queryParams); + $this->assertArrayHasKey('jwt', $queryParams, 'JWT not found in download URL'); + $this->assertNotEmpty($queryParams['jwt']); + + // Test download with JWT + $path = \str_replace('/v1', '', $components['path']); + $downloadWithJwt = $this->client->call(Client::METHOD_GET, $path . '?project=' . $queryParams['project'] . '&jwt=' . $queryParams['jwt']); + $this->assertEquals(200, $downloadWithJwt['headers']['status-code'], 'Failed to download file with JWT'); + $this->assertEquals($csvContent, $downloadWithJwt['body'], 'Downloaded content differs from original'); + + // Test that download without JWT fails + $downloadWithoutJwt = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download'); + $this->assertEquals(404, $downloadWithoutJwt['headers']['status-code'], 'File should not be downloadable without JWT'); + + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [ + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId, [ + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [ + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + } } From 223523f722f4a5b34ccc93e475d364334d1dd5ae Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:38:55 +1200 Subject: [PATCH 086/245] Mount uploads to migrations worker --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index da6362b4c4..0c187dd762 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -698,6 +698,7 @@ services: - appwrite volumes: - appwrite-imports:/storage/imports:rw + - appwrite-uploads:/storage/uploads:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests From 2494c9afb376e89311fd11e4904465b7c40f228c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:39:06 +1200 Subject: [PATCH 087/245] Update to release migrations --- composer.json | 2 +- composer.lock | 133 ++++++++++++++++++++++++++------------------------ 2 files changed, 69 insertions(+), 66 deletions(-) diff --git a/composer.json b/composer.json index 8a1e325f7f..6f89312223 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/locale": "0.8.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.18.*", - "utopia-php/migration": "dev-feat-csv-export as 1.0.0", + "utopia-php/migration": "1.2.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.8.*", diff --git a/composer.lock b/composer.lock index 73fb57f6f8..0fc58749f5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "954529a36566209d6687df9f41a0f2e6", + "content-hash": "5150254cf0d6aa361a31244b7f7d1eb7", "packages": [ { "name": "adhocore/jwt", @@ -1159,20 +1159,20 @@ }, { "name": "open-telemetry/api", - "version": "1.5.0", + "version": "1.6.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "7692075f486c14d8cfd37fba98a08a5667f089e5" + "reference": "ee17d937652eca06c2341b6fadc0f74c1c1a5af2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/7692075f486c14d8cfd37fba98a08a5667f089e5", - "reference": "7692075f486c14d8cfd37fba98a08a5667f089e5", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/ee17d937652eca06c2341b6fadc0f74c1c1a5af2", + "reference": "ee17d937652eca06c2341b6fadc0f74c1c1a5af2", "shasum": "" }, "require": { - "open-telemetry/context": "^1.0", + "open-telemetry/context": "^1.4", "php": "^8.1", "psr/log": "^1.1|^2.0|^3.0", "symfony/polyfill-php82": "^1.26" @@ -1225,20 +1225,20 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-08-07T23:07:38+00:00" + "time": "2025-09-19T00:05:49+00:00" }, { "name": "open-telemetry/context", - "version": "1.3.1", + "version": "1.4.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/context.git", - "reference": "438f71812242db3f196fb4c717c6f92cbc819be6" + "reference": "d4c4470b541ce72000d18c339cfee633e4c8e0cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/438f71812242db3f196fb4c717c6f92cbc819be6", - "reference": "438f71812242db3f196fb4c717c6f92cbc819be6", + "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/d4c4470b541ce72000d18c339cfee633e4c8e0cf", + "reference": "d4c4470b541ce72000d18c339cfee633e4c8e0cf", "shasum": "" }, "require": { @@ -1284,7 +1284,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-08-13T01:12:00+00:00" + "time": "2025-09-19T00:05:49+00:00" }, { "name": "open-telemetry/exporter-otlp", @@ -1415,23 +1415,23 @@ }, { "name": "open-telemetry/sdk", - "version": "1.7.1", + "version": "1.8.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "52690d4b37ae4f091af773eef3c238ed2bc0aa06" + "reference": "105c6e81e3d86150bd5704b00c7e4e165e957b89" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/52690d4b37ae4f091af773eef3c238ed2bc0aa06", - "reference": "52690d4b37ae4f091af773eef3c238ed2bc0aa06", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/105c6e81e3d86150bd5704b00c7e4e165e957b89", + "reference": "105c6e81e3d86150bd5704b00c7e4e165e957b89", "shasum": "" }, "require": { "ext-json": "*", "nyholm/psr7-server": "^1.1", - "open-telemetry/api": "^1.4", - "open-telemetry/context": "^1.0", + "open-telemetry/api": "^1.6", + "open-telemetry/context": "^1.4", "open-telemetry/sem-conv": "^1.0", "php": "^8.1", "php-http/discovery": "^1.14", @@ -1508,7 +1508,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-09-05T07:17:06+00:00" + "time": "2025-09-19T00:05:49+00:00" }, { "name": "open-telemetry/sem-conv", @@ -1569,16 +1569,16 @@ }, { "name": "paragonie/constant_time_encoding", - "version": "v2.7.0", + "version": "v2.8.1", "source": { "type": "git", "url": "https://github.com/paragonie/constant_time_encoding.git", - "reference": "52a0d99e69f56b9ec27ace92ba56897fe6993105" + "reference": "e6352b9f43318821f148c1e8c2d9e944aa9accb5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/52a0d99e69f56b9ec27ace92ba56897fe6993105", - "reference": "52a0d99e69f56b9ec27ace92ba56897fe6993105", + "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/e6352b9f43318821f148c1e8c2d9e944aa9accb5", + "reference": "e6352b9f43318821f148c1e8c2d9e944aa9accb5", "shasum": "" }, "require": { @@ -1632,7 +1632,7 @@ "issues": "https://github.com/paragonie/constant_time_encoding/issues", "source": "https://github.com/paragonie/constant_time_encoding" }, - "time": "2024-05-08T12:18:48+00:00" + "time": "2025-09-24T01:40:13+00:00" }, { "name": "paragonie/random_compat", @@ -4187,16 +4187,16 @@ }, { "name": "utopia-php/migration", - "version": "dev-feat-csv-export", + "version": "1.2.0", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "8435f1db0db4854ca27cb4c9cf275b905fcb3b41" + "reference": "42ff497c5231f5a727d1e229419ff1d2195d8093" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/8435f1db0db4854ca27cb4c9cf275b905fcb3b41", - "reference": "8435f1db0db4854ca27cb4c9cf275b905fcb3b41", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/42ff497c5231f5a727d1e229419ff1d2195d8093", + "reference": "42ff497c5231f5a727d1e229419ff1d2195d8093", "shasum": "" }, "require": { @@ -4237,9 +4237,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/feat-csv-export" + "source": "https://github.com/utopia-php/migration/tree/1.2.0" }, - "time": "2025-08-21T12:56:18+00:00" + "time": "2025-09-24T10:32:24+00:00" }, { "name": "utopia-php/orchestration", @@ -5004,16 +5004,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.3.5", + "version": "1.4.0", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "6fda9e58b37c9872c1a2a424e5467de8de1bc567" + "reference": "3583fa6fddb1d1a902b37ff2048527a5827fc008" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/6fda9e58b37c9872c1a2a424e5467de8de1bc567", - "reference": "6fda9e58b37c9872c1a2a424e5467de8de1bc567", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/3583fa6fddb1d1a902b37ff2048527a5827fc008", + "reference": "3583fa6fddb1d1a902b37ff2048527a5827fc008", "shasum": "" }, "require": { @@ -5049,9 +5049,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.3.5" + "source": "https://github.com/appwrite/sdk-generator/tree/1.4.0" }, - "time": "2025-09-15T04:19:40+00:00" + "time": "2025-09-23T02:27:10+00:00" }, { "name": "doctrine/annotations", @@ -5278,16 +5278,16 @@ }, { "name": "laravel/pint", - "version": "v1.25.0", + "version": "v1.25.1", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "595de38458c6b0ab4cae4bcc769c2e5c5d5b8e96" + "reference": "5016e263f95d97670d71b9a987bd8996ade6d8d9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/595de38458c6b0ab4cae4bcc769c2e5c5d5b8e96", - "reference": "595de38458c6b0ab4cae4bcc769c2e5c5d5b8e96", + "url": "https://api.github.com/repos/laravel/pint/zipball/5016e263f95d97670d71b9a987bd8996ade6d8d9", + "reference": "5016e263f95d97670d71b9a987bd8996ade6d8d9", "shasum": "" }, "require": { @@ -5340,7 +5340,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-09-17T01:36:44+00:00" + "time": "2025-09-19T02:57:12+00:00" }, { "name": "matthiasmullie/minify", @@ -6230,16 +6230,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.27", + "version": "9.6.29", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "0a9aa4440b6a9528cf360071502628d717af3e0a" + "reference": "9ecfec57835a5581bc888ea7e13b51eb55ab9dd3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/0a9aa4440b6a9528cf360071502628d717af3e0a", - "reference": "0a9aa4440b6a9528cf360071502628d717af3e0a", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/9ecfec57835a5581bc888ea7e13b51eb55ab9dd3", + "reference": "9ecfec57835a5581bc888ea7e13b51eb55ab9dd3", "shasum": "" }, "require": { @@ -6264,7 +6264,7 @@ "sebastian/comparator": "^4.0.9", "sebastian/diff": "^4.0.6", "sebastian/environment": "^5.1.5", - "sebastian/exporter": "^4.0.6", + "sebastian/exporter": "^4.0.8", "sebastian/global-state": "^5.0.8", "sebastian/object-enumerator": "^4.0.4", "sebastian/resource-operations": "^3.0.4", @@ -6313,7 +6313,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.27" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.29" }, "funding": [ { @@ -6337,7 +6337,7 @@ "type": "tidelift" } ], - "time": "2025-09-14T06:18:03+00:00" + "time": "2025-09-24T06:29:11+00:00" }, { "name": "psr/cache", @@ -6829,16 +6829,16 @@ }, { "name": "sebastian/exporter", - "version": "4.0.6", + "version": "4.0.8", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", - "reference": "78c00df8f170e02473b682df15bfcdacc3d32d72" + "reference": "14c6ba52f95a36c3d27c835d65efc7123c446e8c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/78c00df8f170e02473b682df15bfcdacc3d32d72", - "reference": "78c00df8f170e02473b682df15bfcdacc3d32d72", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/14c6ba52f95a36c3d27c835d65efc7123c446e8c", + "reference": "14c6ba52f95a36c3d27c835d65efc7123c446e8c", "shasum": "" }, "require": { @@ -6894,15 +6894,27 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/exporter/issues", - "source": "https://github.com/sebastianbergmann/exporter/tree/4.0.6" + "source": "https://github.com/sebastianbergmann/exporter/tree/4.0.8" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/exporter", + "type": "tidelift" } ], - "time": "2024-03-02T06:33:00+00:00" + "time": "2025-09-24T06:03:27+00:00" }, { "name": "sebastian/global-state", @@ -8504,18 +8516,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/migration", - "version": "dev-feat-csv-export", - "alias": "1.0.0", - "alias_normalized": "1.0.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/migration": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 16821a28fcad97655b94c48b02caa6970adc0ab9 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:53:13 +1200 Subject: [PATCH 088/245] Format --- app/controllers/api/migrations.php | 1 - src/Appwrite/Platform/Workers/Migrations.php | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 6e0ecd218d..c69185d0e8 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -1,6 +1,5 @@ Date: Fri, 10 Oct 2025 23:01:26 +0530 Subject: [PATCH 089/245] Dark mode styles for emails --- .../locale/templates/email-base-styled.tpl | 33 ++++++++++++++++++- app/config/locale/templates/email-base.tpl | 32 ++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/app/config/locale/templates/email-base-styled.tpl b/app/config/locale/templates/email-base-styled.tpl index 37ca630d43..c47d276ea8 100644 --- a/app/config/locale/templates/email-base-styled.tpl +++ b/app/config/locale/templates/email-base-styled.tpl @@ -2,6 +2,38 @@ + + +