diff --git a/.env b/.env index 9fbc425977..39cf1e1f3e 100644 --- a/.env +++ b/.env @@ -87,7 +87,7 @@ _APP_GRAPHQL_MAX_DEPTH=3 _APP_DOCKER_HUB_USERNAME= _APP_DOCKER_HUB_PASSWORD= _APP_VCS_GITHUB_APP_NAME= -_APP_VCS_GITHUB_PRIVATE_KEY="" +_APP_VCS_GITHUB_PRIVATE_KEY=disabled _APP_VCS_GITHUB_APP_ID= _APP_VCS_GITHUB_CLIENT_ID= _APP_VCS_GITHUB_CLIENT_SECRET= diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8fb3ac3518..842d61ff1c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,57 +4,119 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + IMAGE: appwrite-dev + CACHE_KEY: appwrite-dev-${{ github.event.pull_request.head.sha }} + on: [pull_request] + jobs: - tests: - name: Unit & E2E + setup: + name: Setup & Build Appwrite Image runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build Appwrite + uses: docker/build-push-action@v3 + with: + context: . + push: false + tags: ${{ env.IMAGE }} + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + outputs: type=docker,dest=/tmp/${{ env.IMAGE }}.tar + build-args: | + DEBUG=false + TESTING=true + VERSION=dev + + - name: Cache Docker Image + uses: actions/cache@v3 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + + unit_test: + name: Unit Test + runs-on: ubuntu-latest + needs: setup steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head. - fetch-depth: 2 - # Fetch submodules - submodules: recursive + - name: checkout + uses: actions/checkout@v2 - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} + - name: Load Cache + uses: actions/cache@v3 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true - # This is a separate action that sets up buildx runner - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + - name: Load and Start Appwrite + run: | + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose up -d + sleep 10 - - name: Build Appwrite - uses: docker/build-push-action@v3 - with: - context: . - push: false - tags: appwrite-dev - load: true - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - DEBUG=false - TESTING=true - VERSION=dev + - name: Doctor + run: docker compose exec -T appwrite doctor - - name: Start Appwrite - run: | - docker compose up -d - sleep 30 + - name: Environment Variables + run: docker compose exec -T appwrite vars - - name: Doctor - run: | - docker compose logs appwrite - docker compose exec -T appwrite doctor + - name: Run Unit Tests + run: docker compose exec appwrite test /usr/src/code/tests/unit - - name: Environment Variables - run: docker compose exec -T appwrite vars + e2e_test: + name: E2E Test + runs-on: ubuntu-latest + needs: setup + strategy: + fail-fast: false + matrix: + service: + [ + Account, + Avatars, + Console, + Databases, + Functions, + GraphQL, + Health, + Locale, + Projects, + Realtime, + Storage, + Teams, + Users, + Webhooks, + VCS, + ] - - name: Run Tests - run: docker compose exec -T appwrite test --debug \ No newline at end of file + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Load Cache + uses: actions/cache@v3 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + + - name: Load and Start Appwrite + run: | + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose up -d + sleep 10 + + - name: Run ${{matrix.service}} Tests + run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{matrix.service}} --debug \ No newline at end of file diff --git a/.gitpod.yml b/.gitpod.yml index 4c081dd275..d740c467cb 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -8,7 +8,7 @@ tasks: command: | docker run --rm --interactive --tty \ --volume $PWD:/app \ - composer update \ + composer install \ --ignore-platform-reqs \ --optimize-autoloader \ --no-plugins \ diff --git a/CHANGES.md b/CHANGES.md index 7e53698376..411b14ac46 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# Version 1.4.8 +# Version 1.4.10 ## Changes - New usage metrics collection flow [#7005](https://github.com/appwrite/appwrite/pull/7005) @@ -36,6 +36,38 @@ - renamed - executions to executionsTotal, builds to buildsTotal, requests to requestsTotal, storage to filesStorage, buckets to bucketsTotal, users to usersTotal, documents to documentsTotal, collections to collectionsTotal, databases to databasesTotal. +# Version 1.4.9 + +## Bug fixes + +* Fix 400 error on function domain execution in [#7059](https://github.com/appwrite/appwrite/pull/7059) + +# Version 1.4.8 + +## Notable changes + +* Fix certificate emails and add support for variables in email template subject in [#6495](https://github.com/appwrite/appwrite/)pull/6495 +* Bump console to version 3.2.5 in [#7027](https://github.com/appwrite/appwrite/pull/7027) +* Bump utopia database and storage versions in [#7002](https://github.com/appwrite/appwrite/pull/7002) + +## Bug fixes + +* Fixes cookie headers not being passed properly by router in [#7024](https://github.com/appwrite/appwrite/pull/7024) +* Fix permission problem in deletes worker in [#7013](https://github.com/appwrite/appwrite/pull/7013) + +## Miscellaneous + +* Improve error handling in the realtime service in [#6998](https://github.com/appwrite/appwrite/pull/6998) +* Update the error code for unsupported protocol in [#7006](https://github.com/appwrite/appwrite/pull/7006) +* Improve CI tests by executing them in parallel in [#6198](https://github.com/appwrite/appwrite/pull/6198) +* Update README.md to add links to orchestration tools in [#7011](https://github.com/appwrite/appwrite/pull/7011) +* Update gitpod setup to install instead of update dependencies in [#6938](https://github.com/appwrite/appwrite/pull/6938) +* Remove analytics from install script in [#7017](https://github.com/appwrite/appwrite/pull/7017) +* Improve database logging in [#7003](https://github.com/appwrite/appwrite/pull/7003) +* Add VCS tests in [#6894](https://github.com/appwrite/appwrite/pull/6894) +* Improve error messages in [#6487](https://github.com/appwrite/appwrite/pull/6487) +* Add command to delete orphaned projects in [#7015](https://github.com/appwrite/appwrite/pull/7015) + # Version 1.4.7 ## Fixes diff --git a/Dockerfile b/Dockerfile index 81762ed5f7..a60db04e93 100755 --- a/Dockerfile +++ b/Dockerfile @@ -99,7 +99,12 @@ RUN chmod +x /usr/local/bin/doctor && \ # Cloud Executabless RUN chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/volume-sync && \ - chmod +x /usr/local/bin/calc-tier-stats + chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ + chmod +x /usr/local/bin/patch-delete-project-collections && \ + chmod +x /usr/local/bin/delete-orphaned-projects && \ + chmod +x /usr/local/bin/clear-card-cache && \ + chmod +x /usr/local/bin/calc-users-stats + # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/README-CN.md b/README-CN.md index 1df799f7ef..4a2f9edd65 100644 --- a/README-CN.md +++ b/README-CN.md @@ -66,7 +66,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.4.7 + appwrite/appwrite:1.4.9 ``` ### Windows @@ -78,7 +78,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.4.7 + appwrite/appwrite:1.4.9 ``` #### PowerShell @@ -88,7 +88,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.4.7 + appwrite/appwrite:1.4.9 ``` 运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index b4980d0e61..9b4ba2bb58 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ Table of Contents: ## Installation -Appwrite is designed to run in a containerized environment. Running your server is as easy as running one command from your terminal. You can either run Appwrite on your localhost using docker-compose or on any other container orchestration tool, such as Kubernetes, Docker Swarm, or Rancher. +Appwrite is designed to run in a containerized environment. Running your server is as easy as running one command from your terminal. You can either run Appwrite on your localhost using docker-compose or on any other container orchestration tool, such as [Kubernetes](https://kubernetes.io/docs/home/), [Docker Swarm](https://docs.docker.com/engine/swarm/), or [Rancher](https://rancher.com/docs/). The easiest way to start running your Appwrite server is by running our docker-compose file. Before running the installation command, make sure you have [Docker](https://www.docker.com/products/docker-desktop) installed on your machine: @@ -76,7 +76,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.4.7 + appwrite/appwrite:1.4.9 ``` ### Windows @@ -88,7 +88,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.4.7 + appwrite/appwrite:1.4.9 ``` #### PowerShell @@ -98,7 +98,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.4.7 + appwrite/appwrite:1.4.9 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/cli.php b/app/cli.php index 992c85eca2..f2b9b454d6 100644 --- a/app/cli.php +++ b/app/cli.php @@ -61,7 +61,11 @@ CLI::setResource('dbForConsole', function ($pools, $cache) { ->getResource(); $dbForConsole = new Database($dbAdapter, $cache); - $dbForConsole->setNamespace('_console'); + + $dbForConsole + ->setNamespace('_console') + ->setMetadata('host', \gethostname()) + ->setMetadata('project', 'console'); // Ensure tables exist $collections = Config::getParam('collections', [])['console']; @@ -111,7 +115,10 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $databases[$databaseName] = $database; - $database->setNamespace('_' . $project->getInternalId()); + $database + ->setNamespace('_' . $project->getInternalId()) + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()); return $database; }; diff --git a/app/config/errors.php b/app/config/errors.php index 159c806175..ed1d97115c 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -86,7 +86,7 @@ return [ Exception::GENERAL_PROTOCOL_UNSUPPORTED => [ 'name' => Exception::GENERAL_PROTOCOL_UNSUPPORTED, 'description' => 'The request cannot be fulfilled with the current protocol. Please check the value of the _APP_OPTIONS_FORCE_HTTPS environment variable.', - 'code' => 500, + 'code' => 426, ], Exception::GENERAL_CODES_DISABLED => [ 'name' => Exception::GENERAL_CODES_DISABLED, @@ -355,7 +355,7 @@ return [ ], Exception::STORAGE_BUCKET_ALREADY_EXISTS => [ 'name' => Exception::STORAGE_BUCKET_ALREADY_EXISTS, - 'description' => 'A storage bucket with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.', + 'description' => 'A storage bucket with the requested ID already exists. Try again with a different ID or use ID.unique() to generate a unique ID.', 'code' => 409, ], Exception::STORAGE_BUCKET_NOT_FOUND => [ @@ -479,7 +479,7 @@ return [ ], Exception::COLLECTION_ALREADY_EXISTS => [ 'name' => Exception::COLLECTION_ALREADY_EXISTS, - 'description' => 'A collection with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.', + 'description' => 'A collection with the requested ID already exists. Try again with a different ID or use ID.unique() to generate a unique ID.', 'code' => 409, ], Exception::COLLECTION_LIMIT_EXCEEDED => [ @@ -511,7 +511,7 @@ return [ ], Exception::DOCUMENT_ALREADY_EXISTS => [ 'name' => Exception::DOCUMENT_ALREADY_EXISTS, - 'description' => 'Document with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.', + 'description' => 'Document with the requested ID already exists. Try again with a different ID or use ID.unique() to generate a unique ID.', 'code' => 409, ], Exception::DOCUMENT_UPDATE_CONFLICT => [ @@ -553,7 +553,7 @@ return [ ], Exception::ATTRIBUTE_ALREADY_EXISTS => [ 'name' => Exception::ATTRIBUTE_ALREADY_EXISTS, - 'description' => 'Attribute with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.', + 'description' => 'Attribute with the requested key already exists. Attribute keys must be unique, try again with a different key.', 'code' => 409, ], Exception::ATTRIBUTE_LIMIT_EXCEEDED => [ @@ -585,7 +585,7 @@ return [ ], Exception::INDEX_ALREADY_EXISTS => [ 'name' => Exception::INDEX_ALREADY_EXISTS, - 'description' => 'Index with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.', + 'description' => 'Index with the requested key already exists. Try again with a different key.', 'code' => 409, ], Exception::INDEX_INVALID => [ @@ -602,7 +602,7 @@ return [ ], Exception::PROJECT_ALREADY_EXISTS => [ 'name' => Exception::PROJECT_ALREADY_EXISTS, - 'description' => 'Project with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.', + 'description' => 'Project with the requested ID already exists. Try again with a different ID or use ID.unique() to generate a unique ID.', 'code' => 409, ], Exception::PROJECT_UNKNOWN => [ @@ -732,4 +732,21 @@ return [ 'description' => 'Migration is already in progress. You can check the status of the migration in your Appwrite Console\'s "Settings" > "Migrations".', 'code' => 409, ], + + /** Realtime */ + Exception::REALTIME_MESSAGE_FORMAT_INVALID => [ + 'name' => Exception::REALTIME_MESSAGE_FORMAT_INVALID, + 'description' => 'Message format is not valid.', + 'code' => 1003, + ], + Exception::REALTIME_POLICY_VIOLATION => [ + 'name' => Exception::REALTIME_POLICY_VIOLATION, + 'description' => 'Policy violation.', + 'code' => 1008, + ], + Exception::REALTIME_TOO_MANY_MESSAGES => [ + 'name' => Exception::REALTIME_TOO_MANY_MESSAGES, + 'description' => 'Too many messages.', + 'code' => 1013, + ], ]; diff --git a/app/console b/app/console index e965738987..9810ce8581 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit e9657389879c8d76a9b3a0d3486c1d86f43c3bb9 +Subproject commit 9810ce85812ca26c95b7d35196848c92e8ba813d diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index fdd5de47df..534bb66a8c 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -993,7 +993,12 @@ App::post('/v1/account/sessions/magic-url') $customTemplate = $project->getAttribute('templates', [])['email.magicSession-' . $locale->default] ?? []; $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl'); - $message->setParam('{{body}}', $body); + $message + ->setParam('{{body}}', $body) + ->setParam('{{hello}}', $locale->getText("emails.magicSession.hello")) + ->setParam('{{footer}}', $locale->getText("emails.magicSession.footer")) + ->setParam('{{thanks}}', $locale->getText("emails.magicSession.thanks")) + ->setParam('{{signature}}', $locale->getText("emails.magicSession.signature")); $body = $message->render(); $smtp = $project->getAttribute('smtp', []); @@ -1043,16 +1048,7 @@ App::post('/v1/account/sessions/magic-url') } $emailVariables = [ - 'subject' => $subject, - 'hello' => $locale->getText("emails.magicSession.hello"), - 'body' => $body, - 'footer' => $locale->getText("emails.magicSession.footer"), - 'thanks' => $locale->getText("emails.magicSession.thanks"), - 'signature' => $locale->getText("emails.magicSession.signature"), 'direction' => $locale->getText('settings.direction'), - 'bg-body' => '#f7f7f7', - 'bg-content' => '#ffffff', - 'text-content' => '#000000', /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */ 'user' => '', 'team' => '', @@ -2431,7 +2427,12 @@ App::post('/v1/account/recovery') $customTemplate = $project->getAttribute('templates', [])['email.recovery-' . $locale->default] ?? []; $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl'); - $message->setParam('{{body}}', $body); + $message + ->setParam('{{body}}', $body) + ->setParam('{{hello}}', $locale->getText("emails.recovery.hello")) + ->setParam('{{footer}}', $locale->getText("emails.recovery.footer")) + ->setParam('{{thanks}}', $locale->getText("emails.recovery.thanks")) + ->setParam('{{signature}}', $locale->getText("emails.recovery.signature")); $body = $message->render(); $smtp = $project->getAttribute('smtp', []); @@ -2481,16 +2482,7 @@ App::post('/v1/account/recovery') } $emailVariables = [ - 'subject' => $subject, - 'hello' => $locale->getText("emails.recovery.hello"), - 'body' => $body, - 'footer' => $locale->getText("emails.recovery.footer"), - 'thanks' => $locale->getText("emails.recovery.thanks"), - 'signature' => $locale->getText("emails.recovery.signature"), 'direction' => $locale->getText('settings.direction'), - 'bg-body' => '#f7f7f7', - 'bg-content' => '#ffffff', - 'text-content' => '#000000', /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */ 'user' => $profile->getAttribute('name'), 'team' => '', @@ -2498,7 +2490,6 @@ App::post('/v1/account/recovery') 'redirect' => $url ]; - $queueForMails ->setRecipient($profile->getAttribute('email', '')) ->setName($profile->getAttribute('name')) @@ -2681,7 +2672,12 @@ App::post('/v1/account/verification') $customTemplate = $project->getAttribute('templates', [])['email.verification-' . $locale->default] ?? []; $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl'); - $message->setParam('{{body}}', $body); + $message + ->setParam('{{body}}', $body) + ->setParam('{{hello}}', $locale->getText("emails.verification.hello")) + ->setParam('{{footer}}', $locale->getText("emails.verification.footer")) + ->setParam('{{thanks}}', $locale->getText("emails.verification.thanks")) + ->setParam('{{signature}}', $locale->getText("emails.verification.signature")); $body = $message->render(); $smtp = $project->getAttribute('smtp', []); @@ -2731,16 +2727,7 @@ App::post('/v1/account/verification') } $emailVariables = [ - 'subject' => $subject, - 'hello' => $locale->getText("emails.verification.hello"), - 'body' => $body, - 'footer' => $locale->getText("emails.verification.footer"), - 'thanks' => $locale->getText("emails.verification.thanks"), - 'signature' => $locale->getText("emails.verification.signature"), 'direction' => $locale->getText('settings.direction'), - 'bg-body' => '#f7f7f7', - 'bg-content' => '#ffffff', - 'text-content' => '#000000', /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */ 'user' => $user->getAttribute('name'), 'team' => '', diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 14cefa79e0..786a419449 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -46,6 +46,7 @@ use Utopia\Validator\Boolean; use Utopia\Database\Exception\Duplicate as DuplicateException; use MaxMind\Db\Reader; use Utopia\VCS\Adapter\Git\GitHub; +use Utopia\VCS\Exception\RepositoryNotFound; include_once __DIR__ . '/../shared/api.php'; @@ -58,7 +59,14 @@ $redeployVcs = function (Request $request, Document $function, Document $project $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); $owner = $github->getOwnerName($providerInstallationId); $providerRepositoryId = $function->getAttribute('providerRepositoryId', ''); - $repositoryName = $github->getRepositoryName($providerRepositoryId); + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } $providerBranch = $function->getAttribute('providerBranch', 'main'); $authorUrl = "https://github.com/$owner"; $repositoryUrl = "https://github.com/$owner/$repositoryName"; @@ -287,9 +295,8 @@ App::post('/v1/functions') $ruleModel = new Rule(); $ruleCreate = $queueForEvents - ->setClass(Event::WEBHOOK_CLASS_NAME) - ->setQueue(Event::WEBHOOK_QUEUE_NAME) - ; + ->setClass(Event::WEBHOOK_CLASS_NAME) + ->setQueue(Event::WEBHOOK_QUEUE_NAME); $ruleCreate ->setProject($project) @@ -848,8 +855,7 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId/download') ->setContentType('application/gzip') ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache ->addHeader('X-Peak', \memory_get_peak_usage()) - ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '.tar.gz"') - ; + ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '.tar.gz"'); $size = $deviceFunctions->getFileSize($path); $rangeHeader = $request->getHeader('range'); diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 8ca1371488..b1195cd878 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -352,7 +352,7 @@ App::get('/v1/health/queue/webhooks') ->inject('response') ->action(function (Connection $queue, Response $response) { $client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); + $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/logs') @@ -370,7 +370,7 @@ App::get('/v1/health/queue/logs') ->inject('response') ->action(function (Connection $queue, Response $response) { $client = new Client(Event::AUDITS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); + $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/certificates') @@ -388,7 +388,7 @@ App::get('/v1/health/queue/certificates') ->inject('response') ->action(function (Connection $queue, Response $response) { $client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); + $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/builds') @@ -406,7 +406,7 @@ App::get('/v1/health/queue/builds') ->inject('response') ->action(function (Connection $queue, Response $response) { $client = new Client(Event::BUILDS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); + $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/databases') @@ -425,7 +425,7 @@ App::get('/v1/health/queue/databases') ->inject('response') ->action(function (string $name, Connection $queue, Response $response) { $client = new Client($name, $queue); - $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); + $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/deletes') @@ -443,7 +443,7 @@ App::get('/v1/health/queue/deletes') ->inject('response') ->action(function (Connection $queue, Response $response) { $client = new Client(Event::DELETE_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); + $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/mails') @@ -461,7 +461,7 @@ App::get('/v1/health/queue/mails') ->inject('response') ->action(function (Connection $queue, Response $response) { $client = new Client(Event::MAILS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); + $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/messaging') @@ -479,7 +479,7 @@ App::get('/v1/health/queue/messaging') ->inject('response') ->action(function (Connection $queue, Response $response) { $client = new Client(Event::MESSAGING_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); + $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/migrations') @@ -497,7 +497,7 @@ App::get('/v1/health/queue/migrations') ->inject('response') ->action(function (Connection $queue, Response $response) { $client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); + $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/functions') @@ -515,7 +515,7 @@ App::get('/v1/health/queue/functions') ->inject('response') ->action(function (Connection $queue, Response $response) { $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); + $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/storage/local') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 200c7372f2..c59c5189da 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -1604,7 +1604,6 @@ App::get('/v1/projects/:projectId/templates/email/:type/:locale') $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl'); $message ->setParam('{{hello}}', $localeObj->getText("emails.{$type}.hello")) - ->setParam('{{user}}', '') ->setParam('{{footer}}', $localeObj->getText("emails.{$type}.footer")) ->setParam('{{body}}', $localeObj->getText('emails.' . $type . '.body')) ->setParam('{{thanks}}', $localeObj->getText("emails.{$type}.thanks")) diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 0a887908bb..2ee351f469 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -555,7 +555,12 @@ App::post('/v1/teams/:teamId/memberships') $customTemplate = $project->getAttribute('templates', [])['email.invitation-' . $locale->default] ?? []; $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl'); - $message->setParam('{{body}}', $body); + $message + ->setParam('{{body}}', $body) + ->setParam('{{hello}}', $locale->getText("emails.invitation.hello")) + ->setParam('{{footer}}', $locale->getText("emails.invitation.footer")) + ->setParam('{{thanks}}', $locale->getText("emails.invitation.thanks")) + ->setParam('{{signature}}', $locale->getText("emails.invitation.signature")); $body = $message->render(); $smtp = $project->getAttribute('smtp', []); @@ -606,16 +611,7 @@ App::post('/v1/teams/:teamId/memberships') $emailVariables = [ 'owner' => $user->getAttribute('name'), - 'subject' => $subject, - 'hello' => $locale->getText("emails.invitation.hello"), - 'body' => $body, - 'footer' => $locale->getText("emails.invitation.footer"), - 'thanks' => $locale->getText("emails.invitation.thanks"), - 'signature' => $locale->getText("emails.invitation.signature"), 'direction' => $locale->getText('settings.direction'), - 'bg-body' => '#f7f7f7', - 'bg-content' => '#ffffff', - 'text-content' => '#000000', /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */ 'user' => $user->getAttribute('name'), 'team' => $team->getAttribute('name'), diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 55602136fc..8b61580b76 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -22,7 +22,6 @@ use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; -use Utopia\Database\Validator\UID; use Utopia\Detector\Adapter\Bun; use Utopia\Detector\Adapter\CPP; use Utopia\Detector\Adapter\Dart; @@ -36,6 +35,7 @@ use Utopia\Detector\Adapter\Ruby; use Utopia\Detector\Adapter\Swift; use Utopia\Detector\Detector; use Utopia\Validator\Boolean; +use Utopia\VCS\Exception\RepositoryNotFound; use function Swoole\Coroutine\batch; @@ -66,7 +66,14 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId } $owner = $github->getOwnerName($providerInstallationId) ?? ''; - $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } if (empty($repositoryName)) { throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); @@ -154,7 +161,14 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId $message = 'Authorization required for external contributor.'; $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); - $repositoryName = $github->getRepositoryName($providerRepositoryId); + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } $owner = $github->getOwnerName($providerInstallationId); $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'failure', $message, $authorizeUrl, $name); continue; @@ -206,7 +220,14 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId $message = 'Starting...'; $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); - $repositoryName = $github->getRepositoryName($providerRepositoryId); + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } $owner = $github->getOwnerName($providerInstallationId); $providerTargetUrl = $request->getProtocol() . '://' . $request->getHostname() . "/console/project-$projectId/functions/function-$functionId"; @@ -273,7 +294,7 @@ App::get('/v1/vcs/github/callback') ->label('scope', 'public') ->label('error', __DIR__ . '/../../views/general/error.phtml') ->param('installation_id', '', new Text(256, 0), 'GitHub installation ID', true) - ->param('setup_action', '', new Text(256, 0), 'GitHub setup actuon type', true) + ->param('setup_action', '', new Text(256, 0), 'GitHub setup action type', true) ->param('state', '', new Text(2048), 'GitHub state. Contains info sent when starting authorization flow.', true) ->param('code', '', new Text(2048, 0), 'OAuth2 code. This is a temporary code that the will be later exchanged for an access token.', true) ->inject('gitHub') @@ -458,9 +479,12 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories/:pr $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); $owner = $github->getOwnerName($providerInstallationId); - $repositoryName = $github->getRepositoryName($providerRepositoryId); - - if (empty($repositoryName)) { + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); } @@ -720,9 +744,12 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:pro $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); $owner = $github->getOwnerName($providerInstallationId) ?? ''; - $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; - - if (empty($repositoryName)) { + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); } @@ -766,9 +793,12 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:pro $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); $owner = $github->getOwnerName($providerInstallationId) ?? ''; - $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; - - if (empty($repositoryName)) { + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); } @@ -1090,7 +1120,14 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor $providerRepositoryId = $repository->getAttribute('providerRepositoryId'); $owner = $github->getOwnerName($providerInstallationId); - $repositoryName = $github->getRepositoryName($providerRepositoryId); + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } $pullRequestResponse = $github->getPullRequest($owner, $repositoryName, $providerPullRequestId); $providerBranch = \explode(':', $pullRequestResponse['head']['label'])[1] ?? ''; diff --git a/app/controllers/general.php b/app/controllers/general.php index 0a67148576..cf383b6710 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -117,12 +117,14 @@ function router(App $utopia, Database $dbForConsole, SwooleRequest $swooleReques $path .= '?' . $query; } + $requestHeaders = $request->getHeaders(); + $body = \json_encode([ 'async' => false, 'body' => $swooleRequest->getContent() ?? '', 'method' => $swooleRequest->server['request_method'], 'path' => $path, - 'headers' => $swooleRequest->header + 'headers' => $requestHeaders ]); $headers = [ diff --git a/app/controllers/mock.php b/app/controllers/mock.php index 593063ae84..bc4f463b5c 100644 --- a/app/controllers/mock.php +++ b/app/controllers/mock.php @@ -17,6 +17,9 @@ use Utopia\Validator\WhiteList; use Utopia\Database\Helpers\ID; use Utopia\Database\Validator\UID; use Utopia\Validator\Nullable; +use Utopia\VCS\Adapter\Git\GitHub; +use Utopia\Database\Helpers\Permission; +use Utopia\Database\Helpers\Role; App::get('/v1/mock/tests/foo') ->desc('Get Foo') @@ -248,8 +251,7 @@ App::get('/v1/mock/tests/general/download') ->addHeader('Content-Disposition', 'attachment; filename="test.txt"') ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache ->addHeader('X-Peak', \memory_get_peak_usage()) - ->send("GET:/v1/mock/tests/general/download:passed") - ; + ->send("GET:/v1/mock/tests/general/download:passed"); }); App::post('/v1/mock/tests/general/upload') @@ -331,7 +333,7 @@ App::post('/v1/mock/tests/general/upload') } if ($file['size'] !== 38756) { - throw new Exception(Exception::GENERAL_MOCK, 'Wrong file size'); + throw new Exception(Exception::GENERAL_MOCK, 'Wrong file size'); } if (\md5(\file_get_contents($file['tmp_name'])) !== 'd80e7e6999a3eb2ae0d631a96fe135a4') { @@ -506,8 +508,7 @@ App::get('/v1/mock/tests/general/502-error') $response ->setStatusCode(502) - ->text('This is a text error') - ; + ->text('This is a text error'); }); App::get('/v1/mock/tests/general/oauth2') @@ -646,6 +647,66 @@ App::patch('/v1/mock/functions-v2') $response->noContent(); }); +App::get('/v1/mock/github/callback') + ->desc('Create installation document using GitHub installation id') + ->groups(['mock', 'api', 'vcs']) + ->label('scope', 'public') + ->label('docs', false) + ->param('providerInstallationId', '', new UID(), 'GitHub installation ID') + ->param('projectId', '', new UID(), 'Project ID of the project where app is to be installed') + ->inject('gitHub') + ->inject('project') + ->inject('response') + ->inject('dbForConsole') + ->action(function (string $providerInstallationId, string $projectId, GitHub $github, Document $project, Response $response, Database $dbForConsole) { + $isDevelopment = App::getEnv('_APP_ENV', 'development') === 'development'; + + if (!$isDevelopment) { + throw new Exception(Exception::GENERAL_NOT_IMPLEMENTED); + } + + $project = $dbForConsole->getDocument('projects', $projectId); + + if ($project->isEmpty()) { + $error = 'Project with the ID from state could not be found.'; + throw new Exception(Exception::PROJECT_NOT_FOUND, $error); + } + + if (!empty($providerInstallationId)) { + $privateKey = App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'); + $githubAppId = App::getEnv('_APP_VCS_GITHUB_APP_ID'); + $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); + $owner = $github->getOwnerName($providerInstallationId) ?? ''; + + $projectInternalId = $project->getInternalId(); + + $teamId = $project->getAttribute('teamId', ''); + + $installation = new Document([ + '$id' => ID::unique(), + '$permissions' => [ + Permission::read(Role::team(ID::custom($teamId))), + Permission::update(Role::team(ID::custom($teamId), 'owner')), + Permission::update(Role::team(ID::custom($teamId), 'developer')), + Permission::delete(Role::team(ID::custom($teamId), 'owner')), + Permission::delete(Role::team(ID::custom($teamId), 'developer')), + ], + 'providerInstallationId' => $providerInstallationId, + 'projectId' => $projectId, + 'projectInternalId' => $projectInternalId, + 'provider' => 'github', + 'organization' => $owner, + 'personal' => false + ]); + + $installation = $dbForConsole->createDocument('installations', $installation); + } + + $response->json([ + 'installationId' => $installation->getId(), + ]); + }); + App::shutdown() ->groups(['mock']) ->inject('utopia') diff --git a/app/init.php b/app/init.php index 0d12c57be4..677af3f9a9 100644 --- a/app/init.php +++ b/app/init.php @@ -109,8 +109,8 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 514; -const APP_VERSION_STABLE = '1.4.7'; +const APP_CACHE_BUSTER = 515; +const APP_VERSION_STABLE = '1.4.9'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; @@ -1025,6 +1025,9 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons } } + $dbForProject->setMetadata('user', $user->getId()); + $dbForConsole->setMetadata('user', $user->getId()); + return $user; }, ['mode', 'project', 'console', 'request', 'response', 'dbForProject', 'dbForConsole']); @@ -1098,6 +1101,8 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, $database ->setNamespace('_' . $project->getInternalId()) + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); return $database; @@ -1114,6 +1119,8 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { $database ->setNamespace('_console') + ->setMetadata('host', \gethostname()) + ->setMetadata('project', 'console') ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); return $database; @@ -1134,6 +1141,8 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $database ->setNamespace('_' . $project->getInternalId()) + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); return $database; @@ -1150,6 +1159,8 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $database ->setNamespace('_' . $project->getInternalId()) + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS); return $database; diff --git a/app/realtime.php b/app/realtime.php index 25b0532b42..f7fc7070a4 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1,8 +1,10 @@ setNamespace('_console'); + $database + ->setNamespace('_console') + ->setMetadata('host', \gethostname()) + ->setMetadata('project', '_console'); return $database; } @@ -70,7 +77,11 @@ function getProjectDB(Document $project): Database ; $database = new Database($dbAdapter, getCache()); - $database->setNamespace('_' . $project->getInternalId()); + + $database + ->setNamespace('_' . $project->getInternalId()) + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()); return $database; } @@ -122,12 +133,12 @@ $server = new Server($adapter); $logError = function (Throwable $error, string $action) use ($register) { $logger = $register->get('logger'); - if ($logger) { + if ($logger && !$error instanceof Exception) { $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); $log = new Log(); $log->setNamespace("realtime"); - $log->setServer(\gethostname()); + $log->setServer(gethostname()); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); $log->setMessage($error->getMessage()); @@ -182,7 +193,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume $statsDocument = Authorization::skip(fn () => $database->createDocument('realtime', $document)); break; - } catch (\Throwable $th) { + } catch (Throwable) { Console::warning("Collection not ready. Retrying connection ({$attempts})..."); sleep(DATABASE_RECONNECT_SLEEP); } @@ -210,7 +221,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume ->setAttribute('value', json_encode($payload)); Authorization::skip(fn () => $database->updateDocument('realtime', $statsDocument->getId(), $statsDocument)); - } catch (\Throwable $th) { + } catch (Throwable $th) { call_user_func($logError, $th, "updateWorkerDocument"); } finally { $register->get('pools')->reclaim(); @@ -362,7 +373,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $stats->incr($event['project'], 'messages', $num); } }); - } catch (\Throwable $th) { + } catch (Throwable $th) { call_user_func($logError, $th, "pubSubConnection"); Console::error('Pub/sub error: ' . $th->getMessage()); @@ -389,19 +400,19 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, App::setResource('response', fn() => $response); try { - /** @var \Utopia\Database\Document $project */ + /** @var Document $project */ $project = $app->getResource('project'); /* * Project Check */ if (empty($project->getId())) { - throw new Exception('Missing or unknown project ID', 1008); + throw new Exception(Exception::REALTIME_POLICY_VIOLATION, 'Missing or unknown project ID'); } $dbForProject = getProjectDB($project); - $console = $app->getResource('console'); /** @var \Utopia\Database\Document $console */ - $user = $app->getResource('user'); /** @var \Utopia\Database\Document $user */ + $console = $app->getResource('console'); /** @var Document $console */ + $user = $app->getResource('user'); /** @var Document $user */ /* * Abuse Check @@ -416,7 +427,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $abuse = new Abuse($timeLimit); if (App::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled' && $abuse->check()) { - throw new Exception('Too many requests', 1013); + throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many requests'); } /* @@ -425,10 +436,10 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, * Skip this check for non-web platforms which are not required to send an origin header. */ $origin = $request->getOrigin(); - $originValidator = new Origin(\array_merge($project->getAttribute('platforms', []), $console->getAttribute('platforms', []))); + $originValidator = new Origin(array_merge($project->getAttribute('platforms', []), $console->getAttribute('platforms', []))); if (!$originValidator->isValid($origin) && $project->getId() !== 'console') { - throw new Exception($originValidator->getDescription(), 1008); + throw new Exception(Exception::REALTIME_POLICY_VIOLATION, $originValidator->getDescription()); } $roles = Auth::getRoles($user); @@ -439,7 +450,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, * Channels Check */ if (empty($channels)) { - throw new Exception('Missing channels', 1008); + throw new Exception(Exception::REALTIME_POLICY_VIOLATION, 'Missing channels'); } $realtime->subscribe($project->getId(), $connection, $roles, $channels); @@ -460,7 +471,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, ]); $stats->incr($project->getId(), 'connections'); $stats->incr($project->getId(), 'connectionsTotal'); - } catch (\Throwable $th) { + } catch (Throwable $th) { call_user_func($logError, $th, "initServer"); $response = [ @@ -486,7 +497,6 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $server->onMessage(function (int $connection, string $message) use ($server, $register, $realtime, $containerId) { try { - $app = new App('UTC'); $response = new Response(new SwooleResponse()); $projectId = $realtime->connections[$connection]['projectId']; $database = getConsoleDB(); @@ -494,6 +504,8 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re if ($projectId !== 'console') { $project = Authorization::skip(fn() => $database->getDocument('projects', $projectId)); $database = getProjectDB($project); + } else { + $project = null; } /* @@ -510,22 +522,22 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $abuse = new Abuse($timeLimit); if ($abuse->check() && App::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') { - throw new Exception('Too many messages', 1013); + throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many messages.'); } $message = json_decode($message, true); if (is_null($message) || (!array_key_exists('type', $message) && !array_key_exists('data', $message))) { - throw new Exception('Message format is not valid.', 1003); + throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Message format is not valid.'); } switch ($message['type']) { - /** + /** * This type is used to authenticate. */ case 'authentication': if (!array_key_exists('session', $message['data'])) { - throw new Exception('Payload is not valid.', 1003); + throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Payload is not valid.'); } $session = Auth::decodeSession($message['data']['session']); @@ -540,7 +552,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re || !Auth::sessionVerify($user->getAttribute('sessions', []), Auth::$secret, $authDuration) // Validate user has valid login token ) { // cookie not valid - throw new Exception('Session is not valid.', 1003); + throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Session is not valid.'); } $roles = Auth::getRoles($user); @@ -560,9 +572,9 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re break; default: - throw new Exception('Message type is not valid.', 1003); + throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Message type is not valid.'); } - } catch (\Throwable $th) { + } catch (Throwable $th) { $response = [ 'type' => 'error', 'data' => [ diff --git a/app/worker.php b/app/worker.php index 461328289f..59119c692f 100644 --- a/app/worker.php +++ b/app/worker.php @@ -35,7 +35,6 @@ use Utopia\Queue\Connection; Authorization::disable(); Runtime::enableCoroutine(SWOOLE_HOOK_ALL); - Server::setResource('register', fn () => $register); Server::setResource('dbForConsole', function (Cache $cache, Registry $register) { diff --git a/bin/delete-orphaned-projects b/bin/delete-orphaned-projects new file mode 100644 index 0000000000..16b9e3184d --- /dev/null +++ b/bin/delete-orphaned-projects @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php delete-orphaned-projects $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 13009a1fa0..6d00a8a50e 100644 --- a/composer.json +++ b/composer.json @@ -56,7 +56,7 @@ "utopia-php/image": "0.5.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.3.*", - "utopia-php/messaging": "0.1.*", + "utopia-php/messaging": "0.2.*", "utopia-php/migration": "0.3.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.5.*", @@ -66,7 +66,7 @@ "utopia-php/registry": "0.5.*", "utopia-php/storage": "0.18.*", "utopia-php/swoole": "0.5.*", - "utopia-php/vcs": "0.5.*", + "utopia-php/vcs": "0.6.*", "utopia-php/websocket": "0.1.*", "matomo/device-detector": "6.1.*", "dragonmantank/cron-expression": "3.3.2", diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index c7ff2e73c0..dc56932cde 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -225,8 +225,13 @@ class Exception extends \Exception public const MIGRATION_ALREADY_EXISTS = 'migration_already_exists'; public const MIGRATION_IN_PROGRESS = 'migration_in_progress'; - protected $type = ''; - protected $errors = []; + /** Realtime */ + public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid'; + public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages'; + public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation'; + + protected string $type = ''; + protected array $errors = []; public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null) { diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 392efe57d6..4e60260abb 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -73,6 +73,9 @@ abstract class Migration '1.4.6' => 'V19', '1.4.7' => 'V19', '1.4.8' => 'V20', + '1.4.8' => 'V19', + '1.4.9' => 'V19', + '1.4.10' => 'V20', ]; /** diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index f597f36028..db067c9195 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -17,6 +17,7 @@ use Appwrite\Platform\Tasks\Version; use Appwrite\Platform\Tasks\VolumeSync; use Appwrite\Platform\Tasks\CalcTierStats; use Appwrite\Platform\Tasks\Upgrade; +use Appwrite\Platform\Tasks\DeleteOrphanedProjects; class Tasks extends Service { @@ -38,6 +39,8 @@ class Tasks extends Service ->addAction(VolumeSync::getName(), new VolumeSync()) ->addAction(Specs::getName(), new Specs()) ->addAction(CalcTierStats::getName(), new CalcTierStats()) + ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects()) + ; } } diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php new file mode 100644 index 0000000000..2824f4e286 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -0,0 +1,115 @@ +desc('Get stats for projects') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->inject('register') + ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($pools, $cache, $dbForConsole, $register); + }); + } + + + public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + { + + Console::title('Delete orphaned projects V1'); + Console::success(APP_NAME . ' Delete orphaned projects started'); + + /** @var array $collections */ + $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; + + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + $projects = [$console]; + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects'); + Console::success("Found a total of: {$totalProjects} projects"); + + $orphans = 0; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + $collectionsCreated = $dbForProject->count(Database::METADATA); + $message = ' (' . $collectionsCreated . ') collections where found on project (' . $project->getId() . '))'; + if ($collectionsCreated < (count($collectionsConfig) + 2)) { + Console::error($message); + $orphans++; + } else { + Console::log($message); + } + } catch (\Throwable $th) { + //$dbForConsole->deleteDocument('projects', $project->getId()); + //Console::success('Deleting project (' . $project->getId() . ')'); + Console::error(' (0) collections where found for project (' . $project->getId() . ')'); + $orphans++; + } finally { + $pools + ->get($db) + ->reclaim(); + } + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + } + + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans'); + } +} diff --git a/src/Appwrite/Platform/Tasks/Install.php b/src/Appwrite/Platform/Tasks/Install.php index 89297a78ed..eb419ade11 100644 --- a/src/Appwrite/Platform/Tasks/Install.php +++ b/src/Appwrite/Platform/Tasks/Install.php @@ -6,9 +6,6 @@ use Appwrite\Auth\Auth; use Appwrite\Docker\Compose; use Appwrite\Docker\Env; use Appwrite\Utopia\View; -use Utopia\Analytics\Adapter; -use Utopia\Analytics\Adapter\GoogleAnalytics; -use Utopia\Analytics\Event; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Validator\Text; @@ -43,12 +40,6 @@ class Install extends Action /** @var array> $vars array whre key is variable name and value is variable */ $vars = []; - /** - * We are using a random value every execution for identification. - * This allows us to collect information without invading the privacy of our users. - */ - $analytics = new GoogleAnalytics('UA-26264668-9', uniqid('server.', true)); - foreach ($config as $category) { foreach ($category['variables'] ?? [] as $var) { $vars[$var['name']] = $var; @@ -82,7 +73,7 @@ class Install extends Action file_put_contents($this->path . '/docker-compose.yml.' . $time . '.backup', $data); $compose = new Compose($data); $appwrite = $compose->getService('appwrite'); - $oldVersion = ($appwrite) ? $appwrite->getImageVersion() : null; + $oldVersion = $appwrite?->getImageVersion(); try { $ports = $compose->getService('traefik')->getPorts(); } catch (\Throwable $th) { @@ -209,14 +200,12 @@ class Install extends Action if (!file_put_contents($this->path . '/docker-compose.yml', $templateForCompose->render(false))) { $message = 'Failed to save Docker Compose file'; - $this->sendEvent($analytics, $message); Console::error($message); Console::exit(1); } if (!file_put_contents($this->path . '/.env', $templateForEnv->render(false))) { $message = 'Failed to save environment variables file'; - $this->sendEvent($analytics, $message); Console::error($message); Console::exit(1); } @@ -237,29 +226,12 @@ class Install extends Action if ($exit !== 0) { $message = 'Failed to install Appwrite dockers'; - $this->sendEvent($analytics, $message); Console::error($message); Console::error($stderr); Console::exit($exit); } else { $message = 'Appwrite installed successfully'; - $this->sendEvent($analytics, $message); Console::success($message); } } - - private function sendEvent(Adapter $analytics, string $message): void - { - $event = new Event(); - $event->setName(APP_VERSION_STABLE); - $event->setValue($message); - $event->setUrl('http://localhost/'); - $event->setProps([ - 'category' => 'install/server', - 'action' => 'install', - ]); - $event->setType('install/server'); - - $analytics->createEvent($event); - } } diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 02c1835dd5..ade2125e4a 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -426,29 +426,31 @@ class Certificates extends Action $locale->setDefault('en'); } - $body = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl'); - $subject = \sprintf($locale->getText("emails.certificate.subject"), $domain); - $body - ->setParam('{{domain}}', $domain) - ->setParam('{{error}}', $errorMessage) - ->setParam('{{attempt}}', $attempt) - ->setParam('{{subject}}', $subject) - ->setParam('{{hello}}', $locale->getText("emails.certificate.hello")) + + $message = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-inner-base.tpl'); + $message ->setParam('{{body}}', $locale->getText("emails.certificate.body")) - ->setParam('{{redirect}}', 'https://' . $domain) + ->setParam('{{hello}}', $locale->getText("emails.certificate.hello")) ->setParam('{{footer}}', $locale->getText("emails.certificate.footer")) ->setParam('{{thanks}}', $locale->getText("emails.certificate.thanks")) - ->setParam('{{signature}}', $locale->getText("emails.certificate.signature")) - ->setParam('{{project}}', 'Console') - ->setParam('{{direction}}', $locale->getText('settings.direction')) - ->setParam('{{bg-body}}', '#f7f7f7') - ->setParam('{{bg-content}}', '#ffffff') - ->setParam('{{text-content}}', '#000000'); + ->setParam('{{signature}}', $locale->getText("emails.certificate.signature")); + $body = $message->render(); + + $emailVariables = [ + 'direction' => $locale->getText('settings.direction'), + 'domain' => $domain, + 'error' => '
' . $errorMessage . '
', + 'attempt' => $attempt, + 'project' => 'Console', + 'redirect' => 'https://' . $domain, + ]; $queueForMails ->setRecipient(App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')) - ->setBody($body->render()) + ->setSubject($subject) + ->setBody($body) + ->setVariables($emailVariables) ->setName('Appwrite Administrator') ->trigger(); } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index c4a40a711f..bef9af8468 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -178,7 +178,8 @@ class Deletes extends Action $project = $dbForConsole->getDocument('projects', $document->getAttribute('projectId')); if ($project->isEmpty()) { - Console::warning('Unable to delete schedule for function ' . $document->getAttribute('resourceId')); + $dbForConsole->deleteDocument('schedules', $document->getId()); + Console::success('Deleted schedule for deleted project ' . $document->getAttribute('projectId')); return; } @@ -921,17 +922,28 @@ class Deletes extends Action $count = 0; $chunk = 0; $limit = 50; + $results = []; $sum = $limit; + $cursor = null; $executionStart = \microtime(true); while ($sum === $limit) { $chunk++; - $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + $mergedQueries = \array_merge([Query::limit($limit)], $queries); + if ($cursor instanceof Document) { + $mergedQueries[] = Query::cursorAfter($cursor); + } + + $results = $database->find($collection, $mergedQueries); $sum = count($results); + if ($sum > 0) { + $cursor = $results[$sum - 1]; + } + foreach ($results as $document) { if (is_callable($callback)) { $callback($document); diff --git a/src/Appwrite/Platform/Workers/Mails.php b/src/Appwrite/Platform/Workers/Mails.php index 7a20212c9c..94a2a46087 100644 --- a/src/Appwrite/Platform/Workers/Mails.php +++ b/src/Appwrite/Platform/Workers/Mails.php @@ -58,13 +58,21 @@ class Mails extends Action $subject = $payload['subject']; $variables = $payload['variables']; $name = $payload['name']; - $body = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl'); + $body = $payload['body']; + $bodyTemplate = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl'); + $bodyTemplate->setParam('{{body}}', $body); foreach ($variables as $key => $value) { - $body->setParam('{{' . $key . '}}', $value); + $bodyTemplate->setParam('{{' . $key . '}}', $value); } + $body = $bodyTemplate->render(); - $body = $body->render(); + $subjectTemplate = Template::fromString($subject); + foreach ($variables as $key => $value) { + $subjectTemplate->setParam('{{' . $key . '}}', $value); + } + // render() will return the subject in

tags, so use strip_tags() to remove them + $subject = \strip_tags($subjectTemplate->render()); /** @var PHPMailer $mail */ $mail = empty($smtp) diff --git a/src/Appwrite/Utopia/Request.php b/src/Appwrite/Utopia/Request.php index 229c9dd53d..cb877afad4 100644 --- a/src/Appwrite/Utopia/Request.php +++ b/src/Appwrite/Utopia/Request.php @@ -95,4 +95,46 @@ class Request extends UtopiaRequest { return self::$route != null; } + + /** + * Get headers + * + * Method for getting all HTTP header parameters, including cookies. + * + * @return array + */ + public function getHeaders(): array + { + $headers = $this->generateHeaders(); + + if (empty($this->swoole->cookie)) { + return $headers; + } + + $cookieHeaders = []; + foreach ($this->swoole->cookie as $key => $value) { + $cookieHeaders[] = "{$key}={$value}"; + } + + if (!empty($cookieHeaders)) { + $headers['cookie'] = \implode('; ', $cookieHeaders); + } + + return $headers; + } + + /** + * Get header + * + * Method for querying HTTP header parameters. If $key is not found $default value will be returned. + * + * @param string $key + * @param string $default + * @return string + */ + public function getHeader(string $key, string $default = ''): string + { + $headers = $this->getHeaders(); + return $headers[$key] ?? $default; + } } diff --git a/tests/e2e/Scopes/ProjectCustom.php b/tests/e2e/Scopes/ProjectCustom.php index 408b8c8fe7..5f7bf85d0e 100644 --- a/tests/e2e/Scopes/ProjectCustom.php +++ b/tests/e2e/Scopes/ProjectCustom.php @@ -81,6 +81,8 @@ trait ProjectCustom 'locale.read', 'avatars.read', 'health.read', + 'rules.read', + 'rules.write' ], ]); diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index cdc9ec846f..e39c6573e4 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -1344,4 +1344,192 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(204, $response['headers']['status-code']); } + + public function testCookieExecution() + { + $timeout = 5; + $code = realpath(__DIR__ . '/../../../resources/functions') . "/php-cookie/code.tar.gz"; + $this->packageCode('php-cookie'); + + $function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'functionId' => ID::unique(), + 'name' => 'Test PHP Cookie executions', + 'runtime' => 'php-8.0', + 'entrypoint' => 'index.php', + 'timeout' => $timeout, + ]); + + $functionId = $function['body']['$id'] ?? ''; + + $this->assertEquals(201, $function['headers']['status-code']); + + $deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'entrypoint' => 'index.php', + 'code' => new CURLFile($code, 'application/x-gzip', basename($code)), + 'activate' => true + ]); + + $deploymentId = $deployment['body']['$id'] ?? ''; + $this->assertEquals(202, $deployment['headers']['status-code']); + + // Poll until deployment is built + while (true) { + $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + if ( + $deployment['headers']['status-code'] >= 400 + || \in_array($deployment['body']['status'], ['ready', 'failed']) + ) { + break; + } + + \sleep(1); + } + + $deployment = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $deployment['headers']['status-code']); + + // Wait a little for activation to finish + sleep(5); + + $cookie = 'cookieName=cookieValue; cookie2=value2; cookie3=value=3; cookie4=val:ue4; cookie5=value5'; + + $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'async' => false, + 'headers' => [ + 'cookie' => $cookie + ] + ]); + + $this->assertEquals(201, $execution['headers']['status-code']); + $this->assertEquals('completed', $execution['body']['status']); + $this->assertEquals(200, $execution['body']['responseStatusCode']); + $this->assertEquals($cookie, $execution['body']['responseBody']); + + // Cleanup : Delete function + $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $functionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(204, $response['headers']['status-code']); + } + + public function testFunctionsDomain() + { + $timeout = 5; + $code = realpath(__DIR__ . '/../../../resources/functions') . "/php-cookie/code.tar.gz"; + $this->packageCode('php-cookie'); + + $function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'functionId' => ID::unique(), + 'name' => 'Test PHP Cookie executions', + 'runtime' => 'php-8.0', + 'entrypoint' => 'index.php', + 'timeout' => $timeout, + 'execute' => ['any'] + ]); + + $functionId = $function['body']['$id'] ?? ''; + + $this->assertEquals(201, $function['headers']['status-code']); + + $rules = $this->client->call(Client::METHOD_GET, '/proxy/rules', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ 'equal("resourceId", "' . $functionId . '")', 'equal("resourceType", "function")' ] + ]); + + $this->assertEquals(200, $rules['headers']['status-code']); + $this->assertEquals(1, $rules['body']['total']); + $this->assertCount(1, $rules['body']['rules']); + $this->assertNotEmpty($rules['body']['rules'][0]['domain']); + + $domain = $rules['body']['rules'][0]['domain']; + + $deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'entrypoint' => 'index.php', + 'code' => new CURLFile($code, 'application/x-gzip', basename($code)), + 'activate' => true + ]); + + $deploymentId = $deployment['body']['$id'] ?? ''; + $this->assertEquals(202, $deployment['headers']['status-code']); + + // Poll until deployment is built + while (true) { + $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + if ( + $deployment['headers']['status-code'] >= 400 + || \in_array($deployment['body']['status'], ['ready', 'failed']) + ) { + break; + } + + \sleep(1); + } + + $deployment = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $deployment['headers']['status-code']); + + // Wait a little for activation to finish + sleep(5); + + $cookie = 'cookieName=cookieValue; cookie2=value2; cookie3=value=3; cookie4=val:ue4; cookie5=value5'; + + $proxyClient = new Client(); + $proxyClient->setEndpoint('http://' . $domain); + + $response = $proxyClient->call(Client::METHOD_GET, '/', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => $cookie + ])); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($cookie, $response['body']); + + // Cleanup : Delete function + $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $functionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(204, $response['headers']['status-code']); + } } diff --git a/tests/e2e/Services/GraphQL/Base.php b/tests/e2e/Services/GraphQL/Base.php index 3a4b88e2c2..f705e8d77b 100644 --- a/tests/e2e/Services/GraphQL/Base.php +++ b/tests/e2e/Services/GraphQL/Base.php @@ -2,6 +2,8 @@ namespace Tests\E2E\Services\GraphQL; +use Utopia\CLI\Console; + trait Base { // Databases @@ -1933,4 +1935,13 @@ trait Base throw new \InvalidArgumentException('Invalid query type'); } + + // Function-related methods + protected string $stdout = ''; + protected string $stderr = ''; + + protected function packageCode($folder) + { + Console::execute('cd ' . realpath(__DIR__ . "/../../../resources/functions") . "/$folder && tar --exclude code.tar.gz -czf code.tar.gz .", '', $this->stdout, $this->stderr); + } } diff --git a/tests/e2e/Services/GraphQL/FunctionsClientTest.php b/tests/e2e/Services/GraphQL/FunctionsClientTest.php index 363f7a0fb1..3f0ee1966a 100644 --- a/tests/e2e/Services/GraphQL/FunctionsClientTest.php +++ b/tests/e2e/Services/GraphQL/FunctionsClientTest.php @@ -82,7 +82,11 @@ class FunctionsClientTest extends Scope { $projectId = $this->getProject()['$id']; $query = $this->getQuery(self::$CREATE_DEPLOYMENT); - $code = realpath(__DIR__ . '/../../../resources/functions') . "/php/code.tar.gz"; + + $folder = 'php'; + $code = realpath(__DIR__ . '/../../../resources/functions') . "/$folder/code.tar.gz"; + $this->packageCode($folder); + $gqlPayload = [ 'operations' => \json_encode([ 'query' => $query, diff --git a/tests/e2e/Services/GraphQL/FunctionsServerTest.php b/tests/e2e/Services/GraphQL/FunctionsServerTest.php index 49a8fd61f2..25a671fa1c 100644 --- a/tests/e2e/Services/GraphQL/FunctionsServerTest.php +++ b/tests/e2e/Services/GraphQL/FunctionsServerTest.php @@ -81,7 +81,11 @@ class FunctionsServerTest extends Scope { $projectId = $this->getProject()['$id']; $query = $this->getQuery(self::$CREATE_DEPLOYMENT); - $code = realpath(__DIR__ . '/../../../resources/functions') . "/php/code.tar.gz"; + + $folder = 'php'; + $code = realpath(__DIR__ . '/../../../resources/functions') . "/$folder/code.tar.gz"; + $this->packageCode($folder); + $gqlPayload = [ 'operations' => \json_encode([ 'query' => $query, diff --git a/tests/e2e/Services/VCS/VCSBase.php b/tests/e2e/Services/VCS/VCSBase.php new file mode 100644 index 0000000000..7531ea3bc8 --- /dev/null +++ b/tests/e2e/Services/VCS/VCSBase.php @@ -0,0 +1,17 @@ +markTestSkipped('VCS is not enabled.'); + } + } +} diff --git a/tests/e2e/Services/VCS/VCSConsoleClientTest.php b/tests/e2e/Services/VCS/VCSConsoleClientTest.php new file mode 100644 index 0000000000..697fbdba91 --- /dev/null +++ b/tests/e2e/Services/VCS/VCSConsoleClientTest.php @@ -0,0 +1,319 @@ +client->call(Client::METHOD_GET, '/mock/github/callback', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'providerInstallationId' => $this->providerInstallationId, + 'projectId' => $this->getProject()['$id'], + ]); + + $this->assertNotEmpty($response['body']['installationId']); + $installationId = $response['body']['installationId']; + return $installationId; + } + + /** + * @depends testGitHubAuthorize + */ + public function testGetInstallation(string $installationId): void + { + /** + * Test for SUCCESS + */ + + $installation = $this->client->call(Client::METHOD_GET, '/vcs/installations/' . $installationId, array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $installation['headers']['status-code']); + $this->assertEquals('github', $installation['body']['provider']); + $this->assertEquals('appwrite-test', $installation['body']['organization']); + } + + /** + * @depends testGitHubAuthorize + */ + public function testDetectRuntime(string $installationId): void + { + /** + * Test for SUCCESS + */ + + $runtime = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/providerRepositories/' . $this->providerRepositoryId . '/detection', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $runtime['headers']['status-code']); + $this->assertEquals($runtime['body']['runtime'], 'ruby-3.1'); + + /** + * Test for FAILURE + */ + + $runtime = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/providerRepositories/randomRepositoryId/detection', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(404, $runtime['headers']['status-code']); + } + + /** + * @depends testGitHubAuthorize + */ + public function testListRepositories(string $installationId): void + { + /** + * Test for SUCCESS + */ + + $repositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $repositories['headers']['status-code']); + $this->assertEquals($repositories['body']['total'], 3); + $this->assertEquals($repositories['body']['providerRepositories'][0]['name'], 'function1.4'); + $this->assertEquals($repositories['body']['providerRepositories'][0]['organization'], 'appwrite-test'); + $this->assertEquals($repositories['body']['providerRepositories'][0]['provider'], 'github'); + $this->assertEquals($repositories['body']['providerRepositories'][1]['name'], 'appwrite'); + $this->assertEquals($repositories['body']['providerRepositories'][2]['name'], 'ruby-starter'); + + + $searchedRepositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'func' + ]); + + $this->assertEquals(200, $searchedRepositories['headers']['status-code']); + $this->assertEquals($searchedRepositories['body']['total'], 1); + $this->assertEquals($searchedRepositories['body']['providerRepositories'][0]['name'], 'function1.4'); + + /** + * Test for FAILURE + */ + + $repositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/randomInstallationId/providerRepositories', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(404, $repositories['headers']['status-code']); + } + + /** + * @depends testGitHubAuthorize + */ + public function testGetRepository(string $installationId): void + { + /** + * Test for SUCCESS + */ + + $repository = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories/' . $this->providerRepositoryId, array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $repository['headers']['status-code']); + $this->assertEquals($repository['body']['name'], 'ruby-starter'); + $this->assertEquals($repository['body']['organization'], 'appwrite-test'); + $this->assertEquals($repository['body']['private'], false); + + $repository = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories/' . $this->providerRepositoryId2, array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $repository['headers']['status-code']); + $this->assertEquals($repository['body']['name'], 'function1.4'); + $this->assertEquals($repository['body']['organization'], 'appwrite-test'); + $this->assertEquals($repository['body']['private'], true); + + /** + * Test for FAILURE + */ + + $repository = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories/randomRepositoryId', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(404, $repository['headers']['status-code']); + } + + /** + * @depends testGitHubAuthorize + */ + public function testListRepositoryBranches(string $installationId): void + { + /** + * Test for SUCCESS + */ + + $repositoryBranches = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories/' . $this->providerRepositoryId . '/branches', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $repositoryBranches['headers']['status-code']); + $this->assertEquals($repositoryBranches['body']['total'], 2); + $this->assertEquals($repositoryBranches['body']['branches'][0]['name'], 'main'); + $this->assertEquals($repositoryBranches['body']['branches'][1]['name'], 'test'); + + /** + * Test for FAILURE + */ + + $repositoryBranches = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories/randomRepositoryId/branches', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(404, $repositoryBranches['headers']['status-code']); + } + + /** + * @depends testGitHubAuthorize + */ + public function testCreateFunctionUsingVCS(string $installationId): array + { + $function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'functionId' => ID::unique(), + 'name' => 'Test', + 'execute' => [Role::user($this->getUser()['$id'])->toString()], + 'runtime' => 'php-8.0', + 'entrypoint' => 'index.php', + 'events' => [ + 'users.*.create', + 'users.*.delete', + ], + 'schedule' => '0 0 1 1 *', + 'timeout' => 10, + 'installationId' => $installationId, + 'providerRepositoryId' => $this->providerRepositoryId, + 'providerBranch' => 'main', + ]); + + $this->assertEquals(201, $function['headers']['status-code']); + $this->assertEquals('Test', $function['body']['name']); + $this->assertEquals('php-8.0', $function['body']['runtime']); + $this->assertEquals('index.php', $function['body']['entrypoint']); + $this->assertEquals('705764267', $function['body']['providerRepositoryId']); + $this->assertEquals('main', $function['body']['providerBranch']); + + return [ + 'installationId' => $installationId, + 'functionId' => $function['body']['$id'] + ]; + } + + /** + * @depends testCreateFunctionUsingVCS + */ + public function testUpdateFunctionUsingVCS(array $data): string + { + $function = $this->client->call(Client::METHOD_PUT, '/functions/' . $data['functionId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'functionId' => ID::unique(), + 'name' => 'Test', + 'execute' => [Role::user($this->getUser()['$id'])->toString()], + 'runtime' => 'php-8.0', + 'entrypoint' => 'index.php', + 'events' => [ + 'users.*.create', + 'users.*.delete', + ], + 'schedule' => '0 0 1 1 *', + 'timeout' => 10, + 'installationId' => $data['installationId'], + 'providerRepositoryId' => $this->providerRepositoryId2, + 'providerBranch' => 'main', + ]); + + $this->assertEquals(200, $function['headers']['status-code']); + $this->assertEquals('Test', $function['body']['name']); + $this->assertEquals('php-8.0', $function['body']['runtime']); + $this->assertEquals('index.php', $function['body']['entrypoint']); + $this->assertEquals('708688544', $function['body']['providerRepositoryId']); + $this->assertEquals('main', $function['body']['providerBranch']); + + return $function['body']['$id']; + } + + /** + * @depends testGitHubAuthorize + */ + public function testCreateRepository(string $installationId): void + { + /** + * Test for SUCCESS + */ + + $github = new GitHub(new Cache(new None())); + $privateKey = App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'); + $githubAppId = App::getEnv('_APP_VCS_GITHUB_APP_ID'); + $github->initializeVariables($this->providerInstallationId, $privateKey, $githubAppId); + + $repository = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'test-repo-1', + 'private' => true + ]); + + $this->assertEquals('test-repo-1', $repository['body']['name']); + $this->assertEquals('appwrite-test', $repository['body']['organization']); + $this->assertEquals('github', $repository['body']['provider']); + + /** + * Test for FAILURE + */ + + $repository = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'test-repo-1', + 'private' => true + ]); + + $this->assertEquals(400, $repository['headers']['status-code']); + $this->assertEquals('Provider Error: Repository creation failed. name already exists on this account', $repository['body']['message']); + + /** + * Test for SUCCESS + */ + + $result = $github->deleteRepository('appwrite-test', 'test-repo-1'); + $this->assertEquals($result, true); + } +} diff --git a/tests/e2e/Services/VCS/VCSCustomClientTest.php b/tests/e2e/Services/VCS/VCSCustomClientTest.php new file mode 100644 index 0000000000..084255cd69 --- /dev/null +++ b/tests/e2e/Services/VCS/VCSCustomClientTest.php @@ -0,0 +1,24 @@ +client->call(Client::METHOD_GET, '/vcs/installations/' . $installationId, array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(401, $installation['headers']['status-code']); + } +} diff --git a/tests/e2e/Services/VCS/VCSCustomServerTest.php b/tests/e2e/Services/VCS/VCSCustomServerTest.php new file mode 100644 index 0000000000..dee4cd7ced --- /dev/null +++ b/tests/e2e/Services/VCS/VCSCustomServerTest.php @@ -0,0 +1,25 @@ +client->call(Client::METHOD_GET, '/vcs/installations/' . $installationId, array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders())); + + $this->assertEquals(401, $installation['headers']['status-code']); + } +} diff --git a/tests/resources/functions/php-cookie/index.php b/tests/resources/functions/php-cookie/index.php new file mode 100644 index 0000000000..8f38f752cb --- /dev/null +++ b/tests/resources/functions/php-cookie/index.php @@ -0,0 +1,5 @@ +res->send($context->req->headers['cookie'] ?? ''); +};