Add benchmarks base

2026-05-26 13:51:13 +00:00 · 2022-10-31 19:46:20 +13:00
parent 1b0abf4643
commit f52e4c3e63
18 changed files with 1761 additions and 98 deletions
@@ -0,0 +1,20 @@
+name: "Benchmark"
+
+on: [pull_request]
+
+jobs:
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 2
+
+    - run: git checkout HEAD^2
+
+    - name: Run benchmark against baseline
+      run: docker run --rm -v $PWD:/app composer sh -c \
+        "composer install --profile --ignore-platform-reqs && composer benchmark"
@@ -873,6 +873,11 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId')
            throw new Exception(Exception::DEPLOYMENT_NOT_FOUND);
        }

+        $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''));
+        $deployment->setAttribute('status', $build->getAttribute('status', 'processing'));
+        $deployment->setAttribute('buildStderr', $build->getAttribute('stderr', ''));
+        $deployment->setAttribute('buildStdout', $build->getAttribute('stdout', ''));
+
        $response->dynamic($deployment, Response::MODEL_DEPLOYMENT);
    });

@@ -12,7 +12,19 @@
    "scripts": {
        "test": "vendor/bin/phpunit",
        "lint": "vendor/bin/phpcs",
-        "format": "vendor/bin/phpcbf"
+        "format": "vendor/bin/phpcbf",
+        "benchmark": [
+            "Composer\\Config::disableProcessTimeout",
+            "vendor/bin/phpbench run --report=appwrite"
+        ],
+        "benchmark-tag": [
+            "Composer\\Config::disableProcessTimeout",
+            "vendor/bin/phpbench run --tag=baseline --report=appwrite"
+        ],
+        "benchmark-compare": [
+            "Composer\\Config::disableProcessTimeout",
+            "vendor/bin/phpbench run --ref=baseline --report=appwrite"
+        ]
    },
    "autoload": {
        "psr-4": {
@@ -24,6 +36,7 @@
        "psr-4": {
            "Tests\\E2E\\": "tests/e2e",
            "Tests\\Unit\\": "tests/unit",
+            "Tests\\Benchmarks\\": "tests/benchmarks",
            "Appwrite\\Tests\\": "tests/extensions"
        }
    },
@@ -82,7 +95,8 @@
        "phpunit/phpunit": "9.5.20",
        "squizlabs/php_codesniffer": "^3.6",
        "swoole/ide-helper": "4.8.9",
-        "textalk/websocket": "1.5.7"
+        "textalk/websocket": "1.5.7",
+        "phpbench/phpbench": "^1.2"
    },
    "provide": {
        "ext-phpiredis": "*"
@@ -0,0 +1,16 @@
+{
+  "$schema": "./vendor/phpbench/phpbench/phpbench.schema.json",
+  "runner.path": "tests/benchmarks",
+  "runner.file_pattern": "*Bench.php",
+  "runner.bootstrap": "app/init.php",
+  "runner.revs": 500,
+  "runner.iterations": 3,
+  "runner.warmup": 1,
+  "report.generators": {
+    "appwrite": {
+      "extends": "aggregate",
+      "cols": ["benchmark", "subject", "revs", "its", "worst", "best", "mean", "rstdev"],
+      "break": ["benchmark"]
+    }
+  }
+}
@@ -0,0 +1,153 @@
+<?php
+
+namespace Tests\Benchmarks\Database;
+
+use PhpBench\Attributes\BeforeMethods;
+use PhpBench\Attributes\ParamProviders;
+use Tests\Benchmarks\Scope;
+use Tests\E2E\Client;
+use Tests\E2E\Scopes\ProjectCustom;
+use Utopia\Database\ID;
+use Utopia\Database\Permission;
+use Utopia\Database\Role;
+
+abstract class Base extends Scope
+{
+    use ProjectCustom;
+
+    protected static string $databaseId;
+    protected static string $collectionId;
+    protected static string $documentId;
+
+    #[BeforeMethods(['createDatabase', 'createCollection'])]
+    public function benchDocumentCreate()
+    {
+        $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'documentId' => ID::unique(),
+            'data' => [
+                'title' => 'The Matrix',
+            ],
+            'permissions' => [
+                Permission::read(Role::user($this->getUser()['$id'])),
+                Permission::write(Role::user($this->getUser()['$id'])),
+            ],
+        ]);
+    }
+
+    #[ParamProviders(['provideCounts'])]
+    #[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])]
+    public function benchDocumentReadList(array $params)
+    {
+        $this->client->call(Client::METHOD_GET, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'queries' => ['limit(' . $params['documents'] . ')'],
+        ]);
+    }
+
+    #[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])]
+    public function benchDocumentRead()
+    {
+        $this->client->call(Client::METHOD_GET, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents/' . static::$documentId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()));
+    }
+
+    #[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])]
+    public function benchDocumentUpdate()
+    {
+        $this->client->call(Client::METHOD_PATCH, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents/' . static::$documentId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'data' => [
+                'title' => 'The Matrix Reloaded',
+            ],
+        ]);
+    }
+
+    public function provideCounts(): array
+    {
+        return [
+            '1 Document' => ['documents' => 1],
+            '10 Documents' => ['documents' => 10],
+            '100 Documents' => ['documents' => 100],
+        ];
+    }
+
+    public function createDatabase(array $params = [])
+    {
+        // Create database
+        $database = $this->client->call(Client::METHOD_POST, '/databases', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'databaseId' => ID::unique(),
+            'name' => 'Test Database'
+        ]);
+        static::$databaseId = $database['body']['$id'];
+    }
+
+    public function createCollection(array $params = [])
+    {
+        // Create collection
+        $movies = $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'collectionId' => ID::unique(),
+            'name' => 'Movies',
+            'documentSecurity' => true,
+            'permissions' => [
+                Permission::read(Role::user($this->getUser()['$id'])),
+                Permission::write(Role::user($this->getUser()['$id'])),
+            ],
+        ]);
+        static::$collectionId = $movies['body']['$id'];
+
+        // Create attribute
+        $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/attributes/string', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'key' => 'title',
+            'size' => 256,
+            'required' => true,
+        ]);
+
+        sleep(2);
+    }
+
+    public function createDocuments(array $params = [])
+    {
+        $count = $params['documents'] ?? 1;
+
+        // Create documents
+        for ($i = 0; $i < $count; $i++) {
+            $response = $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', [
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+                'x-appwrite-key' => $this->getProject()['apiKey']
+            ], [
+                'documentId' => ID::unique(),
+                'data' => [
+                    'title' => 'Captain America' . $i,
+                ],
+                'permissions' => [
+                    Permission::read(Role::user($this->getUser()['$id'])),
+                    Permission::write(Role::user($this->getUser()['$id'])),
+                ]
+            ]);
+
+            static::$documentId = $response['body']['$id'];
+        }
+    }
+}
@@ -0,0 +1,10 @@
+<?php
+
+namespace Tests\Benchmarks\Database;
+
+use Tests\E2E\Scopes\SideClient;
+
+class DatabasesCustomClientBench extends Base
+{
+    use SideClient;
+}
@@ -0,0 +1,10 @@
+<?php
+
+namespace Tests\Benchmarks\Database;
+
+use Tests\E2E\Scopes\SideServer;
+
+class DatabasesCustomServerBench extends Base
+{
+    use SideServer;
+}
@@ -0,0 +1,110 @@
+<?php
+
+namespace Tests\Benchmarks\Functions;
+
+use CURLFile;
+use PhpBench\Attributes\BeforeMethods;
+use Tests\Benchmarks\Scope;
+use Tests\E2E\Client;
+use Tests\E2E\Scopes\ProjectCustom;
+use Utopia\CLI\Console;
+use Utopia\Database\ID;
+use Utopia\Database\Role;
+
+abstract class Base extends Scope
+{
+    use ProjectCustom;
+
+    protected static string $functionId;
+    protected static string $deploymentId;
+    protected static string $executionId;
+    protected static string $codePath;
+
+    #[BeforeMethods(['createFunction', 'prepareDeployment', 'createDeployment', 'patchDeployment'])]
+    public function benchExecutionCreate()
+    {
+        $this->client->call(Client::METHOD_POST, '/functions/' . static::$functionId . '/executions', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()));
+    }
+
+    public function createFunction()
+    {
+        $response = $this->client->call(Client::METHOD_POST, '/functions', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'functionId' => ID::unique(),
+            'name' => 'Test',
+            'runtime' => 'php-8.0',
+            'timeout' => 10,
+            'execute' => [Role::users()->toString()]
+        ]);
+        static::$functionId = $response['body']['$id'];
+    }
+
+    public function prepareDeployment()
+    {
+        static::$codePath = realpath(__DIR__ . '/../../resources/functions') . "/php/code.tar.gz";
+
+        $stdout = '';
+        $stderr = '';
+
+        Console::execute(
+            'cd ' . realpath(__DIR__ . "/../../resources/functions") . '/' . static::$codePath . "  && \
+            tar --exclude code.tar.gz -czf code.tar.gz .",
+            '',
+            $stdout,
+            $stderr
+        );
+    }
+
+    public function createDeployment()
+    {
+        $response = $this->client->call(Client::METHOD_POST, '/functions/' . static::$functionId . '/deployments', [
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'entrypoint' => 'index.php',
+            'code' => new CURLFile(static::$codePath, 'application/x-gzip', \basename(static::$codePath)),
+        ]);
+
+        static::$deploymentId = $response['body']['$id'];
+
+        while (true) {
+            $response = $this->client->call(Client::METHOD_GET, '/functions/' . static::$functionId . '/deployments/' . static::$deploymentId, [
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+                'x-appwrite-key' => $this->getProject()['apiKey']
+            ]);
+
+            $status = $response['body']['status'] ?? '';
+
+            switch ($status) {
+                case '':
+                case 'processing':
+                case 'building':
+                    usleep(500);
+                    break;
+                case 'ready':
+                    break 2;
+                case 'failed':
+                    throw new \Exception('Failed to build function');
+            }
+        }
+
+        sleep(1);
+    }
+
+    public function patchDeployment()
+    {
+        $this->client->call(Client::METHOD_PATCH, '/functions/' . static::$functionId . '/deployments/' . static::$deploymentId, [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey'],
+        ], []);
+    }
+}
@@ -0,0 +1,10 @@
+<?php
+
+namespace Tests\Benchmarks\Functions;
+
+use Tests\E2E\Scopes\SideClient;
+
+class FunctionsCustomClientBench extends Base
+{
+    use SideClient;
+}
@@ -0,0 +1,17 @@
+<?php
+
+namespace Tests\Benchmarks\Functions;
+
+use PhpBench\Attributes\BeforeMethods;
+use Tests\E2E\Scopes\SideServer;
+
+class FunctionsCustomServerBench extends Base
+{
+    use SideServer;
+
+    #[BeforeMethods(['createFunction', 'prepareDeployment'])]
+    public function benchDeploymentCreate()
+    {
+        $this->createDeployment();
+    }
+}
@@ -0,0 +1,13 @@
+<?php
+
+namespace Tests\Benchmarks;
+
+use PhpBench\Attributes\AfterMethods;
+use PhpBench\Attributes\BeforeMethods;
+use Tests\E2E\Scopes\Scope as E2EScope;
+
+#[BeforeMethods(['setUp'])]
+#[AfterMethods(['tearDown'])]
+abstract class Scope extends E2EScope
+{
+}
@@ -0,0 +1,116 @@
+<?php
+
+namespace Tests\Benchmarks\Storage;
+
+use CURLFile;
+use PhpBench\Attributes\BeforeMethods;
+use PhpBench\Attributes\ParamProviders;
+use Tests\Benchmarks\Scope;
+use Tests\E2E\Client;
+use Tests\E2E\Scopes\ProjectCustom;
+use Utopia\Database\ID;
+use Utopia\Database\Permission;
+use Utopia\Database\Role;
+
+abstract class Base extends Scope
+{
+    use ProjectCustom;
+
+    protected static string $bucketId;
+    protected static string $fileId;
+
+    #[BeforeMethods(['createBucket'])]
+    public function benchFileCreate()
+    {
+        $this->client->call(Client::METHOD_POST, '/storage/buckets/' . static::$bucketId . '/files', array_merge([
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'fileId' => ID::unique(),
+            'permissions' => [
+                Permission::read(Role::user($this->getUser()['$id'])),
+                Permission::write(Role::user($this->getUser()['$id'])),
+            ],
+            'file' => new CURLFile(realpath(__DIR__ . '/../../resources/logo.png'), 'image/png', 'logo.png'),
+        ]);
+    }
+
+    #[ParamProviders(['provideCounts'])]
+    #[BeforeMethods(['createBucket', 'createFiles'])]
+    public function benchFileReadList(array $params)
+    {
+        $this->client->call(Client::METHOD_GET, '/storage/buckets/' . static::$bucketId . '/files', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'queries' => ['limit(' . $params['files'] . ')'],
+        ]);
+    }
+
+    #[BeforeMethods(['createBucket', 'createFiles'])]
+    public function benchFileRead()
+    {
+        $this->client->call(Client::METHOD_GET, '/storage/buckets/' . static::$bucketId . '/files/' . static::$fileId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()));
+    }
+
+    #[BeforeMethods(['createBucket', 'createFiles'])]
+    public function benchFileUpdate()
+    {
+        $this->client->call(Client::METHOD_PUT, '/storage/buckets/' . static::$bucketId . '/files/' . static::$fileId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'name' => 'Updated name',
+            'permissions' => [],
+        ]);
+    }
+
+    public function provideCounts(): array
+    {
+        return [
+            '10 Files' => ['files' => 10],
+            '100 Files' => ['files' => 100],
+        ];
+    }
+
+    public function createBucket(array $params = [])
+    {
+        // Create bucket
+        $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'bucketId' => ID::unique(),
+            'name' => 'Test Bucket',
+            'permissions' => [
+                Permission::read(Role::user($this->getUser()['$id'])),
+                Permission::write(Role::user($this->getUser()['$id'])),
+            ],
+            'fileSecurity' => true
+        ]);
+        static::$bucketId = $bucket['body']['$id'];
+    }
+
+    public function createFiles(array $params = [])
+    {
+        $count = $params['files'] ?? 1;
+
+        // Create files
+        for ($i = 0; $i < $count; $i++) {
+            $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . static::$bucketId . '/files', [
+                'content-type' => 'multipart/form-data',
+                'x-appwrite-project' => $this->getProject()['$id'],
+                'x-appwrite-key' => $this->getProject()['apiKey']
+            ], [
+                'fileId' => ID::unique(),
+                'file' => new CURLFile(realpath(__DIR__ . '/../../resources/logo.png'), 'image/png', 'logo.png'),
+            ]);
+
+            static::$fileId = $response['body']['$id'];
+        }
+    }
+}
@@ -0,0 +1,10 @@
+<?php
+
+namespace Tests\Benchmarks\Storage;
+
+use Tests\E2E\Scopes\SideClient;
+
+class StorageCustomClientBench extends Base
+{
+    use SideClient;
+}
@@ -0,0 +1,10 @@
+<?php
+
+namespace Tests\Benchmarks\Storage;
+
+use Tests\E2E\Scopes\SideServer;
+
+class StorageCustomServerBench extends Base
+{
+    use SideServer;
+}
@@ -1,34 +0,0 @@
-import http from 'k6/http';
-import { check } from 'k6';
-import { Counter } from 'k6/metrics';
-
-// A simple counter for http requests
-export const requests = new Counter('http_reqs');
-
-// you can specify stages of your test (ramp up/down patterns) through the options object
-// target is the number of VUs you are aiming for
-
-export const options = {
-    stages: [
-    { target: 50, duration: '1m' },
-    // { target: 15, duration: '1m' },
-    // { target: 0, duration: '1m' },
-    ],
-    thresholds: {
-        requests: ['count < 100'],
-    },
-};
-
-export default function () {
-    const config = {
-        headers: {
-            'X-Appwrite-Key': '24356eb021863f81eb7dd77c7750304d0464e141cad6e9a8befa1f7d2b066fde190df3dab1e8d2639dbb82ee848da30501424923f4cd80d887ee40ad77ded62763ee489448523f6e39667f290f9a54b2ab8fad131a0bc985e6c0f760015f7f3411e40626c75646bb19d2bb2f7bf2f63130918220a206758cbc48845fd725a695',
-            'X-Appwrite-Project': '60479fe35d95d'
-        }}
-
-    const resDb = http.get('http://localhost:9501/', config);
-
-    check(resDb, {
-        'status is 200': (r) => r.status === 200,
-    });
-}
@@ -1,59 +0,0 @@
-// k6 run tests/benchmarks/ws.js
-
-import { URL } from 'https://jslib.k6.io/url/1.0.0/index.js';
-import ws from 'k6/ws';
-import { check } from 'k6';
-
-export let options = {
-    stages: [
-        {
-            duration: '10s',
-            target: 500
-    },
-        {
-            duration: '1m',
-            target: 500
-    },
-    ],
-}
-
-export default function () {
-    // const url = new URL('wss://appwrite-realtime.monitor-api.com/v1/realtime');
-    // url.searchParams.append('project', '604249e6b1a9f');
-    const url = new URL('ws://localhost/v1/realtime');
-    url.searchParams.append('project', 'console');
-    url.searchParams.append('channels[]', 'files');
-
-    const res = ws.connect(url.toString(), function (socket) {
-        let connection = false;
-        let checked = false;
-        let payload = null;
-        socket.on('open', () => {
-            connection = true;
-        });
-
-        socket.on('message', (data) => {
-            payload = data;
-            checked = true;
-        });
-
-        socket.setTimeout(function () {
-            check(payload, {
-                'connection opened': (r) => connection,
-                'message received': (r) => checked,
-                'channels are right': (r) => r === JSON.stringify({
-                    "type": "connected",
-                    "data": {
-                        "channels": [
-                            "files"
-                        ],
-                        "user": null
-                    }
-                })
-            })
-            socket.close();
-        }, 5000);
-    });
-
-    check(res, { 'status is 101': (r) => r && r.status === 101 });
-}
@@ -21,7 +21,7 @@ abstract class Scope extends TestCase
     */
    protected $endpoint = 'http://localhost/v1';

-    protected function setUp(): void
+    public function setUp(): void
    {
        $this->client = new Client();

@@ -30,7 +30,7 @@ abstract class Scope extends TestCase
        ;
    }

-    protected function tearDown(): void
+    public function tearDown(): void
    {
        $this->client = null;
    }