Add usage based adapter todo

.env

@@ -70,7 +70,8 @@ _APP_FUNCTIONS_MEMORY_SWAP=0
 _APP_FUNCTIONS_INACTIVE_THRESHOLD=60
 OPEN_RUNTIMES_NETWORK=appwrite_runtimes
 _APP_EXECUTOR_SECRET=your-secret-key
-_APP_EXECUTOR_HOST=http://appwrite-executor/v1
+_APP_FUNCTIONS_PROXY_SECRET=your-secret-key
+_APP_FUNCTIONS_PROXY_HOST=http://appwrite-functions-proxy/v1
 _APP_MAINTENANCE_INTERVAL=86400
 _APP_MAINTENANCE_RETENTION_CACHE=2592000
 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600
@@ -83,4 +84,7 @@ _APP_LOGGING_PROVIDER=
 _APP_LOGGING_CONFIG=
 DOCKERHUB_PULL_USERNAME=
 DOCKERHUB_PULL_PASSWORD=
-DOCKERHUB_PULL_EMAIL=
+DOCKERHUB_PULL_EMAIL=
+_APP_FUNCTIONS_PROXY_ADAPTER=round-robin
+_APP_EXECUTORS=exc1,exc2
+# _APP_EXECUTORS=exc1

Dockerfile

@@ -241,7 +241,10 @@ ENV _APP_SERVER=swoole \
     _APP_FUNCTIONS_MEMORY=128 \
     _APP_FUNCTIONS_MEMORY_SWAP=128 \
     _APP_EXECUTOR_SECRET=a-random-secret \
-    _APP_EXECUTOR_HOST=http://appwrite-executor/v1 \
+    _APP_FUNCTIONS_PROXY_SECRET=a-random-secret \
+    _APP_FUNCTIONS_PROXY_HOST=http://appwrite-functions-proxy/v1 \
+    _APP_EXECUTORS=exc1 \
+    _APP_FUNCTIONS_PROXY_ADAPTER=round-robin \
     _APP_EXECUTOR_RUNTIME_NETWORK=appwrite_runtimes \
     _APP_SETUP=self-hosted \
     _APP_VERSION=$VERSION \
@@ -337,6 +340,7 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/migrate && \
     chmod +x /usr/local/bin/realtime && \
     chmod +x /usr/local/bin/executor && \
+    chmod +x /usr/local/bin/functions-proxy && \
     chmod +x /usr/local/bin/schedule && \
     chmod +x /usr/local/bin/sdks && \
     chmod +x /usr/local/bin/specs && \

app/config/variables.php

@@ -728,7 +728,7 @@ return [
             ],
             [
                 'name' => '_APP_EXECUTOR_SECRET',
-                'description' => 'The secret key used by Appwrite to communicate with the function executor. Make sure to change this!',
+                'description' => 'The secret key used by Appwrite functions proxy to communicate with the function executor. Make sure to change this!',
                 'introduction' => '0.13.0',
                 'default' => 'your-secret-key',
                 'required' => false,
@@ -737,10 +737,9 @@ return [
             ],
             [
                 'name' => '_APP_EXECUTOR_HOST',
-                'description' => 'The host used by Appwrite to communicate with the function executor!',
+                'description' => 'Deprecated since 1.2.0, use `_APP_FUNCTIONS_PROXY_HOST` and `_APP_EXECUTORS` instead!',
                 'introduction' => '0.13.0',
                 'default' => 'http://appwrite-executor/v1',
-                'required' => false,
                 'question' => '',
                 'filter' => ''
             ],
@@ -807,6 +806,42 @@ return [
                 'question' => '',
                 'filter' => ''
             ],
+            [
+                'name' => '_APP_FUNCTIONS_PROXY_HOST',
+                'description' => 'The host used by Appwrite to communicate with the functions proxy to execute a function.',
+                'introduction' => '1.2.0',
+                'default' => 'http://appwrite-functions-proxy/v1',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => '_APP_FUNCTIONS_PROXY_SECRET',
+                'description' => 'The secret key used by Appwrite containers to communicate with functions proxy. Make sure to change this!',
+                'introduction' => '1.2.0',
+                'default' => 'your-secret-key',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => '_APP_EXECUTORS',
+                'description' => 'List of Appwrite executor hostnames to use for executing functions separated by comma.',
+                'introduction' => '1.2.0',
+                'default' => 'exc1',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => '_APP_FUNCTIONS_PROXY_ADAPTER',
+                'description' => 'Select functions proxy adapter depending if performance or efficiency is more important. The default value is \'round-robin\'. List of supported adapters are \'round-robin\', \'usage-based\'.',
+                'introduction' => '1.2.0',
+                'default' => 'round-robin',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
         ],
     ],
     [

app/controllers/api/functions.php

@@ -1145,7 +1145,7 @@ App::post('/v1/functions/:functionId/executions')
         ]);
 
         /** Execute function */
-        $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
+        $executor = new Executor(App::getEnv('_APP_FUNCTIONS_PROXY_HOST'));
         try {
             $executionResponse = $executor->createExecution(
                 projectId: $project->getId(),

app/executor.php

@@ -3,6 +3,7 @@
 require_once __DIR__ . '/../vendor/autoload.php';
 
 use Appwrite\Runtimes\Runtimes;
+use Executor\Executor;
 use Swoole\ConnectionPool;
 use Swoole\Http\Request as SwooleRequest;
 use Swoole\Http\Response as SwooleResponse;
@@ -13,6 +14,7 @@ use Swoole\Timer;
 use Utopia\App;
 use Utopia\CLI\Console;
 use Utopia\Database\DateTime;
+use Utopia\Database\ID;
 use Utopia\Logger\Log;
 use Utopia\Logger\Logger;
 use Utopia\Orchestration\Adapter\DockerCLI;
@@ -27,12 +29,14 @@ use Utopia\Storage\Device\S3;
 use Utopia\Storage\Storage;
 use Utopia\Swoole\Request;
 use Utopia\Swoole\Response;
+use Utopia\System\System;
 use Utopia\Validator\ArrayList;
 use Utopia\Validator\Assoc;
 use Utopia\Validator\Boolean;
 use Utopia\Validator\Range;
 use Utopia\Validator\Text;
 
+use function Swoole\Coroutine\batch;
 
 Runtime::enableCoroutine(true, SWOOLE_HOOK_ALL);
 
@@ -46,7 +50,8 @@ $activeRuntimes = new Swoole\Table(1024);
 $activeRuntimes->column('id', Swoole\Table::TYPE_STRING, 256);
 $activeRuntimes->column('created', Swoole\Table::TYPE_INT, 8);
 $activeRuntimes->column('updated', Swoole\Table::TYPE_INT, 8);
-$activeRuntimes->column('name', Swoole\Table::TYPE_STRING, 128);
+$activeRuntimes->column('name', Swoole\Table::TYPE_STRING, 256);
+$activeRuntimes->column('hostname', Swoole\Table::TYPE_STRING, 256);
 $activeRuntimes->column('status', Swoole\Table::TYPE_STRING, 128);
 $activeRuntimes->column('key', Swoole\Table::TYPE_STRING, 256);
 $activeRuntimes->create();
@@ -177,8 +182,13 @@ App::post('/v1/runtimes')
     ->inject('activeRuntimes')
     ->inject('response')
     ->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $baseImage, string $entrypoint, bool $remove, string $workdir, $orchestrationPool, $activeRuntimes, Response $response) {
-        if ($activeRuntimes->exists($runtimeId)) {
-            if ($activeRuntimes->get($runtimeId)['status'] == 'pending') {
+        $activeRuntimeId = $runtimeId; // Used with Swoole table (key)
+        $runtimeId = System::getHostname() . '-' . $runtimeId; // Used in Docker (name)
+
+        $runtimeHostname = ID::unique();
+
+        if ($activeRuntimes->exists($activeRuntimeId)) {
+            if ($activeRuntimes->get($activeRuntimeId)['status'] == 'pending') {
                 throw new \Exception('A runtime with the same ID is already being created. Attempt a execution soon.', 500);
             }
 
@@ -197,9 +207,10 @@ App::post('/v1/runtimes')
         $secret = \bin2hex(\random_bytes(16));
 
         if (!$remove) {
-            $activeRuntimes->set($runtimeId, [
+            $activeRuntimes->set($activeRuntimeId, [
                 'id' => $containerId,
                 'name' => $runtimeId,
+                'hostname' => $runtimeHostname,
                 'created' => $startTimeUnix,
                 'updated' => $endTimeUnix,
                 'status' => 'pending',
@@ -241,6 +252,7 @@ App::post('/v1/runtimes')
             $vars = \array_merge($vars, [
                 'INTERNAL_RUNTIME_KEY' => $secret,
                 'INTERNAL_RUNTIME_ENTRYPOINT' => $entrypoint,
+                'INERNAL_EXECUTOR_HOSTNAME' => System::getHostname()
             ]);
             $vars = array_map(fn ($v) => strval($v), $vars);
             $orchestration
@@ -258,14 +270,16 @@ App::post('/v1/runtimes')
             $containerId = $orchestration->run(
                 image: $baseImage,
                 name: $runtimeId,
-                hostname: $runtimeId,
+                hostname: $runtimeHostname,
                 vars: $vars,
                 command: $entrypoint,
                 labels: [
                     'openruntimes-id' => $runtimeId,
+                    'openruntimes-hostname' => $runtimeHostname,
                     'openruntimes-type' => 'runtime',
+                    'openruntimes-executor' => System::getHostname(),
                     'openruntimes-created' => strval($startTimeUnix),
-                    'openruntimes-runtime' => $runtime,
+                    'openruntimes-runtime' => $runtime
                 ],
                 workdir: $workdir,
                 volumes: [
@@ -334,11 +348,11 @@ App::post('/v1/runtimes')
                 'duration' => $duration,
             ]);
 
-
             if (!$remove) {
-                $activeRuntimes->set($runtimeId, [
+                $activeRuntimes->set($activeRuntimeId, [
                     'id' => $containerId,
                     'name' => $runtimeId,
+                    'hostname' => $runtimeHostname,
                     'created' => $startTimeUnix,
                     'updated' => $endTimeUnix,
                     'status' => 'Up ' . \round($duration, 2) . 's',
@@ -357,13 +371,13 @@ App::post('/v1/runtimes')
                 if (!empty($containerId)) {
                     // If container properly created
                     $orchestration->remove($containerId, true);
-                    $activeRuntimes->del($runtimeId);
+                    $activeRuntimes->del($activeRuntimeId);
                 } else {
                     // If whole creation failed, but container might have been initialized
                     try {
                         // Try to remove with contaier name instead of ID
                         $orchestration->remove($runtimeId, true);
-                        $activeRuntimes->del($runtimeId);
+                        $activeRuntimes->del($activeRuntimeId);
                     } catch (Throwable $th) {
                         // If fails, means initialization also failed.
                         // Contianer is not there, no need to remove
@@ -403,12 +417,13 @@ App::get('/v1/runtimes/:runtimeId')
     ->inject('activeRuntimes')
     ->inject('response')
     ->action(function ($runtimeId, $activeRuntimes, Response $response) {
+        $activeRuntimeId = $runtimeId; // Used with Swoole table (key)
 
-        if (!$activeRuntimes->exists($runtimeId)) {
+        if (!$activeRuntimes->exists($activeRuntimeId)) {
             throw new Exception('Runtime not found', 404);
         }
 
-        $runtime = $activeRuntimes->get($runtimeId);
+        $runtime = $activeRuntimes->get($activeRuntimeId);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_OK)
@@ -422,8 +437,10 @@ App::delete('/v1/runtimes/:runtimeId')
     ->inject('activeRuntimes')
     ->inject('response')
     ->action(function (string $runtimeId, $orchestrationPool, $activeRuntimes, Response $response) {
+        $activeRuntimeId = $runtimeId; // Used with Swoole table (key)
+        $runtimeId = System::getHostname() . '-' . $runtimeId; // Used in Docker (name)
 
-        if (!$activeRuntimes->exists($runtimeId)) {
+        if (!$activeRuntimes->exists($activeRuntimeId)) {
             throw new Exception('Runtime not found', 404);
         }
 
@@ -432,7 +449,7 @@ App::delete('/v1/runtimes/:runtimeId')
         try {
             $orchestration = $orchestrationPool->get();
             $orchestration->remove($runtimeId, true);
-            $activeRuntimes->del($runtimeId);
+            $activeRuntimes->del($activeRuntimeId);
             Console::success('Removed runtime container: ' . $runtimeId);
         } finally {
             $orchestrationPool->put($orchestration);
@@ -457,22 +474,62 @@ App::delete('/v1/runtimes/:runtimeId')
 
 App::post('/v1/execution')
     ->desc('Create an execution')
+    // execution-related
     ->param('runtimeId', '', new Text(64), 'The runtimeID to execute.')
-    ->param('vars', [], new Assoc(), 'Environment variables required for the build.')
+    ->param('vars', [], new Assoc(), 'Environment variables required for the build and execution.')
     ->param('data', '', new Text(8192), 'Data to be forwarded to the function, this is user specified.', true)
     ->param('timeout', 15, new Range(1, (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)), 'Function maximum execution time in seconds.')
+    // runtime-related
+    ->param('source', '', new Text(0), 'Path to source files.')
+    ->param('runtime', '', new Text(128), 'Runtime for the cloud function.')
+    ->param('baseImage', '', new Text(128), 'Base image name of the runtime.')
+    ->param('entrypoint', '', new Text(256), 'Entrypoint of the code file.', true)
     ->inject('activeRuntimes')
     ->inject('response')
     ->action(
-        function (string $runtimeId, array $vars, string $data, $timeout, $activeRuntimes, Response $response) {
-            if (!$activeRuntimes->exists($runtimeId)) {
-                throw new Exception('Runtime not found. Please create the runtime.', 404);
+        function (string $runtimeId, array $vars, string $data, $timeout, string $source, string $runtime, string $baseImage, string $entrypoint, $activeRuntimes, Response $response) {
+            $activeRuntimeId = $runtimeId; // Used with Swoole table (key)
+            $originalRuntimeId = $runtimeId; // Used in Docker (createRuntime request)
+            $runtimeId = System::getHostname() . '-' . $runtimeId; // Used in Docker (name)
+
+            $vars = \array_merge($vars, [
+                'INERNAL_EXECUTOR_HOSTNAME' => System::getHostname()
+            ]);
+
+            // Prepare runtime
+            if (!$activeRuntimes->exists($activeRuntimeId)) {
+                $executor = new Executor('http://localhost/v1');
+
+                for ($i = 0; $i < 5; $i++) {
+                    try {
+                        [ $projectId, $deploymentId ] = \explode('-', $originalRuntimeId);
+                        $runtimeResponse = $executor->createRuntime(
+                            projectId: $projectId,
+                            deploymentId: $deploymentId,
+                            source: $source,
+                            runtime: $runtime,
+                            baseImage: $baseImage,
+                            vars: $vars,
+                            entrypoint: $entrypoint,
+                            commands: [],
+                            key: App::getEnv('_APP_EXECUTOR_SECRET', '')
+                        );
+
+                        break;
+                    } catch (\Exception $error) {
+                        if ($i === 4) {
+                            throw new Exception('Runtime could not be created in allocated time: ' . $error->getMessage(), 500);
+                        }
+
+                        \sleep(1);
+                    }
+                }
             }
 
+            // Ensure runtime started
             for ($i = 0; $i < 5; $i++) {
-                if ($activeRuntimes->get($runtimeId)['status'] === 'pending') {
+                if ($activeRuntimes->get($activeRuntimeId)['status'] === 'pending') {
                     Console::info('Waiting for runtime to be ready...');
-                    sleep(1);
                 } else {
                     break;
                 }
@@ -480,9 +537,13 @@ App::post('/v1/execution')
                 if ($i === 4) {
                     throw new Exception('Runtime failed to launch in allocated time.', 500);
                 }
+
+                \sleep(1);
             }
 
-            $runtime = $activeRuntimes->get($runtimeId);
+            // Ensure we have secret
+            $runtime = $activeRuntimes->get($activeRuntimeId);
+            $hostname = $runtime['hostname'];
             $secret = $runtime['key'];
             if (empty($secret)) {
                 throw new Exception('Runtime secret not found. Please re-create the runtime.', 500);
@@ -490,75 +551,103 @@ App::post('/v1/execution')
 
             Console::info('Executing Runtime: ' . $runtimeId);
 
-            $execution = [];
             $executionStart = \microtime(true);
+
+            // Prepare request to executor
+            $sendExecuteRequest = function () use ($vars, $data, $secret, $hostname, &$executionStart, &$timeout) {
+                // Restart execution timer to not could failed attempts
+                $executionStart = \microtime(true);
+
+                $statusCode = 0;
+                $errNo = -1;
+                $executorResponse = '';
+
+                $timeout ??= (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900);
+
+                $ch = \curl_init();
+                $body = \json_encode([
+                    'variables' => $vars,
+                    'payload' => $data,
+                    'timeout' => $timeout
+                ]);
+
+                \curl_setopt($ch, CURLOPT_URL, "http://" . $hostname . ":3000/");
+                \curl_setopt($ch, CURLOPT_POST, true);
+                \curl_setopt($ch, CURLOPT_POSTFIELDS, $body);
+                \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+                \curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
+                \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
+
+                \curl_setopt($ch, CURLOPT_HTTPHEADER, [
+                    'Content-Type: application/json',
+                    'Content-Length: ' . \strlen($body),
+                    'x-internal-challenge: ' . $secret,
+                    'host: null'
+                ]);
+
+                $executorResponse = \curl_exec($ch);
+
+                $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE);
+
+                $error = \curl_error($ch);
+
+                $errNo = \curl_errno($ch);
+
+                \curl_close($ch);
+
+                return [
+                    'errNo' => $errNo,
+                    'error' => $error,
+                    'statusCode' => $statusCode,
+                    'executorResponse' => $executorResponse
+                ];
+            };
+
+            // Execute function
+            for ($i = 0; $i < 5; $i++) {
+                [ 'errNo' => $errNo, 'error' => $error, 'statusCode' => $statusCode, 'executorResponse' => $executorResponse ] = \call_user_func($sendExecuteRequest);
+
+                // No error
+                if ($errNo === 0) {
+                    break;
+                }
+
+                if ($errNo !== 111) { // Connection Refused - see https://openswoole.com/docs/swoole-error-code
+                    throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 500);
+                }
+
+                Console::info('Waiting for runtime to respond...');
+
+                if ($i === 4) {
+                    throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 500);
+                }
+
+                \sleep(1);
+            }
+
+            // Extract response
+            $execution = [];
             $stdout = '';
             $stderr = '';
             $res = '';
-            $statusCode = 0;
-            $errNo = -1;
-            $executorResponse = '';
 
-            $timeout ??= (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900);
-
-            $ch = \curl_init();
-            $body = \json_encode([
-                'variables' => $vars,
-                'payload' => $data,
-                'timeout' => $timeout
-            ]);
-            \curl_setopt($ch, CURLOPT_URL, "http://" . $runtimeId . ":3000/");
-            \curl_setopt($ch, CURLOPT_POST, true);
-            \curl_setopt($ch, CURLOPT_POSTFIELDS, $body);
-            \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
-            \curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
-            \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
-
-            \curl_setopt($ch, CURLOPT_HTTPHEADER, [
-                'Content-Type: application/json',
-                'Content-Length: ' . \strlen($body),
-                'x-internal-challenge: ' . $secret,
-                'host: null'
-            ]);
-
-            $executorResponse = \curl_exec($ch);
-            $executorResponse = json_decode($executorResponse, true);
-
-            $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE);
-
-            $error = \curl_error($ch);
-
-            $errNo = \curl_errno($ch);
-
-            \curl_close($ch);
-
-            switch (true) {
-                /** No Error. */
-                case $errNo === 0:
-                    break;
-                /** Runtime not ready for requests yet. 111 is the swoole error code for Connection Refused - see https://openswoole.com/docs/swoole-error-code */
-                case $errNo === 111:
-                    throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 406);
-                /** Any other CURL error */
-                default:
-                    throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 500);
-            }
+            $executorResponse = json_decode($executorResponse ?? '{}', true);
 
             switch (true) {
                 case $statusCode >= 500:
-                    $stderr = ($executorResponse ?? [])['stderr'] ?? 'Internal Runtime error.';
-                    $stdout = ($executorResponse ?? [])['stdout'] ?? 'Internal Runtime error.';
+                    $stderr = $executorResponse['stderr'] ?? '';
+                    $stdout = $executorResponse['stdout'] ?? '';
                     break;
                 case $statusCode >= 100:
-                    $stdout = $executorResponse['stdout'];
-                    $res = $executorResponse['response'];
+                    $stdout = $executorResponse['stdout'] ?? '';
+                    $res = $executorResponse['response'] ?? '';
                     if (is_array($res)) {
                         $res = json_encode($res, JSON_UNESCAPED_UNICODE);
                     }
                     break;
                 default:
-                    $stderr = ($executorResponse ?? [])['stderr'] ?? 'Execution failed.';
-                    $stdout = ($executorResponse ?? [])['stdout'] ?? '';
+                    $stderr = $executorResponse['stderr'] ?? '';
+                    $stdout = $executorResponse['stdout'] ?? '';
                     break;
             }
 
@@ -574,13 +663,15 @@ App::post('/v1/execution')
                 'response' => \mb_strcut($res, 0, 1000000), // Limit to 1MB
                 'stdout' => \mb_strcut($stdout, 0, 1000000), // Limit to 1MB
                 'stderr' => \mb_strcut($stderr, 0, 1000000), // Limit to 1MB
-                'duration' => $executionTime,
+                'duration' => $executionTime, // TODO: @Meldiron This is not timeout, but timeout + 2 (probably coming from Executor.php client)
             ];
 
-            /** Update swoole table */
+            // Update swoole table
+            $runtime = $activeRuntimes->get($activeRuntimeId);
             $runtime['updated'] = \time();
-            $activeRuntimes->set($runtimeId, $runtime);
+            $activeRuntimes->set($activeRuntimeId, $runtime);
 
+            // Finish request
             $response
                 ->setStatusCode(Response::STATUS_CODE_OK)
                 ->json($execution);
@@ -591,6 +682,21 @@ App::setMode(App::MODE_TYPE_PRODUCTION); // Define Mode
 
 $http = new Server("0.0.0.0", 80);
 
+$payloadSize = 6 * (1024 * 1024); // 6MB
+$workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
+
+$http
+    ->set([
+        'worker_num' => $workerNumber,
+        'open_http2_protocol' => true,
+        // 'document_root' => __DIR__.'/../public',
+        // 'enable_static_handler' => true,
+        'http_compression' => true,
+        'http_compression_level' => 6,
+        'package_max_length' => $payloadSize,
+        'buffer_output_size' => $payloadSize,
+    ]);
+
 /** Set Resources */
 App::setResource('orchestrationPool', fn() => $orchestrationPool);
 App::setResource('activeRuntimes', fn() => $activeRuntimes);
@@ -660,6 +766,83 @@ $http->on('start', function ($http) {
     global $orchestrationPool;
     global $activeRuntimes;
 
+    /**
+     * Start separate HTTP server for Health API
+     */
+    \go(function () use ($orchestrationPool) {
+        $healthServer = new Swoole\Coroutine\Http\Server('0.0.0.0', 3000, false);
+
+        // Get usage stats of host machine CPU usage from 0 to 100.
+        $healthServer->handle('/v1/health', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($orchestrationPool) {
+            $orchestration = null;
+
+            try {
+                $secretKey = $swooleRequest->header['x-appwrite-executor-key'] ?? '';
+                if (empty($secretKey)) {
+                    throw new Exception('Missing executor key', 401);
+                }
+
+                if ($secretKey !== App::getEnv('_APP_EXECUTOR_SECRET', '')) {
+                    throw new Exception('Missing executor key', 401);
+                }
+
+
+                $output = [
+                    'status' => 'pass'
+                ];
+
+                batch([
+                    function () use (&$output) {
+                        $output['hostUsage'] = System::getCPUUsage(5);
+                    },
+                    function () use (&$output, &$orchestration, $orchestrationPool) {
+                        $functionsUsage = [];
+
+                        $orchestration = $orchestrationPool->get();
+                        $containerUsages = $orchestration->getStats(
+                            filters: [ 'label' => 'openruntimes-executor=' . System::getHostname() ],
+                            cycles: 3
+                        );
+
+                        foreach ($containerUsages as $containerUsage) {
+                            $functionsUsage[$containerUsage['name']] = $containerUsage['cpu'] * 100;
+                        }
+
+                        $output['functionsUsage'] = $functionsUsage;
+                    }
+                ]);
+
+                $swooleResponse->setStatusCode(200);
+                $swooleResponse->header('content-type', 'application/json; charset=UTF-8');
+                $swooleResponse->write(\json_encode($output));
+                $swooleResponse->end();
+            } catch (\Throwable $th) {
+                logError($th, "healthError");
+
+                $output = [
+                    'status' => 'fail',
+
+                    'message' => 'Error: ' . $th->getMessage(),
+                    'code' => 500,
+                    'file' => $th->getFile(),
+                    'line' => $th->getLine(),
+                    'trace' => $th->getTrace()
+                ];
+
+                $swooleResponse->setStatusCode(500);
+                $swooleResponse->header('content-type', 'application/json; charset=UTF-8');
+                $swooleResponse->write(\json_encode($output));
+                $swooleResponse->end();
+            } finally {
+                if ($orchestration !== null) {
+                    $orchestrationPool->put($orchestration);
+                }
+            }
+        });
+
+        $healthServer->start();
+    });
+
     /**
      * Warmup: make sure images are ready to run fast 🚀
      */
@@ -690,7 +873,7 @@ $http->on('start', function ($http) {
     Console::info('Removing orphan runtimes...');
     try {
         $orchestration = $orchestrationPool->get();
-        $orphans = $orchestration->list(['label' => 'openruntimes-type=runtime']);
+        $orphans = $orchestration->list(['label' => 'openruntimes-executor=' . System::getHostname()]);
     } finally {
         $orchestrationPool->put($orchestration);
     }
@@ -733,14 +916,14 @@ $http->on('start', function ($http) {
      */
     Timer::tick(MAINTENANCE_INTERVAL * 1000, function () use ($orchestrationPool, $activeRuntimes) {
         Console::warning("Running maintenance task ...");
-        foreach ($activeRuntimes as $runtime) {
+        foreach ($activeRuntimes as $activeRuntimeId => $runtime) {
             $inactiveThreshold = \time() - App::getEnv('_APP_FUNCTIONS_INACTIVE_THRESHOLD', 60);
             if ($runtime['updated'] < $inactiveThreshold) {
-                go(function () use ($runtime, $orchestrationPool, $activeRuntimes) {
+                go(function () use ($activeRuntimeId, $runtime, $orchestrationPool, $activeRuntimes) {
                     try {
                         $orchestration = $orchestrationPool->get();
                         $orchestration->remove($runtime['name'], true);
-                        $activeRuntimes->del($runtime['name']);
+                        $activeRuntimes->del($activeRuntimeId);
                         Console::success("Successfully removed {$runtime['name']}");
                     } catch (\Throwable $th) {
                         Console::error('Inactive Runtime deletion failed: ' . $th->getMessage());

app/functionsProxy.php

@@ -0,0 +1,264 @@
+<?php
+
+require_once __DIR__ . '/../vendor/autoload.php';
+
+use FunctionsProxy\Adapter\RoundRobin;
+use FunctionsProxy\Adapter\UsageBased;
+use Swoole\Coroutine\Http\Client;
+use Utopia\Logger\Log;
+use Utopia\Logger\Logger;
+use Swoole\Coroutine\Http\Server;
+use function Swoole\Coroutine\run;
+use Swoole\Database\RedisConfig;
+use Swoole\Database\RedisPool;
+use Swoole\Http\Request as SwooleRequest;
+use Swoole\Http\Response as SwooleResponse;
+use Swoole\Runtime;
+use Swoole\Timer;
+use Utopia\App;
+use Utopia\Cache\Adapter\Redis;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+
+Runtime::enableCoroutine(true, SWOOLE_HOOK_ALL);
+
+// Redis setup
+$redisHost = App::getEnv('_APP_REDIS_HOST', '');
+$redisPort = App::getEnv('_APP_REDIS_PORT', '');
+$redisUser = App::getEnv('_APP_REDIS_USER', '');
+$redisPass = App::getEnv('_APP_REDIS_PASS', '');
+$redisAuth = '';
+
+if ($redisUser && $redisPass) {
+    $redisAuth = $redisUser . ':' . $redisPass;
+}
+
+$redisPool = new RedisPool(
+    (new RedisConfig())
+        ->withHost($redisHost)
+        ->withPort($redisPort)
+        ->withAuth($redisAuth)
+        ->withDbIndex(0),
+    64
+);
+
+$adapterType = App::getEnv('_APP_FUNCTIONS_PROXY_ADAPTER', '');
+$adapter = match ($adapterType) {
+    'round-robin' => new RoundRobin($redisPool),
+    'usage-based' => new UsageBased($redisPool),
+    default => new RoundRobin($redisPool)
+};
+
+Console::info("Using adpater: " . $adapterType);
+
+function markOffline(Cache $cache, string $executorHostname, string $error, bool $forceShowError = false)
+{
+    $data = $cache->load('executors-' . $executorHostname, 60 * 60 * 24 * 30 * 3); // 3 months
+
+    $cache->save('executors-' . $executorHostname, ['status' => 'offline', 'health' => []]);
+
+    if (!$data || $data['status'] === 'online' || $forceShowError) {
+        Console::warning('Executor "' . $executorHostname . '" went down! Message:');
+        Console::warning($error);
+    }
+}
+
+function markOnline(cache $cache, string $executorHostname, bool $forceShowError = false, mixed $health = [])
+{
+    $data = $cache->load('executors-' . $executorHostname, 60 * 60 * 24 * 30 * 3); // 3 months
+
+    $cache->save('executors-' . $executorHostname, ['status' => 'online', 'health' => $health]);
+
+    if (!$data || $data['status'] === 'offline' || $forceShowError) {
+        Console::success('Executor "' . $executorHostname . '" went online.');
+    }
+}
+
+// Fetch info about executors
+function fetchExecutorsState(RedisPool $redisPool, bool $forceShowError = false)
+{
+    $executors = \explode(',', App::getEnv('_APP_EXECUTORS', ''));
+
+    foreach ($executors as $executor) {
+        go(function () use ($redisPool, $executor, $forceShowError) {
+            $redis = $redisPool->get();
+            $cache = new Cache(new Redis($redis));
+
+            try {
+                $endpoint = 'http://' . $executor . ':3000/v1/health';
+
+                $ch = \curl_init();
+
+                \curl_setopt($ch, CURLOPT_URL, $endpoint);
+                \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+                \curl_setopt($ch, CURLOPT_TIMEOUT, 10);
+                \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
+                \curl_setopt($ch, CURLOPT_HTTPHEADER, [
+                    'Content-Type: application/json',
+                    'x-appwrite-executor-key: ' . App::getEnv('_APP_EXECUTOR_SECRET', '')
+                ]);
+
+                $executorResponse = \curl_exec($ch);
+                $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE);
+                $error = \curl_error($ch);
+
+                \curl_close($ch);
+
+                if ($statusCode === 200) {
+                    markOnline($cache, $executor, $forceShowError, \json_decode($executorResponse, true));
+                } else {
+                    $message = 'Code: ' . $statusCode . ' with response "' . $executorResponse .  '" and error error: ' . $error;
+                    markOffline($cache, $executor, $message, $forceShowError);
+                }
+            } catch (\Exception $err) {
+                throw $err;
+            } finally {
+                $redisPool->put($redis);
+            }
+        });
+    }
+}
+
+/**
+ * Create logger instance
+ */
+$providerName = App::getEnv('_APP_LOGGING_PROVIDER', '');
+$providerConfig = App::getEnv('_APP_LOGGING_CONFIG', '');
+$logger = null;
+
+if (!empty($providerName) && !empty($providerConfig) && Logger::hasProvider($providerName)) {
+    $classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName);
+    $adapter = new $classname($providerConfig);
+    $logger = new Logger($adapter);
+}
+
+function logError(Throwable $error, string $action, Utopia\Route $route = null)
+{
+    global $logger;
+
+    if ($logger) {
+        $version = App::getEnv('_APP_VERSION', 'UNKNOWN');
+
+        $log = new Log();
+        $log->setNamespace("executor");
+        $log->setServer(\gethostname());
+        $log->setVersion($version);
+        $log->setType(Log::TYPE_ERROR);
+        $log->setMessage($error->getMessage());
+
+        if ($route) {
+            $log->addTag('method', $route->getMethod());
+            $log->addTag('url', $route->getPath());
+        }
+
+        $log->addTag('code', $error->getCode());
+        $log->addTag('verboseType', get_class($error));
+
+        $log->addExtra('file', $error->getFile());
+        $log->addExtra('line', $error->getLine());
+        $log->addExtra('trace', $error->getTraceAsString());
+        $log->addExtra('detailedTrace', $error->getTrace());
+
+        $log->setAction($action);
+
+        $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
+        $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
+
+        $responseCode = $logger->addLog($log);
+        Console::info('Executor log pushed with status code: ' . $responseCode);
+    }
+
+    Console::error('[Error] Type: ' . get_class($error));
+    Console::error('[Error] Message: ' . $error->getMessage());
+    Console::error('[Error] File: ' . $error->getFile());
+    Console::error('[Error] Line: ' . $error->getLine());
+}
+
+Console::success("Waiting for executors to start...");
+
+\sleep(5); // Wait a little so executors can start
+
+fetchExecutorsState($redisPool, true);
+
+Console::log("State of executors at startup:");
+
+go(function () use ($redisPool) {
+    $executors = \explode(',', App::getEnv('_APP_EXECUTORS', ''));
+
+    $redis = $redisPool->get();
+    $cache = new Cache(new Redis($redis));
+
+    foreach ($executors as $executor) {
+        $data = $cache->load('executors-' . $executor, 60 * 60 * 24 * 30 * 3); // 3 months
+        Console::log('Executor ' . $executor . ' is ' . ($data['status'] ?? 'unknown') . '.');
+    }
+});
+
+Swoole\Event::wait();
+
+$run = function (SwooleRequest $request, SwooleResponse $response) use ($adapter) {
+    $secretKey = $request->header['x-appwrite-executor-key'] ?? '';
+
+    if (empty($secretKey)) {
+        throw new Exception('Missing proxy key');
+    }
+    if ($secretKey !== App::getEnv('_APP_FUNCTIONS_PROXY_SECRET', '')) {
+        throw new Exception('Missing proxy key');
+    }
+
+    $body = \json_decode($request->getContent(), true);
+    $runtimeId = $body['runtimeId'] ?? null;
+    $executor = $adapter->getNextExecutor($runtimeId);
+
+    Console::success("Executing on " . $executor['hostname']);
+
+    $client = new Client($executor['hostname'], 80);
+    $client->setMethod($request->server['request_method'] ?? 'GET');
+    $client->setHeaders(\array_merge($request->header, [
+        'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '')
+    ]));
+    $client->setData($request->getContent());
+
+    $status = $client->execute($request->server['request_uri'] ?? '/');
+
+    $response->setStatusCode($client->getStatusCode());
+    $response->header('content-type', 'application/json; charset=UTF-8');
+    $response->write($client->getBody());
+    $response->end();
+};
+
+run(function () use ($redisPool, $run) {
+    // TODO: @Meldiron Allow scaling. Only do this on one machine, or only worry about executors on my host machine
+
+    // Keep updating executors state
+    Timer::tick(10000, function (int $timerId) use ($redisPool) {
+        fetchExecutorsState($redisPool, false);
+    });
+
+    $server = new Server('0.0.0.0', 80, false);
+
+    $server->handle('/', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($run) {
+        try {
+            call_user_func($run, $swooleRequest, $swooleResponse);
+        } catch (\Throwable $th) {
+            logError($th, "serverError");
+
+            $output = [
+                'message' => 'Error: ' . $th->getMessage(),
+                'code' => 500,
+                'file' => $th->getFile(),
+                'line' => $th->getLine(),
+                'trace' => $th->getTrace()
+            ];
+
+            $swooleResponse->setStatusCode(500);
+            $swooleResponse->header('content-type', 'application/json; charset=UTF-8');
+            $swooleResponse->write(\json_encode($output));
+            $swooleResponse->end();
+        }
+    });
+
+    Console::success("Functions proxy is ready.");
+
+    $server->start();
+});

app/views/install/compose.phtml

@@ -139,8 +139,8 @@ services:
       - _APP_FUNCTIONS_MEMORY
       - _APP_FUNCTIONS_MEMORY_SWAP
       - _APP_FUNCTIONS_RUNTIMES
-      - _APP_EXECUTOR_SECRET
-      - _APP_EXECUTOR_HOST
+      - _APP_FUNCTIONS_PROXY_SECRET
+      - _APP_FUNCTIONS_PROXY_HOST
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
       - _APP_STATSD_HOST
@@ -295,8 +295,8 @@ services:
       - _APP_STORAGE_WASABI_BUCKET
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
-      - _APP_EXECUTOR_SECRET
-      - _APP_EXECUTOR_HOST
+      - _APP_FUNCTIONS_PROXY_SECRET
+      - _APP_FUNCTIONS_PROXY_HOST
 
   appwrite-worker-databases:
     image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
@@ -338,8 +338,8 @@ services:
     environment:
       - _APP_ENV
       - _APP_OPENSSL_KEY_V1
-      - _APP_EXECUTOR_SECRET
-      - _APP_EXECUTOR_HOST
+      - _APP_FUNCTIONS_PROXY_SECRET
+      - _APP_FUNCTIONS_PROXY_HOST
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -384,6 +384,27 @@ services:
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
+  appwrite-functions-proxy:
+    image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
+    entrypoint: functions-proxy
+    <<: *x-logging
+    container_name: appwrite-functions-proxy
+    restart: unless-stopped
+    networks:
+      - appwrite
+    depends_on:
+      - redis
+    environment:
+      - _APP_ENV
+      - _APP_FUNCTIONS_PROXY_ADAPTER
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_EXECUTORS
+      - _APP_EXECUTOR_SECRET
+      - _APP_FUNCTIONS_PROXY_SECRET
+
   appwrite-worker-functions:
     image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
     entrypoint: worker-functions
@@ -395,7 +416,6 @@ services:
     depends_on:
       - redis
       - mariadb
-      - appwrite-executor
     environment:
       - _APP_ENV
       - _APP_OPENSSL_KEY_V1
@@ -409,8 +429,8 @@ services:
       - _APP_DB_USER
       - _APP_DB_PASS
       - _APP_FUNCTIONS_TIMEOUT
-      - _APP_EXECUTOR_SECRET
-      - _APP_EXECUTOR_HOST
+      - _APP_FUNCTIONS_PROXY_SECRET
+      - _APP_FUNCTIONS_PROXY_HOST
       - _APP_USAGE_STATS
       - DOCKERHUB_PULL_USERNAME
       - DOCKERHUB_PULL_PASSWORD
@@ -420,6 +440,7 @@ services:
     entrypoint: executor
     <<: *x-logging
     container_name: appwrite-executor
+    hostname: exc1
     restart: unless-stopped
     stop_signal: SIGINT
     networks:
@@ -437,6 +458,7 @@ services:
     environment:
       - _APP_ENV
       - _APP_VERSION
+      - _APP_WORKER_PER_CORE
       - _APP_FUNCTIONS_TIMEOUT
       - _APP_FUNCTIONS_BUILD_TIMEOUT
       - _APP_FUNCTIONS_CONTAINERS
@@ -707,4 +729,3 @@ volumes:
   appwrite-builds:
   appwrite-influxdb:
   appwrite-config:
-  appwrite-executor:

app/workers/builds.php

@@ -34,7 +34,7 @@ class BuildsV1 extends Worker
 
     public function init(): void
     {
-        $this->executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
+        $this->executor = new Executor(App::getEnv('_APP_FUNCTIONS_PROXY_HOST'));
     }
 
     public function run(): void
@@ -203,6 +203,7 @@ class BuildsV1 extends Worker
             $build->setAttribute('status', 'failed');
             $build->setAttribute('stderr', $th->getMessage());
             Console::error($th->getMessage());
+            Console::error($th->getTraceAsString());
         } finally {
             $build = $dbForProject->updateDocument('builds', $buildId, $build);
 

app/workers/deletes.php

@@ -467,7 +467,7 @@ class DeletesV1 extends Worker
          * Request executor to delete all deployment containers
          */
         Console::info("Requesting executor to delete all deployment containers for function " . $functionId);
-        $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
+        $executor = new Executor(App::getEnv('_APP_FUNCTIONS_PROXY_HOST'));
         foreach ($deploymentIds as $deploymentId) {
             try {
                 $executor->deleteRuntime($projectId, $deploymentId);
@@ -518,7 +518,7 @@ class DeletesV1 extends Worker
          */
         Console::info("Requesting executor to delete deployment container for deployment " . $deploymentId);
         try {
-            $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
+            $executor = new Executor(App::getEnv('_APP_FUNCTIONS_PROXY_HOST'));
             $executor->deleteRuntime($projectId, $deploymentId);
         } catch (Throwable $th) {
             Console::error($th->getMessage());

app/workers/functions.php

@@ -37,7 +37,7 @@ class FunctionsV1 extends Worker
 
     public function init(): void
     {
-        $this->executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
+        $this->executor = new Executor(App::getEnv('_APP_FUNCTIONS_PROXY_HOST'));
     }
 
     public function run(): void

bin/functions-proxy

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php -e /usr/src/code/app/functionsProxy.php -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php

composer.json

@@ -42,7 +42,8 @@
         "ext-zlib": "*",
         "ext-sockets": "*",
         "appwrite/php-clamav": "1.1.*",
-        "appwrite/php-runtimes": "0.11.*",
+        "appwrite/php-runtimes": "dev-feat-upgrade-system-version as 0.11.0",
+        "utopia-php/system": "dev-feat-cpu-usage as 0.5.0",
         "utopia-php/framework": "0.21.*",
         "utopia-php/logger": "0.3.*",
         "utopia-php/abuse": "0.14.*",
@@ -60,7 +61,7 @@
         "utopia-php/storage": "0.11.*",
         "utopia-php/websocket": "0.1.0",
         "utopia-php/image": "0.5.*",
-        "utopia-php/orchestration": "0.6.*",
+        "utopia-php/orchestration": "dev-feat-usage-stats as 0.6.0",
         "resque/php-resque": "1.3.6",
         "matomo/device-detector": "6.0.0",
         "dragonmantank/cron-expression": "3.3.1",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "08fdd139ad1285b02c4b4e555679e7de",
+    "content-hash": "eeafbe6edd7f9b3c378a76acc7057a3d",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -115,15 +115,15 @@
         },
         {
             "name": "appwrite/php-runtimes",
-            "version": "0.11.0",
+            "version": "dev-feat-upgrade-system-version",
             "source": {
                 "type": "git",
                 "url": "https://github.com/appwrite/runtimes.git",
-                "reference": "547fc026e11c0946846a8ac690898f5bf53be101"
+                "reference": "8679a0d36b1bd54f4b5a295e7097630d6cd4de9c"
             },
             "require": {
                 "php": ">=8.0",
-                "utopia-php/system": "0.4.*"
+                "utopia-php/system": "0.5.*"
             },
             "require-dev": {
                 "phpunit/phpunit": "^9.3",
@@ -154,7 +154,7 @@
                 "php",
                 "runtimes"
             ],
-            "time": "2022-08-15T14:03:36+00:00"
+            "time": "2022-10-13T12:30:32+00:00"
         },
         {
             "name": "chillerlan/php-qrcode",
@@ -931,6 +931,72 @@
             },
             "time": "2021-02-04T16:20:16+00:00"
         },
+        {
+            "name": "laravel/pint",
+            "version": "v1.2.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/laravel/pint.git",
+                "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/laravel/pint/zipball/1d276e4c803397a26cc337df908f55c2a4e90d86",
+                "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86",
+                "shasum": ""
+            },
+            "require": {
+                "ext-json": "*",
+                "ext-mbstring": "*",
+                "ext-tokenizer": "*",
+                "ext-xml": "*",
+                "php": "^8.0"
+            },
+            "require-dev": {
+                "friendsofphp/php-cs-fixer": "^3.11.0",
+                "illuminate/view": "^9.27",
+                "laravel-zero/framework": "^9.1.3",
+                "mockery/mockery": "^1.5.0",
+                "nunomaduro/larastan": "^2.2",
+                "nunomaduro/termwind": "^1.14.0",
+                "pestphp/pest": "^1.22.1"
+            },
+            "bin": [
+                "builds/pint"
+            ],
+            "type": "project",
+            "autoload": {
+                "psr-4": {
+                    "App\\": "app/",
+                    "Database\\Seeders\\": "database/seeders/",
+                    "Database\\Factories\\": "database/factories/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Nuno Maduro",
+                    "email": "enunomaduro@gmail.com"
+                }
+            ],
+            "description": "An opinionated code formatter for PHP.",
+            "homepage": "https://laravel.com",
+            "keywords": [
+                "format",
+                "formatter",
+                "lint",
+                "linter",
+                "php"
+            ],
+            "support": {
+                "issues": "https://github.com/laravel/pint/issues",
+                "source": "https://github.com/laravel/pint"
+            },
+            "time": "2022-09-13T15:07:15+00:00"
+        },
         {
             "name": "matomo/device-detector",
             "version": "6.0.0",
@@ -2390,16 +2456,16 @@
         },
         {
             "name": "utopia-php/orchestration",
-            "version": "0.6.0",
+            "version": "dev-feat-usage-stats",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/orchestration.git",
-                "reference": "94263976413871efb6b16157a7101a81df3b6d78"
+                "reference": "114046b9010ccb331deb7e23ef4aff22f903b890"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/94263976413871efb6b16157a7101a81df3b6d78",
-                "reference": "94263976413871efb6b16157a7101a81df3b6d78",
+                "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/114046b9010ccb331deb7e23ef4aff22f903b890",
+                "reference": "114046b9010ccb331deb7e23ef4aff22f903b890",
                 "shasum": ""
             },
             "require": {
@@ -2439,9 +2505,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/orchestration/issues",
-                "source": "https://github.com/utopia-php/orchestration/tree/0.6.0"
+                "source": "https://github.com/utopia-php/orchestration/tree/feat-usage-stats"
             },
-            "time": "2022-07-13T16:47:18+00:00"
+            "time": "2022-10-16T14:31:23+00:00"
         },
         {
             "name": "utopia-php/preloader",
@@ -2661,23 +2727,25 @@
         },
         {
             "name": "utopia-php/system",
-            "version": "0.4.0",
+            "version": "dev-feat-cpu-usage",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/system.git",
-                "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0"
+                "reference": "9847eb53d5fc6ddc008909cacdd39509e9966eb3"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/system/zipball/67c92c66ce8f0cc925a00bca89f7a188bf9183c0",
-                "reference": "67c92c66ce8f0cc925a00bca89f7a188bf9183c0",
+                "url": "https://api.github.com/repos/utopia-php/system/zipball/9847eb53d5fc6ddc008909cacdd39509e9966eb3",
+                "reference": "9847eb53d5fc6ddc008909cacdd39509e9966eb3",
                 "shasum": ""
             },
             "require": {
+                "laravel/pint": "1.2.*",
                 "php": ">=7.4"
             },
             "require-dev": {
                 "phpunit/phpunit": "^9.3",
+                "squizlabs/php_codesniffer": "^3.6",
                 "vimeo/psalm": "4.0.1"
             },
             "type": "library",
@@ -2710,9 +2778,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/system/issues",
-                "source": "https://github.com/utopia-php/system/tree/0.4.0"
+                "source": "https://github.com/utopia-php/system/tree/feat-cpu-usage"
             },
-            "time": "2021-02-04T14:14:49+00:00"
+            "time": "2022-10-19T06:19:09+00:00"
         },
         {
             "name": "utopia-php/websocket",
@@ -3413,25 +3481,30 @@
         },
         {
             "name": "phpdocumentor/type-resolver",
-            "version": "1.6.1",
+            "version": "1.6.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/phpDocumentor/TypeResolver.git",
-                "reference": "77a32518733312af16a44300404e945338981de3"
+                "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/77a32518733312af16a44300404e945338981de3",
-                "reference": "77a32518733312af16a44300404e945338981de3",
+                "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/48f445a408c131e38cab1c235aa6d2bb7a0bb20d",
+                "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d",
                 "shasum": ""
             },
             "require": {
-                "php": "^7.2 || ^8.0",
+                "php": "^7.4 || ^8.0",
                 "phpdocumentor/reflection-common": "^2.0"
             },
             "require-dev": {
                 "ext-tokenizer": "*",
-                "psalm/phar": "^4.8"
+                "phpstan/extension-installer": "^1.1",
+                "phpstan/phpstan": "^1.8",
+                "phpstan/phpstan-phpunit": "^1.1",
+                "phpunit/phpunit": "^9.5",
+                "rector/rector": "^0.13.9",
+                "vimeo/psalm": "^4.25"
             },
             "type": "library",
             "extra": {
@@ -3457,9 +3530,9 @@
             "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names",
             "support": {
                 "issues": "https://github.com/phpDocumentor/TypeResolver/issues",
-                "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.1"
+                "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.2"
             },
-            "time": "2022-03-15T21:29:03+00:00"
+            "time": "2022-10-14T12:47:21+00:00"
         },
         {
             "name": "phpspec/prophecy",
@@ -5352,9 +5425,32 @@
             "time": "2022-09-28T08:42:51+00:00"
         }
     ],
-    "aliases": [],
+    "aliases": [
+        {
+            "package": "appwrite/php-runtimes",
+            "version": "dev-feat-upgrade-system-version",
+            "alias": "0.11.0",
+            "alias_normalized": "0.11.0.0"
+        },
+        {
+            "package": "utopia-php/orchestration",
+            "version": "dev-feat-usage-stats",
+            "alias": "0.6.0",
+            "alias_normalized": "0.6.0.0"
+        },
+        {
+            "package": "utopia-php/system",
+            "version": "dev-feat-cpu-usage",
+            "alias": "0.5.0",
+            "alias_normalized": "0.5.0.0"
+        }
+    ],
     "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {
+        "appwrite/php-runtimes": 20,
+        "utopia-php/system": 20,
+        "utopia-php/orchestration": 20
+    },
     "prefer-stable": false,
     "prefer-lowest": false,
     "platform": {

docker-compose.yml

@@ -5,10 +5,10 @@
 
 x-logging: &x-logging
   logging:
-    driver: 'json-file'
+    driver: "json-file"
     options:
-      max-file: '5'
-      max-size: '10m'
+      max-file: "5"
+      max-size: "10m"
 
 x-env-storage: &x-env-storage |-
   _APP_STORAGE_DEVICE
@@ -34,7 +34,7 @@ x-env-storage: &x-env-storage |-
   _APP_STORAGE_WASABI_REGION
   _APP_STORAGE_WASABI_BUCKET
 
-version: '3'
+version: "3"
 
 services:
   traefik:
@@ -112,7 +112,7 @@ services:
       - mariadb
       - redis
       # - clamav
-    entrypoint: 
+    entrypoint:
       - php
       - -e
       - app/http.php
@@ -164,8 +164,8 @@ services:
       - _APP_FUNCTIONS_MEMORY
       - _APP_FUNCTIONS_MEMORY_SWAP
       - _APP_FUNCTIONS_RUNTIMES
-      - _APP_EXECUTOR_SECRET
-      - _APP_EXECUTOR_HOST
+      - _APP_FUNCTIONS_PROXY_SECRET
+      - _APP_FUNCTIONS_PROXY_HOST
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
       - _APP_STATSD_HOST
@@ -184,7 +184,7 @@ services:
     container_name: appwrite-realtime
     image: appwrite-dev
     restart: unless-stopped
-    ports: 
+    ports:
       - 9505:80
     labels:
       - "traefik.enable=true"
@@ -288,7 +288,7 @@ services:
     depends_on:
       - redis
       - mariadb
-    volumes: 
+    volumes:
       - appwrite-uploads:/storage/uploads:rw
       - appwrite-cache:/storage/cache:rw
       - appwrite-functions:/storage/functions:rw
@@ -311,8 +311,8 @@ services:
       - *x-env-storage
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
-      - _APP_EXECUTOR_SECRET
-      - _APP_EXECUTOR_HOST
+      - _APP_FUNCTIONS_PROXY_SECRET
+      - _APP_FUNCTIONS_PROXY_HOST
 
   appwrite-worker-databases:
     entrypoint: worker-databases
@@ -321,7 +321,7 @@ services:
     image: appwrite-dev
     networks:
       - appwrite
-    volumes: 
+    volumes:
       - ./app:/usr/src/code/app
       - ./src:/usr/src/code/src
       #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database
@@ -350,7 +350,7 @@ services:
     image: appwrite-dev
     networks:
       - appwrite
-    volumes: 
+    volumes:
       - ./app:/usr/src/code/app
       - ./src:/usr/src/code/src
     depends_on:
@@ -359,8 +359,8 @@ services:
     environment:
       - _APP_ENV
       - _APP_OPENSSL_KEY_V1
-      - _APP_EXECUTOR_SECRET
-      - _APP_EXECUTOR_HOST
+      - _APP_FUNCTIONS_PROXY_SECRET
+      - _APP_FUNCTIONS_PROXY_HOST
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -383,7 +383,7 @@ services:
     depends_on:
       - redis
       - mariadb
-    volumes: 
+    volumes:
       - appwrite-config:/storage/config:rw
       - appwrite-certificates:/storage/certificates:rw
       - ./app:/usr/src/code/app
@@ -406,6 +406,31 @@ services:
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
+  appwrite-functions-proxy:
+    entrypoint: functions-proxy
+    <<: *x-logging
+    container_name: appwrite-functions-proxy
+    image: appwrite-dev
+    ports:
+      - 5544:80
+    networks:
+      - appwrite
+    volumes:
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+    depends_on:
+      - redis
+    environment:
+      - _APP_ENV
+      - _APP_FUNCTIONS_PROXY_ADAPTER
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_EXECUTORS
+      - _APP_EXECUTOR_SECRET
+      - _APP_FUNCTIONS_PROXY_SECRET
+
   appwrite-worker-functions:
     entrypoint: worker-functions
     <<: *x-logging
@@ -419,7 +444,6 @@ services:
     depends_on:
       - redis
       - mariadb
-      - appwrite-executor
     environment:
       - _APP_ENV
       - _APP_OPENSSL_KEY_V1
@@ -433,23 +457,25 @@ services:
       - _APP_DB_USER
       - _APP_DB_PASS
       - _APP_FUNCTIONS_TIMEOUT
-      - _APP_EXECUTOR_SECRET
-      - _APP_EXECUTOR_HOST
+      - _APP_FUNCTIONS_PROXY_SECRET
+      - _APP_FUNCTIONS_PROXY_HOST
       - _APP_USAGE_STATS
       - DOCKERHUB_PULL_USERNAME
       - DOCKERHUB_PULL_PASSWORD
 
   appwrite-executor:
     container_name: appwrite-executor
+    hostname: exc1
     <<: *x-logging
     entrypoint: executor
     stop_signal: SIGINT
     image: appwrite-dev
+    ports:
+      - 9507:80
+      - 9508:3000
     networks:
       appwrite:
       runtimes:
-    ports:
-      - 9519:80
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./app:/usr/src/code/app
@@ -463,6 +489,51 @@ services:
       - appwrite
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
+      - _APP_VERSION
+      - _APP_FUNCTIONS_TIMEOUT
+      - _APP_FUNCTIONS_BUILD_TIMEOUT
+      - _APP_FUNCTIONS_CONTAINERS
+      - _APP_FUNCTIONS_RUNTIMES
+      - _APP_FUNCTIONS_CPUS
+      - _APP_FUNCTIONS_MEMORY
+      - _APP_FUNCTIONS_MEMORY_SWAP
+      - _APP_FUNCTIONS_INACTIVE_THRESHOLD
+      - _APP_EXECUTOR_SECRET
+      - OPEN_RUNTIMES_NETWORK
+      - _APP_LOGGING_PROVIDER
+      - _APP_LOGGING_CONFIG
+      - *x-env-storage
+      - DOCKERHUB_PULL_USERNAME
+      - DOCKERHUB_PULL_PASSWORD
+
+  appwrite-executor2:
+    container_name: appwrite-executor2
+    hostname: exc2
+    <<: *x-logging
+    entrypoint: executor
+    stop_signal: SIGINT
+    image: appwrite-dev
+    ports:
+      - 9509:80
+      - 9510:3000
+    networks:
+      appwrite:
+      runtimes:
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+      - appwrite-functions:/storage/functions:rw
+      - appwrite-builds:/storage/builds:rw
+      - /tmp:/tmp:rw
+    depends_on:
+      - redis
+      - mariadb
+      - appwrite
+    environment:
+      - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_VERSION
       - _APP_FUNCTIONS_TIMEOUT
       - _APP_FUNCTIONS_BUILD_TIMEOUT
@@ -671,7 +742,7 @@ services:
       - MYSQL_DATABASE=${_APP_DB_SCHEMA}
       - MYSQL_USER=${_APP_DB_USER}
       - MYSQL_PASSWORD=${_APP_DB_PASS}
-    command: 'mysqld --innodb-flush-method=fsync' # add ' --query_cache_size=0' for DB tests
+    command: "mysqld --innodb-flush-method=fsync" # add ' --query_cache_size=0' for DB tests
     # command: mv /var/lib/mysql/ib_logfile0 /var/lib/mysql/ib_logfile0.bu && mv /var/lib/mysql/ib_logfile1 /var/lib/mysql/ib_logfile1.bu
 
   # smtp:
@@ -685,7 +756,7 @@ services:
   #     - RELAY_FROM_HOSTS=192.168.0.0/16 ; *.yourdomain.com
   #     - SMARTHOST_HOST=smtp
   #     - SMARTHOST_PORT=587
-  
+
   redis:
     image: redis:7.0.4-alpine
     <<: *x-logging
@@ -725,12 +796,12 @@ services:
     <<: *x-logging
     networks:
       - appwrite
-    environment: 
+    environment:
       - _APP_INFLUXDB_HOST
       - _APP_INFLUXDB_PORT
 
   # Dev Tools Start ------------------------------------------------------------------------------------------
-  # 
+  #
   # The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack
   #
   # Here is a description of the different tools and why are we using them:
@@ -747,7 +818,7 @@ services:
     container_name: appwrite-mailcatcher
     <<: *x-logging
     ports:
-      - '9503:1080'
+      - "9503:1080"
     networks:
       - appwrite
 
@@ -756,7 +827,7 @@ services:
     container_name: appwrite-requestcatcher
     <<: *x-logging
     ports:
-      - '9504:5000'
+      - "9504:5000"
     networks:
       - appwrite
 
@@ -818,7 +889,7 @@ services:
   #     - './debug:/tmp'
   #   ports:
   #     - '3001:80'
-  
+
   # Dev Tools End   ------------------------------------------------------------------------------------------
 
 networks:
@@ -836,5 +907,4 @@ volumes:
   appwrite-builds:
   appwrite-influxdb:
   appwrite-config:
-  appwrite-executor:
   # appwrite-chronograf:

src/Executor/Executor.php

@@ -39,8 +39,8 @@ class Executor
      *
      * Launches a runtime container for a deployment ready for execution
      *
-     * @param string $deploymentId
      * @param string $projectId
+     * @param string $deploymentId
      * @param string $source
      * @param string $runtime
      * @param string $baseImage
@@ -53,8 +53,8 @@ class Executor
      * @param array $commands
      */
     public function createRuntime(
-        string $deploymentId,
         string $projectId,
+        string $deploymentId,
         string $source,
         string $runtime,
         string $baseImage,
@@ -63,15 +63,16 @@ class Executor
         string $workdir = '',
         string $destination = '',
         array $vars = [],
-        array $commands = []
+        array $commands = [],
+        string $key = null
     ) {
         $route = "/runtimes";
         $headers = [
             'content-type' => 'application/json',
-            'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '')
+            'x-appwrite-executor-key' => $key ?? App::getEnv('_APP_FUNCTIONS_PROXY_SECRET', '')
         ];
         $params = [
-            'runtimeId' => "$projectId-$deploymentId",
+            'runtimeId' => $projectId . '-' . $deploymentId,
             'source' => $source,
             'destination' => $destination,
             'runtime' => $runtime,
@@ -109,7 +110,7 @@ class Executor
         $route = "/runtimes/$runtimeId";
         $headers = [
             'content-type' =>  'application/json',
-            'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '')
+            'x-appwrite-executor-key' => App::getEnv('_APP_FUNCTIONS_PROXY_SECRET', '')
         ];
 
         $params = [];
@@ -153,13 +154,24 @@ class Executor
         $route = "/execution";
         $headers = [
             'content-type' =>  'application/json',
-            'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '')
+            'x-appwrite-executor-key' => App::getEnv('_APP_FUNCTIONS_PROXY_SECRET', '')
         ];
         $params = [
+            // execution-related
             'runtimeId' => "$projectId-$deploymentId",
             'vars' => $vars,
             'data' => $data,
             'timeout' => $timeout,
+
+            // runtime-related
+            'deploymentId' => $deploymentId,
+            'projectId' => $projectId,
+            'source' => $path,
+            'runtime' => $runtime,
+            'baseImage' => $baseImage,
+            'vars' => $vars,
+            'entrypoint' => $entrypoint,
+            'commands' => []
         ];
 
         /* Add 2 seconds as a buffer to the actual timeout value since there can be a slight variance*/
@@ -168,44 +180,11 @@ class Executor
         $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout);
         $status = $response['headers']['status-code'];
 
-        for ($attempts = 0; $attempts < 10; $attempts++) {
-            try {
-                switch (true) {
-                    case $status < 400:
-                        return $response['body'];
-                    case $status === 404:
-                        $response = $this->createRuntime(
-                            deploymentId: $deploymentId,
-                            projectId: $projectId,
-                            source: $path,
-                            runtime: $runtime,
-                            baseImage: $baseImage,
-                            vars: $vars,
-                            entrypoint: $entrypoint,
-                            commands: []
-                        );
-                        $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout);
-                        $status = $response['headers']['status-code'];
-
-                        if ($status < 400) {
-                            return $response['body'];
-                        }
-                        break;
-                    case $status === 406:
-                        $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout);
-                        $status = $response['headers']['status-code'];
-
-                        if ($status < 400) {
-                            return $response['body'];
-                        }
-                        break;
-                    default:
-                        throw new \Exception($response['body']['message'], $status);
-                }
-            } catch (\Exception $e) {
-                throw new \Exception($e->getMessage(), $e->getCode());
-            }
-            sleep(2);
+        switch (true) {
+            case $status < 400:
+                return $response['body'];
+            default:
+                throw new \Exception($response['body']['message'], $status);
         }
 
         throw new Exception($response['body']['message'], 503);
@@ -301,7 +280,16 @@ class Executor
         }
 
         if ((curl_errno($ch)/* || 200 != $responseStatus*/)) {
-            throw new Exception(curl_error($ch) . ' with status code ' . $responseStatus, $responseStatus);
+            $errorCode = $responseStatus > 0 ? $responseStatus : 500;
+            $errorMessage = curl_error($ch) . ' with status code ' . $errorCode;
+
+
+            if (curl_errno($ch) === 28) {
+                $errorCode = 500;
+                $errorMessage = 'Execution timed out after ' . $timeout . ' seconds.';
+            }
+
+            throw new Exception($errorMessage, $errorCode);
         }
 
         curl_close($ch);

src/FunctionsProxy/Adapter.php

@@ -0,0 +1,61 @@
+<?php
+
+namespace FunctionsProxy;
+
+use Exception;
+use Swoole\Database\RedisPool;
+use Utopia\App;
+use Utopia\Cache\Adapter\Redis;
+use Utopia\Cache\Cache;
+
+abstract class Adapter
+{
+    private RedisPool $redisPool;
+
+    public function __construct(RedisPool $redisPool)
+    {
+        $this->redisPool = $redisPool;
+    }
+
+    private function getConnection(): array
+    {
+        $redis = $this->redisPool->get();
+        $cache = new Cache(new Redis($redis));
+
+        return [$cache, fn () => $this->redisPool->put($redis)];
+    }
+
+    protected function getExecutors(): array
+    {
+        [$cache, $returnCache] = $this->getConnection();
+
+        $responseExecutors = [];
+
+        try {
+            $executors = \explode(',', App::getEnv('_APP_EXECUTORS', ''));
+
+            foreach ($executors as $executor) {
+                $data = $cache->load('executors-' . $executor, 60 * 60 * 24 * 30 * 3); // 3 months
+
+                if ($data === false || $data['status'] !== 'online') {
+                    continue;
+                }
+
+                $responseExecutors[] = [
+                    'hostname' => $executor,
+                    'state' => $data
+                ];
+            }
+        } finally {
+            call_user_func($returnCache);
+        }
+
+        if (\count($responseExecutors) <= 0) {
+            throw new Exception("No executor is online.");
+        }
+
+        return $responseExecutors;
+    }
+
+    abstract public function getNextExecutor(?string $contaierId): array;
+}

src/FunctionsProxy/Adapter/Random.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace FunctionsProxy\Adapter;
+
+use FunctionsProxy\Adapter;
+
+class Random extends Adapter
+{
+    public function getNextExecutor(?string $contaierId): array
+    {
+        $executors = $this->getExecutors();
+        $executor = $executors[\array_rand($executors)] ?? null;
+
+        return $executor ?? null;
+    }
+}

src/FunctionsProxy/Adapter/RoundRobin.php

@@ -0,0 +1,32 @@
+<?php
+
+namespace FunctionsProxy\Adapter;
+
+use FunctionsProxy\Adapter;
+use Swoole\Database\RedisPool;
+use Swoole\Atomic;
+
+class RoundRobin extends Adapter
+{
+    private Atomic $counter;
+
+    public function __construct(RedisPool $redisPool)
+    {
+        parent::__construct($redisPool);
+        $this->counter = new Atomic(-1);
+    }
+
+    public function getNextExecutor(?string $contaierId): array
+    {
+        $index = $this->counter->add();
+        $executors = $this->getExecutors();
+        $executor = $executors[$index] ?? null;
+
+        if (!$executor) {
+            $executor = $executors[0];
+            $this->counter->cmpset($index, 0);
+        }
+
+        return $executor ?? null;
+    }
+}

src/FunctionsProxy/Adapter/UsageBased.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace FunctionsProxy\Adapter;
+
+use FunctionsProxy\Adapter;
+
+class UsageBased extends Adapter
+{
+    public function getNextExecutor(?string $contaierId): array
+    {
+        // Adapter configuration
+        $idealMaxUsage = 80; // Means 80%
+        $hostWeight = 0.3;
+        $containerWeight = 0.7; //Weights should add up to 1. For maintanance reasons
+
+        $executors = $this->getExecutors();
+
+        // Remove offline and unknown-status executors
+        $executors = \array_filter($executors, fn($executor) => (($executor['state'] ?? [])['status']) ?? 'offline' === 'online');
+
+        // Ideal executor is one that is already running a contianer for this function deployment
+        $idealExecutors = [];
+
+        // For whatever reason we don't know container yet. We consider all executors ideal
+        if (!(isset($contaierId))) {
+            $idealExecutors = \array_map(fn($executor) => $executor['hostname'], $executors);
+        } else {
+            foreach ($executors as $executor) {
+                $executorId = $executor['hostname'] ?? '';
+                $executorState = $executor['state']['health'] ?? [];
+                $hostUsage = intval($executorState['hostUsage'] ?? 100);
+                $containerUsage = intval(($executorState['functionsUsage'] ?? [])[$executorId . '-' . $contaierId] ?? 100); // Forcing 100 to mark that starting runtime is the least ideal.
+
+                // If host or contianer usage above idealMaxUsage, executor is not ideal
+                if ($hostUsage < $idealMaxUsage && $containerUsage < $idealMaxUsage) {
+                    $idealExecutors[] = $executorId;
+                }
+            }
+        }
+
+        if (\count($idealExecutors) <= 0) {
+            // If no ideal, let's consider all of them ideal. Since there is no prefference.
+            $idealExecutors = \array_map(fn($executor) => $executor['hostname'], $executors);
+        }
+
+        // Sort containers based on usage
+        $sortedExecutors = [];
+
+        foreach ($idealExecutors as $executorId) {
+            $executorIndex = \array_search($executorId, \array_map(fn($executor) => $executor['hostname'], $executors));
+            $executor = $executors[$executorIndex];
+
+            if (!isset($executor)) {
+                continue;
+            }
+
+            $executorState = $executor['state']['health'] ?? [];
+            $hostUsage = intval($executorState['hostUsage'] ?? 10);
+            $containerUsage = intval(($executorState['functionsUsage'] ?? [])[$executorId . '-' . $contaierId] ?? 0);
+
+            $usageIndex = ($hostUsage * $hostWeight) + ($containerUsage * $containerWeight);
+
+            $sortedExecutors[$executorId] = $usageIndex;
+        }
+
+        \asort($sortedExecutors);
+
+        // TODO: @Meldiron Dont pick best. Find best with similar usage and pick randomly from them
+        // Pick the least used executor
+        $idealExecutorId = \array_keys($sortedExecutors)[0] ?? null;
+
+        // Null if no executor found
+        if ($idealExecutorId === null) {
+            return null;
+        }
+
+        $executorIndex = \array_search($idealExecutorId, \array_map(fn($executor) => $executor['hostname'], $executors));
+        $executor = $executors[$executorIndex];
+
+        // Null if can't match executor to ID
+        return $executor ?? null;
+    }
+}

tests/e2e/Services/Functions/FunctionsCustomServerTest.php

@@ -913,7 +913,7 @@ class FunctionsCustomServerTest extends Scope
         $this->assertLessThan(6, $executions['body']['executions'][0]['duration']);
         $this->assertGreaterThan(4, $executions['body']['executions'][0]['duration']);
         $this->assertEquals($executions['body']['executions'][0]['response'], '');
-        $this->assertEquals($executions['body']['executions'][0]['stderr'], 'An internal curl error has occurred within the executor! Error Msg: Operation timed out');
+        $this->assertEquals($executions['body']['executions'][0]['stderr'], 'Execution timed out after 4 seconds.');
 
         // Cleanup : Delete function
         $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $functionId, [

tests/resources/docker/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: "3"
 
 services:
   traefik:
@@ -27,7 +27,7 @@ services:
     networks:
       - gateway
       - appwrite
-  
+
   appwrite:
     container_name: appwrite
     build:
@@ -36,7 +36,7 @@ services:
         - TESTING=true
         - VERSION=dev
     restart: unless-stopped
-    ports: 
+    ports:
       - 9501:80
     networks:
       - appwrite
@@ -88,7 +88,7 @@ services:
       - _APP_FUNCTIONS_CPUS
       - _APP_FUNCTIONS_MEMORY
       - _APP_FUNCTIONS_MEMORY_SWAP
-      - _APP_EXECUTOR_HOST
+      - _APP_FUNCTIONS_PROXY_HOST
 
   appwrite-worker-usage:
     entrypoint: worker-usage
@@ -182,7 +182,7 @@ services:
     depends_on:
       - redis
       - mariadb
-    volumes: 
+    volumes:
       - appwrite-uploads:/storage/uploads:rw
       - appwrite-cache:/storage/cache:rw
     environment:
@@ -194,7 +194,7 @@ services:
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_EXECUTOR_HOST
+      - _APP_FUNCTIONS_PROXY_HOST
 
   appwrite-worker-certificates:
     entrypoint: worker-certificates
@@ -207,7 +207,7 @@ services:
     depends_on:
       - redis
       - mariadb
-    volumes: 
+    volumes:
       - appwrite-config:/storage/config:rw
       - appwrite-certificates:/storage/certificates:rw
     environment:
@@ -249,7 +249,7 @@ services:
       - _APP_FUNCTIONS_CPUS
       - _APP_FUNCTIONS_MEMORY
       - _APP_FUNCTIONS_MEMORY_SWAP
-      - _APP_EXECUTOR_HOST
+      - _APP_FUNCTIONS_PROXY_HOST
 
   appwrite-worker-mails:
     entrypoint: worker-mails
@@ -269,7 +269,7 @@ services:
       - _APP_REDIS_PORT
       - _APP_SMTP_HOST
       - _APP_SMTP_PORT
-  
+
   appwrite-worker-builds:
     entrypoint: worker-builds
     container_name: appwrite-worker-builds
@@ -277,7 +277,7 @@ services:
       context: .
     networks:
       - appwrite
-    volumes: 
+    volumes:
       - ./app:/usr/src/code/app
       - ./src:/usr/src/code/src
     depends_on:
@@ -296,8 +296,8 @@ services:
       - _APP_DB_PASS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
-      - _APP_EXECUTOR_SECRET
-      - _APP_EXECUTOR_HOST
+      - _APP_FUNCTIONS_PROXY_SECRET
+      - _APP_FUNCTIONS_PROXY_HOST
 
   appwrite-schedule:
     entrypoint: schedule
@@ -329,14 +329,14 @@ services:
       - MYSQL_DATABASE=appwrite
       - MYSQL_USER=user
       - MYSQL_PASSWORD=password
-    command: 'mysqld --innodb-flush-method=fsync'
+    command: "mysqld --innodb-flush-method=fsync"
 
   maildev:
     image: djfarrelly/maildev
     container_name: appwrite-maildev
     restart: unless-stopped
     ports:
-      - '1080:80'
+      - "1080:80"
     networks:
       - appwrite