From b79bdc56a1f4dfe2a24b747c4b48eebd1faa64f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 16 Oct 2022 14:28:36 +0000 Subject: [PATCH] Implement cpu usage stats --- app/executor.php | 6 +- app/functionsProxy.php | 12 ++- composer.lock | 31 ++++--- src/FunctionsProxy/Adapter.php | 2 +- src/FunctionsProxy/Adapter/Random.php | 4 +- src/FunctionsProxy/Adapter/RoundRobin.php | 2 +- src/FunctionsProxy/Adapter/UsageBased.php | 108 ++++++++++++---------- 7 files changed, 91 insertions(+), 74 deletions(-) diff --git a/app/executor.php b/app/executor.php index 04946e17df..81a36ddb56 100644 --- a/app/executor.php +++ b/app/executor.php @@ -650,11 +650,11 @@ App::post('/v1/execution') App::get('/v1/health') ->param('name', '', new Text(64), 'Name of the executor.') - ->desc("Get usage stats of host machine") + ->desc("Get usage stats of host machine CPU usage from 0 to 100.") ->inject('response') ->action(function (string $name, Response $response) use ($orchestrationPool) { $systemCores = System::getCPUCores(); - $systemUsage = System::getCPUUtilisation() / $systemCores / 100; + $systemUsage = System::getCPUUtilisation() / $systemCores; $functionsUsage = []; try { @@ -662,7 +662,7 @@ App::get('/v1/health') $containerUsages = $orchestration->getStats(filters: [ 'label' => 'openruntimes-executor=' . $name ], cycles: 3); foreach ($containerUsages as $containerUsage) { - $functionsUsage[$containerUsage['name']] = $containerUsage['cpu']; + $functionsUsage[$containerUsage['name']] = $containerUsage['cpu'] * 100; } } catch (\Exception $err) { // TODO: @Meldiron Handle better diff --git a/app/functionsProxy.php b/app/functionsProxy.php index c075860ca6..ae84169d1a 100644 --- a/app/functionsProxy.php +++ b/app/functionsProxy.php @@ -2,8 +2,7 @@ require_once __DIR__ . '/../vendor/autoload.php'; -use FunctionsProxy\Adapter\Random; -use FunctionsProxy\Adapter\RoundRobin; +use FunctionsProxy\Adapter\UsageBased; use Swoole\Coroutine\Http\Client; use Utopia\Logger\Log; use Utopia\Logger\Logger; @@ -41,7 +40,7 @@ $redisPool = new RedisPool( 64 ); -$adapter = new Random($redisPool); +$adapter = new UsageBased($redisPool); // TODO: @Meldiron allow env variable to switch; log what is picked function markOffline(Cache $cache, string $executorId, string $error, bool $forceShowError = false) { @@ -218,7 +217,10 @@ $run = function (SwooleRequest $request, SwooleResponse $response) use ($adapter throw new Exception('Missing proxy key'); } - $executor = $adapter->getNextExecutor(); + + $body = \json_decode($request->getContent(), true); + $runtimeId = $body['runtimeId'] ?? null; + $executor = $adapter->getNextExecutor($runtimeId); Console::success("Executing on " . $executor['hostname']); @@ -247,7 +249,7 @@ $run = function (SwooleRequest $request, SwooleResponse $response) use ($adapter $http->on('start', function () use ($redisPool) { // Keep updating executors state - Timer::tick(30000, function (int $timerId) use ($redisPool) { + Timer::tick(15000, function (int $timerId) use ($redisPool) { fetchExecutorsState($redisPool, false); }); }); diff --git a/composer.lock b/composer.lock index 4292b84e3d..7312abf7fe 100644 --- a/composer.lock +++ b/composer.lock @@ -2400,12 +2400,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/orchestration.git", - "reference": "e3434f1d5b7f7fada785281901350ead2ce10465" + "reference": "5fb741ea8152167966ec51e17e78cfbc691d1a1f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/e3434f1d5b7f7fada785281901350ead2ce10465", - "reference": "e3434f1d5b7f7fada785281901350ead2ce10465", + "url": "https://api.github.com/repos/utopia-php/orchestration/zipball/5fb741ea8152167966ec51e17e78cfbc691d1a1f", + "reference": "5fb741ea8152167966ec51e17e78cfbc691d1a1f", "shasum": "" }, "require": { @@ -2447,7 +2447,7 @@ "issues": "https://github.com/utopia-php/orchestration/issues", "source": "https://github.com/utopia-php/orchestration/tree/feat-usage-stats" }, - "time": "2022-10-14T07:14:57+00:00" + "time": "2022-10-14T19:36:45+00:00" }, { "name": "utopia-php/preloader", @@ -3420,25 +3420,30 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.6.1", + "version": "1.6.2", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "77a32518733312af16a44300404e945338981de3" + "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/77a32518733312af16a44300404e945338981de3", - "reference": "77a32518733312af16a44300404e945338981de3", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/48f445a408c131e38cab1c235aa6d2bb7a0bb20d", + "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d", "shasum": "" }, "require": { - "php": "^7.2 || ^8.0", + "php": "^7.4 || ^8.0", "phpdocumentor/reflection-common": "^2.0" }, "require-dev": { "ext-tokenizer": "*", - "psalm/phar": "^4.8" + "phpstan/extension-installer": "^1.1", + "phpstan/phpstan": "^1.8", + "phpstan/phpstan-phpunit": "^1.1", + "phpunit/phpunit": "^9.5", + "rector/rector": "^0.13.9", + "vimeo/psalm": "^4.25" }, "type": "library", "extra": { @@ -3464,9 +3469,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.1" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.2" }, - "time": "2022-03-15T21:29:03+00:00" + "time": "2022-10-14T12:47:21+00:00" }, { "name": "phpspec/prophecy", @@ -5401,5 +5406,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/src/FunctionsProxy/Adapter.php b/src/FunctionsProxy/Adapter.php index 7a01dcb6a7..eecd63bcdc 100644 --- a/src/FunctionsProxy/Adapter.php +++ b/src/FunctionsProxy/Adapter.php @@ -60,5 +60,5 @@ abstract class Adapter return $responseExecutors; } - abstract public function getNextExecutor(): array; + abstract public function getNextExecutor(?string $contaierId): array; } diff --git a/src/FunctionsProxy/Adapter/Random.php b/src/FunctionsProxy/Adapter/Random.php index 168c150913..dcbb9c8d94 100644 --- a/src/FunctionsProxy/Adapter/Random.php +++ b/src/FunctionsProxy/Adapter/Random.php @@ -6,13 +6,11 @@ use FunctionsProxy\Adapter; class Random extends Adapter { - public function getNextExecutor(): array + public function getNextExecutor(?string $contaierId): array { $executors = $this->getExecutors(); $executor = $executors[\array_rand($executors)] ?? null; - \var_dump($executor); - return $executor ?? null; } } diff --git a/src/FunctionsProxy/Adapter/RoundRobin.php b/src/FunctionsProxy/Adapter/RoundRobin.php index d5b41d4e1e..d2c5e23353 100644 --- a/src/FunctionsProxy/Adapter/RoundRobin.php +++ b/src/FunctionsProxy/Adapter/RoundRobin.php @@ -8,7 +8,7 @@ class RoundRobin extends Adapter { private $currentIndex = 0; // TODO: @Meldiron Put into utopia app resource or/and utopia registry - public function getNextExecutor(): array + public function getNextExecutor(?string $contaierId): array { $executors = $this->getExecutors(); $executor = $executors[$this->currentIndex] ?? null; diff --git a/src/FunctionsProxy/Adapter/UsageBased.php b/src/FunctionsProxy/Adapter/UsageBased.php index 8e10631e6c..ebb2b35987 100644 --- a/src/FunctionsProxy/Adapter/UsageBased.php +++ b/src/FunctionsProxy/Adapter/UsageBased.php @@ -6,62 +6,74 @@ use FunctionsProxy\Adapter; class UsageBased extends Adapter { - public function getNextExecutor($contaierId): array + public function getNextExecutor(?string $contaierId): array { $executors = $this->getExecutors(); - /* - appwrite-functions-proxy | array(3) { - appwrite-functions-proxy | ["id"]=> - appwrite-functions-proxy | string(4) "exc2" - appwrite-functions-proxy | ["hostname"]=> - appwrite-functions-proxy | string(18) "appwrite-executor2" - appwrite-functions-proxy | ["state"]=> - appwrite-functions-proxy | array(2) { - appwrite-functions-proxy | ["status"]=> - appwrite-functions-proxy | string(6) "online" - appwrite-functions-proxy | ["health"]=> - appwrite-functions-proxy | array(3) { - appwrite-functions-proxy | ["status"]=> - appwrite-functions-proxy | string(4) "pass" - appwrite-functions-proxy | ["hostUsage"]=> - appwrite-functions-proxy | float(0.348125) - appwrite-functions-proxy | ["functionsUsage"]=> - appwrite-functions-proxy | array(0) { - // container1: 0.5 - // container1: 0.2 - appwrite-functions-proxy | } - appwrite-functions-proxy | } - appwrite-functions-proxy | } - appwrite-functions-proxy | } - */ + // Remove offline and unknown-status executors + $executors = \array_filter($executors, fn($executor) => (($executor['state'] ?? [])['status']) ?? 'offline' === 'online'); - // low, mid, high + // Ideal executor is one that is already running a contianer for this function deployment + $idealExecutors = []; - // 10% on host, 90% on chontainer - // 90% on host, 10% on chontainer - - // TODO: @Meldiron Proper functionality - // TODO: @Meldiron Use this adapter and test it - - /* - // If 2+ running, do this to sort: - - $hostUsage = 20; - $containerUsage = 40; - - $hostWeight = 0.33; - $contiainerWeight = 0.66; - - $totalUsage = $hostUsage * $hostWeight + $containerUsage * $contiainerWeight; - - // If all existing are above 80%, we should start on new executor ideally - if(host < 0.8 & container < 0.8) { + // For whatever reason we don't know container yet. We consider all executors ideal + if(!(isset($contaierId))) { + $idealExecutors = \array_map(fn($executor) => $executor['id'], $executors); + } else { + foreach($executors as $executor) { + $executorId = $executor['id'] ?? ''; + $executorState = $executor['state']['health'] ?? []; + $hostUsage = intval($executorState['hostUsage'] ?? 100); + $containerUsage = intval(($executorState['functionsUsage'] ?? [])[$executorId . '-' . $contaierId] ?? 100); + // If host or contianer usage above 80, executor is not ideal + if($hostUsage < 80 && $containerUsage < 80) { + $idealExecutors[] = $executorId; + } + } } - */ - $executor = $executors[\array_rand($executors)] ?? null; + if(\count($idealExecutors) <= 0) { + // If no ideal, let's consider all of them ideal. Since there is no prefference. + $idealExecutors = \array_map(fn($executor) => $executor['id'], $executors); + } + + // Sort containers based on usage + $sortedExecutors = []; + $hostWeight = 0.3; + $containerWeight = 0.7; + + foreach($idealExecutors as $executorId) { + $executorIndex = \array_search($executorId, \array_map(fn($executor) => $executor['id'], $executors)); + $executor = $executors[$executorIndex]; + + if(!isset($executor)) { + continue; + } + + $executorState = $executor['state']['health'] ?? []; + $hostUsage = intval($executorState['hostUsage'] ?? 10); + $containerUsage = intval(($executorState['functionsUsage'] ?? [])[$executorId . '-' . $contaierId] ?? 100); + + $usageIndex = ($hostUsage * $hostWeight) + ($containerUsage * $containerWeight); + + $sortedExecutors[$executorId] = $usageIndex; + } + + \asort($sortedExecutors); + + // Pick the least used executor + $idealExecutorId = $idealExecutors[0] ?? null; + + // Null if no executor found + if($idealExecutorId === null) { + return null; + } + + $executorIndex = \array_search($idealExecutorId, \array_map(fn($executor) => $executor['id'], $executors)); + $executor = $executors[$executorIndex]; + + // Null if can't match executor to ID return $executor ?? null; } }