Remove CPU/Memory from document, add envs, add tests

This commit is contained in:
Bradley Schofield
2024-07-22 18:12:43 +09:00
parent 667b69e146
commit f0c1a62aa2
9 changed files with 140 additions and 119 deletions
-22
View File
@@ -3055,28 +3055,6 @@ $projectCollections = array_merge([
'default' => null,
'filters' => [],
],
[
'array' => false,
'$id' => ID::custom('memory'),
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 512,
'signed' => false,
'required' => false,
'default' => 512,
'filters' => [],
],
[
'array' => false,
'$id' => ID::custom('cpus'),
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 2,
'signed' => false,
'required' => false,
'default' => 1,
'filters' => [],
],
[
'array' => false,
'$id' => ID::custom('size'),
+8 -11
View File
@@ -237,8 +237,6 @@ App::post('/v1/functions')
'providerBranch' => $providerBranch,
'providerRootDirectory' => $providerRootDirectory,
'providerSilentMode' => $providerSilentMode,
'memory' => $spec['memory'],
'cpus' => $spec['cpus'],
'size' => $size
]));
@@ -742,7 +740,6 @@ App::put('/v1/functions/:functionId')
->inject('gitHub')
->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $size, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github) use ($redeployVcs) {
// TODO: If only branch changes, re-deploy
$function = $dbForProject->getDocument('functions', $functionId);
if ($function->isEmpty()) {
@@ -847,8 +844,7 @@ App::put('/v1/functions/:functionId')
}
// Enforce Cold Start if spec limits change.
if (($function->getAttribute('cpus') !== $spec['cpus'] ||
$function->getAttribute('memory') !== $spec['memory']) && !empty($function->getAttribute('deployment'))) {
if ($function->getAttribute('size') !== $size && !empty($function->getAttribute('deployment'))) {
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
try {
$executor->deleteRuntime($project->getId(), $function->getAttribute('deployment'));
@@ -877,8 +873,6 @@ App::put('/v1/functions/:functionId')
'providerBranch' => $providerBranch,
'providerRootDirectory' => $providerRootDirectory,
'providerSilentMode' => $providerSilentMode,
'memory' => $spec['memory'],
'cpus' => $spec['cpus'],
'size' => $size,
'search' => implode(' ', [$functionId, $name, $runtime]),
])));
@@ -1601,6 +1595,7 @@ App::post('/v1/functions/:functionId/executions')
$version = $function->getAttribute('version', 'v2');
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);
$spec = Config::getParam('runtime-sizes')[$function->getAttribute('size', 's-1vcpu-512mb')];
$runtime = (isset($runtimes[$function->getAttribute('runtime', '')])) ? $runtimes[$function->getAttribute('runtime', '')] : null;
@@ -1767,6 +1762,8 @@ App::post('/v1/functions/:functionId/executions')
'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
'APPWRITE_FUNCTION_CPUS' => $spec['cpus'] ?? 1,
'APPWRITE_FUNCTION_MEMORY' => $spec['memory'] ?? 512
]);
/** Execute function */
@@ -1789,8 +1786,8 @@ App::post('/v1/functions/:functionId/executions')
method: $method,
headers: $headers,
runtimeEntrypoint: $command,
cpus: $function->getAttribute('cpus', 1),
memory: $function->getAttribute('memory', 512),
cpus: $spec['cpus'] ?? 1,
memory: $spec['memory'] ?? 512,
requestTimeout: 30
);
@@ -1828,8 +1825,8 @@ App::post('/v1/functions/:functionId/executions')
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
->addMetric(METRIC_EXECUTIONS_MB_SECONDS, $function->getAttribute('memory', 512) * $execution->getAttribute('duration', 0) * $function->getAttribute('cpus', 1))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), $function->getAttribute('memory', 512) * $execution->getAttribute('duration', 0) * $function->getAttribute('cpus', 1))
->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? 512) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? 1)))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? 512) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? 1)))
;
if ($function->getAttribute('logging')) {
+7 -2
View File
@@ -130,6 +130,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
$version = $function->getAttribute('version', 'v2');
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);
$spec = Config::getParam('runtime-sizes')[$function->getAttribute('size', 's-1vcpu-512mb')];
$runtime = (isset($runtimes[$function->getAttribute('runtime', '')])) ? $runtimes[$function->getAttribute('runtime', '')] : null;
@@ -250,6 +251,8 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
'APPWRITE_FUNCTION_CPUS' => $spec['cpus'] ?? 1,
'APPWRITE_FUNCTION_MEMORY' => $spec['memory'] ?? 512,
]);
/** Execute function */
@@ -273,8 +276,8 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
headers: $headers,
runtimeEntrypoint: $command,
requestTimeout: 30,
cpus: $function->getAttribute('cpus', 1),
memory: $function->getAttribute('memory', 512)
cpus: $spec['cpus'] ?? 1,
memory: $spec['memory'] ?? 512,
);
$headersFiltered = [];
@@ -312,6 +315,8 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? 512) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? 1)))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? 512) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? 1)))
;
if ($function->getAttribute('logging')) {
Generated
+11 -10
View File
@@ -355,16 +355,16 @@
},
{
"name": "chillerlan/php-settings-container",
"version": "2.1.5",
"version": "2.1.6",
"source": {
"type": "git",
"url": "https://github.com/chillerlan/php-settings-container.git",
"reference": "f705310389264c3578fdd9ffb15aa2cd6d91772e"
"reference": "5553558bd381fce5108c6d0343c12e488cfec6bb"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/f705310389264c3578fdd9ffb15aa2cd6d91772e",
"reference": "f705310389264c3578fdd9ffb15aa2cd6d91772e",
"url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/5553558bd381fce5108c6d0343c12e488cfec6bb",
"reference": "5553558bd381fce5108c6d0343c12e488cfec6bb",
"shasum": ""
},
"require": {
@@ -372,15 +372,16 @@
"php": "^7.4 || ^8.0"
},
"require-dev": {
"phan/phan": "^5.4",
"phpcsstandards/php_codesniffer": "^3.8",
"phpmd/phpmd": "^2.13",
"phpunit/phpunit": "^9.6"
"phpmd/phpmd": "^2.15",
"phpstan/phpstan": "^1.11",
"phpstan/phpstan-deprecation-rules": "^1.2",
"phpunit/phpunit": "^9.6",
"squizlabs/php_codesniffer": "^3.10"
},
"type": "library",
"autoload": {
"psr-4": {
"chillerlan\\Settings\\": "src/"
"chillerlan\\Settings\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
@@ -417,7 +418,7 @@
"type": "ko_fi"
}
],
"time": "2024-01-05T23:20:55+00:00"
"time": "2024-07-17T01:04:28+00:00"
},
{
"name": "dragonmantank/cron-expression",
+8 -5
View File
@@ -140,6 +140,7 @@ class Builds extends Action
}
$version = $function->getAttribute('version', 'v2');
$spec = Config::getParam('runtime-sizes')[$function->getAttribute('size', 's-1vcpu-512mb')];
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);
$key = $function->getAttribute('runtime');
$runtime = $runtimes[$key] ?? null;
@@ -391,6 +392,8 @@ class Builds extends Action
'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
'APPWRITE_FUNCTION_CPUS' => $spec['cpus'] ?? 1,
'APPWRITE_FUNCTION_MEMORY' => $spec['memory'] ?? 512
]);
$command = $deployment->getAttribute('commands', '');
@@ -399,7 +402,7 @@ class Builds extends Action
$err = null;
Co::join([
Co\go(function () use ($executor, &$response, $project, $deployment, $source, $function, $runtime, $vars, $command, &$err) {
Co\go(function () use ($executor, &$response, $project, $deployment, $source, $function, $runtime, $vars, $command, $spec, &$err) {
try {
$version = $function->getAttribute('version', 'v2');
$command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . \trim(\escapeshellarg($command), "\'") . '"';
@@ -410,8 +413,8 @@ class Builds extends Action
source: $source,
image: $runtime['image'],
version: $version,
cpus: $function->getAttribute('cpus', 1),
memory: $function->getAttribute('memory', 512),
cpus: $spec['cpus'] ?? 1,
memory: $spec['memory'] ?? 512,
remove: true,
entrypoint: $deployment->getAttribute('entrypoint'),
destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}",
@@ -542,11 +545,11 @@ class Builds extends Action
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(METRIC_BUILDS_MB_SECONDS, $function->getAttribute('memory', 512) * $build->getAttribute('duration', 0) * $function->getAttribute('cpus', 1))
->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? 512) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? 1)))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), $function->getAttribute('memory', 512) * $build->getAttribute('duration', 0) * $function->getAttribute('cpus', 1))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? 512) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? 1)))
->setProject($project)
->trigger();
}
+7 -4
View File
@@ -308,6 +308,7 @@ class Functions extends Action
$user ??= new Document();
$functionId = $function->getId();
$deploymentId = $function->getAttribute('deployment', '');
$spec = Config::getParam('runtime-sizes')[$function->getAttribute('size', 's-1vcpu-512mb')];
$log->addTag('deploymentId', $deploymentId);
@@ -448,6 +449,8 @@ class Functions extends Action
'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
'APPWRITE_FUNCTION_CPUS' => ($spec['cpus'] ?? 1),
'APPWRITE_FUNCTION_MEMORY' => ($spec['memory'] ?? 512)
]);
/** Execute function */
@@ -470,8 +473,8 @@ class Functions extends Action
method: $method,
headers: $headers,
runtimeEntrypoint: $command,
cpus: $function->getAttribute('cpus', 1),
memory: $function->getAttribute('memory', 512),
cpus: $spec['cpus'] ?? 1,
memory: $spec['memory'] ?? 512,
);
$status = $executionResponse['statusCode'] >= 400 ? 'failed' : 'completed';
@@ -509,8 +512,8 @@ class Functions extends Action
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->addMetric(METRIC_EXECUTIONS_MB_SECONDS, $function->getAttribute('memory', 512) * $execution->getAttribute('duration', 0) * $function->getAttribute('cpus', 1))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), $function->getAttribute('memory', 512) * $execution->getAttribute('duration', 0) * $function->getAttribute('cpus', 1))
->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? 512) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? 1)))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? 512) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? 1)))
->trigger()
;
}
@@ -386,69 +386,6 @@ class FunctionsCustomServerTest extends Scope
/**
* @depends testUpdate
*/
public function testUpdateSpecs($data): array
{
/**
* Test for SUCCESS
*/
$response1 = $this->client->call(Client::METHOD_PUT, '/functions/' . $data['functionId'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Test1',
'events' => [
'users.*.update.name',
'users.*.update.email',
],
'timeout' => 15,
'runtime' => 'php-8.0',
'entrypoint' => 'index.php',
'size' => 's-1vcpu-1gb',
]);
$this->assertEquals(200, $response1['headers']['status-code']);
$this->assertNotEmpty($response1['body']['$id']);
$this->assertEquals('Test1', $response1['body']['name']);
$dateValidator = new DatetimeValidator();
$this->assertEquals(true, $dateValidator->isValid($response1['body']['$createdAt']));
$this->assertEquals(true, $dateValidator->isValid($response1['body']['$updatedAt']));
$this->assertEquals('', $response1['body']['deployment']);
$this->assertEquals([
'users.*.update.name',
'users.*.update.email',
], $response1['body']['events']);
$this->assertEquals(15, $response1['body']['timeout']);
$this->assertEquals(1024, $response1['body']['memory']);
$this->assertEquals(1, $response1['body']['cpus']);
$this->assertEquals('s-1vcpu-1gb', $response1['body']['size']);
/**
* Test for FAILURE
*/
$response2 = $this->client->call(Client::METHOD_PUT, '/functions/' . $data['functionId'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Test1',
'events' => [
'users.*.update.name',
'users.*.update.email',
],
'timeout' => 15,
'runtime' => 'php-8.0',
'entrypoint' => 'index.php',
'size' => 's-2vcpu-512mb', // Invalid size
]);
$this->assertEquals(400, $response2['headers']['status-code']);
$this->assertEquals('Invalid `size` param: String must be a valid size value of s-1vcpu-512mb, s-1vcpu-1gb', $response2['body']['message']);
return $data;
}
/**
* @depends testUpdateSpecs
*/
public function testCreateDeployment($data): array
{
/**
@@ -899,6 +836,99 @@ class FunctionsCustomServerTest extends Scope
return $data;
}
/**
* @depends testGetExecution
*/
#[Retry(count: 2)]
public function testUpdateSpecs($data): array
{
/**
* Test for SUCCESS
*/
$response1 = $this->client->call(Client::METHOD_PUT, '/functions/' . $data['functionId'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Test1',
'events' => [
'users.*.update.name',
'users.*.update.email',
],
'timeout' => 15,
'runtime' => 'php-8.0',
'entrypoint' => 'index.php',
'size' => 's-1vcpu-1gb',
]);
$this->assertEquals(200, $response1['headers']['status-code']);
$this->assertNotEmpty($response1['body']['$id']);
$this->assertEquals('s-1vcpu-1gb', $response1['body']['size']);
// Test Execution
$execution = $this->client->call(Client::METHOD_POST, '/functions/' . $data['functionId'] . '/executions', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$output = json_decode($execution['body']['responseBody'], true);
$this->assertEquals(1, $output['APPWRITE_FUNCTION_CPUS']);
$this->assertEquals(1024, $output['APPWRITE_FUNCTION_MEMORY']);
$response2 = $this->client->call(Client::METHOD_PUT, '/functions/' . $data['functionId'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Test1',
'events' => [
'users.*.update.name',
'users.*.update.email',
],
'timeout' => 15,
'runtime' => 'php-8.0',
'entrypoint' => 'index.php',
'size' => 's-1vcpu-512mb',
]);
$this->assertEquals(200, $response2['headers']['status-code']);
$this->assertNotEmpty($response2['body']['$id']);
$this->assertEquals('s-1vcpu-512mb', $response2['body']['size']);
// Test Execution
$execution = $this->client->call(Client::METHOD_POST, '/functions/' . $data['functionId'] . '/executions', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$output = json_decode($execution['body']['responseBody'], true);
$this->assertEquals(1, $output['APPWRITE_FUNCTION_CPUS']);
$this->assertEquals(512, $output['APPWRITE_FUNCTION_MEMORY']);
/**
* Test for FAILURE
*/
$response3 = $this->client->call(Client::METHOD_PUT, '/functions/' . $data['functionId'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Test1',
'events' => [
'users.*.update.name',
'users.*.update.email',
],
'timeout' => 15,
'runtime' => 'php-8.0',
'entrypoint' => 'index.php',
'size' => 's-2vcpu-512mb', // Invalid size
]);
$this->assertEquals(400, $response3['headers']['status-code']);
$this->assertEquals('Invalid `size` param: String must be a valid size value of s-1vcpu-512mb, s-1vcpu-1gb', $response3['body']['message']);
return $data;
}
/**
* @depends testGetExecution
*/
@@ -8,6 +8,8 @@ return function ($context) {
'APPWRITE_FUNCTION_TRIGGER' => $context->req->headers['x-appwrite-trigger'] ?? '',
'APPWRITE_FUNCTION_RUNTIME_NAME' => \getenv('APPWRITE_FUNCTION_RUNTIME_NAME') ?: '',
'APPWRITE_FUNCTION_RUNTIME_VERSION' => \getenv('APPWRITE_FUNCTION_RUNTIME_VERSION') ?: '',
'UNICODE_TEST' => "êä"
'UNICODE_TEST' => "êä",
'APPWRITE_FUNCTION_CPUS' => \getenv('APPWRITE_FUNCTION_CPUS') ?: '',
'APPWRITE_FUNCTION_MEMORY' => \getenv('APPWRITE_FUNCTION_MEMORY') ?: '',
]);
};
+3 -1
View File
@@ -9,6 +9,8 @@ return function ($context) {
'APPWRITE_FUNCTION_RUNTIME_NAME' => \getenv('APPWRITE_FUNCTION_RUNTIME_NAME') ?: '',
'APPWRITE_FUNCTION_RUNTIME_VERSION' => \getenv('APPWRITE_FUNCTION_RUNTIME_VERSION') ?: '',
'UNICODE_TEST' => "êä",
'GLOBAL_VARIABLE' => \getenv('GLOBAL_VARIABLE') ?: ''
'GLOBAL_VARIABLE' => \getenv('GLOBAL_VARIABLE') ?: '',
'APPWRITE_FUNCTION_CPUS' => \getenv('APPWRITE_FUNCTION_CPUS') ?: '',
'APPWRITE_FUNCTION_MEMORY' => \getenv('APPWRITE_FUNCTION_MEMORY') ?: '',
]);
};