functions/builds/deletes worker

This commit is contained in:
shimon
2023-06-11 13:29:04 +03:00
parent 62c619434e
commit fd4f86b1c1
6 changed files with 293 additions and 325 deletions
+203 -227
View File
@@ -6,14 +6,14 @@ use Appwrite\Event\Build;
use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Event\Validator\Event as ValidatorEvent;
use Appwrite\Extend\Exception;
use Appwrite\Utopia\Database\Validator\CustomId;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\UID;
use Appwrite\Usage\Stats;
use Utopia\Storage\Device;
use Utopia\Storage\Validator\File;
use Utopia\Storage\Validator\FileExt;
@@ -58,7 +58,7 @@ App::post('/v1/functions')
->label('sdk.response.code', Response::STATUS_CODE_CREATED)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_FUNCTION)
->param('functionId', '', new CustomId(), 'Function ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
->param('functionId', '', new CustomId(), 'Function ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
->param('name', '', new Text(128), 'Function name. Max length: 128 chars.')
->param('execute', [], new Roles(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of strings with execution roles. By default no user is granted with any execute permissions. [learn more about permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 64 characters long.')
->param('runtime', '', new WhiteList(array_keys(Config::getParam('runtimes')), true), 'Execution runtime.')
@@ -81,10 +81,11 @@ App::post('/v1/functions')
'enabled' => $enabled,
'name' => $name,
'runtime' => $runtime,
'deploymentInternalId' => '',
'deployment' => '',
'events' => $events,
'schedule' => $schedule,
'scheduleUpdatedAt' => DateTime::now(),
'scheduleInternalId' => '',
'timeout' => $timeout,
'search' => implode(' ', [$functionId, $name, $runtime])
]));
@@ -94,6 +95,7 @@ App::post('/v1/functions')
'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region
'resourceType' => 'function',
'resourceId' => $function->getId(),
'resourceInternalId' => $function->getInternalId(),
'resourceUpdatedAt' => DateTime::now(),
'projectId' => $project->getId(),
'schedule' => $function->getAttribute('schedule'),
@@ -102,6 +104,7 @@ App::post('/v1/functions')
);
$function->setAttribute('scheduleId', $schedule->getId());
$function->setAttribute('scheduleInternalId', $schedule->getInternalId());
$dbForProject->updateDocument('functions', $function->getId(), $function);
$queueForEvents->setParam('functionId', $function->getId());
@@ -230,92 +233,66 @@ App::get('/v1/functions/:functionId/usage')
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS),
str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE),
];
$metrics = [
"executions.$functionId.compute.total",
"executions.$functionId.compute.success",
"executions.$functionId.compute.failure",
"executions.$functionId.compute.time",
"builds.$functionId.compute.total",
"builds.$functionId.compute.success",
"builds.$functionId.compute.failure",
"builds.$functionId.compute.time",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'executionsTotal' => $stats["executions.$functionId.compute.total"] ?? [],
'executionsFailure' => $stats["executions.$functionId.compute.failure"] ?? [],
'executionsSuccesse' => $stats["executions.$functionId.compute.success"] ?? [],
'executionsTime' => $stats["executions.$functionId.compute.time"] ?? [],
'buildsTotal' => $stats["builds.$functionId.compute.total"] ?? [],
'buildsFailure' => $stats["builds.$functionId.compute.failure"] ?? [],
'buildsSuccess' => $stats["builds.$functionId.compute.success"] ?? [],
'buildsTime' => $stats["builds.$functionId.compute.time" ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTION);
$response->dynamic(new Document([
'range' => $range,
'deploymentsTotal' => $usage[$metrics[0]],
'deploymentsStorage' => $usage[$metrics[1]],
'buildsTotal' => $usage[$metrics[2]],
'buildsStorage' => $usage[$metrics[3]],
'buildsTime' => $usage[$metrics[4]],
'executionsTotal' => $usage[$metrics[5]],
'executionsTime' => $usage[$metrics[6]],
]), Response::MODEL_USAGE_FUNCTION);
});
App::get('/v1/functions/usage')
@@ -333,92 +310,67 @@ App::get('/v1/functions/usage')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_FUNCTIONS,
METRIC_DEPLOYMENTS,
METRIC_DEPLOYMENTS_STORAGE,
METRIC_BUILDS,
METRIC_BUILDS_STORAGE,
METRIC_BUILDS_COMPUTE,
METRIC_EXECUTIONS,
METRIC_EXECUTIONS_COMPUTE,
];
$metrics = [
'executions.$all.compute.total',
'executions.$all.compute.failure',
'executions.$all.compute.success',
'executions.$all.compute.time',
'builds.$all.compute.total',
'builds.$all.compute.failure',
'builds.$all.compute.success',
'builds.$all.compute.time',
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'executionsTotal' => $stats[$metrics[0]] ?? [],
'executionsFailure' => $stats[$metrics[1]] ?? [],
'executionsSuccess' => $stats[$metrics[2]] ?? [],
'executionsTime' => $stats[$metrics[3]] ?? [],
'buildsTotal' => $stats[$metrics[4]] ?? [],
'buildsFailure' => $stats[$metrics[5]] ?? [],
'buildsSuccess' => $stats[$metrics[6]] ?? [],
'buildsTime' => $stats[$metrics[7]] ?? [],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS);
}
$response->dynamic(new Document([
'range' => $range,
'functionsTotal' => $usage[$metrics[0]],
'deploymentsTotal' => $usage[$metrics[1]],
'deploymentsStorage' => $usage[$metrics[2]],
'buildsTotal' => $usage[$metrics[3]],
'buildsStorage' => $usage[$metrics[4]],
'buildsTime' => $usage[$metrics[5]],
'executionsTotal' => $usage[$metrics[6]],
'executionsTime' => $usage[$metrics[7]],
]), Response::MODEL_USAGE_FUNCTIONS);
});
App::put('/v1/functions/:functionId')
@@ -463,26 +415,16 @@ App::put('/v1/functions/:functionId')
'name' => $name,
'events' => $events,
'schedule' => $schedule,
'scheduleUpdatedAt' => DateTime::now(),
'timeout' => $timeout,
'enabled' => $enabled,
'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]),
])));
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
/**
* In case we want to clear the schedule
*/
if (!empty($function->getAttribute('deployment'))) {
$schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt'));
}
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$queueForEvents->setParam('functionId', $function->getId());
@@ -534,19 +476,15 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId')
}
$function = $dbForProject->updateDocument('functions', $function->getId(), new Document(array_merge($function->getArrayCopy(), [
'deployment' => $deployment->getId()
'deploymentInternalId' => $deployment->getInternalId(),
'deployment' => $deployment->getId(),
])));
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$active = !empty($function->getAttribute('schedule'));
if ($active) {
$schedule->setAttribute('resourceUpdatedAt', datetime::now());
}
$schedule->setAttribute('active', $active);
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$queueForEvents
@@ -589,12 +527,9 @@ App::delete('/v1/functions/:functionId')
}
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('active', false)
;
->setAttribute('active', false);
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$queueForDeletes
@@ -644,14 +579,20 @@ App::post('/v1/functions/:functionId/deployments')
}
$file = $request->getFiles('code');
$fileExt = new FileExt([FileExt::TYPE_GZIP]);
$fileSizeValidator = new FileSize(App::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', 0));
$upload = new Upload();
// GraphQL multipart spec adds files with index keys
if (empty($file)) {
$file = $request->getFiles(0);
}
if (empty($file)) {
throw new Exception(Exception::STORAGE_FILE_EMPTY, 'No file sent');
}
$fileExt = new FileExt([FileExt::TYPE_GZIP]);
$fileSizeValidator = new FileSize(App::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', 0));
$upload = new Upload();
// Make sure we handle a single file and multiple files the same way
$fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
@@ -696,7 +637,6 @@ App::post('/v1/functions/:functionId/deployments')
// Save to storage
$fileSize ??= $deviceLocal->getFileSize($fileTmpName);
$path = $deviceFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)];
@@ -741,8 +681,10 @@ App::post('/v1/functions/:functionId/deployments')
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'resourceInternalId' => $function->getInternalId(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'buildInternalId' => '',
'entrypoint' => $entrypoint,
'path' => $path,
'size' => $fileSize,
@@ -770,8 +712,10 @@ App::post('/v1/functions/:functionId/deployments')
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'resourceInternalId' => $function->getInternalId(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'buildInternalId' => '',
'entrypoint' => $entrypoint,
'path' => $path,
'size' => $fileSize,
@@ -786,22 +730,6 @@ App::post('/v1/functions/:functionId/deployments')
}
}
/**
* TODO Should we update also the function collection with the scheduleUpdatedAt attr?
*/
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$active = !empty($function->getAttribute('schedule'));
if ($active) {
$schedule->setAttribute('resourceUpdatedAt', datetime::now());
}
$schedule->setAttribute('active', $active);
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$metadata = null;
$queueForEvents
@@ -1063,10 +991,10 @@ App::post('/v1/functions/:functionId/executions')
->inject('dbForProject')
->inject('user')
->inject('queueForEvents')
->inject('usage')
->inject('mode')
->inject('queueForFunctions')
->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, Stats $usage, string $mode, Func $queueForFunctions) {
->inject('queueForUsage')
->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, string $mode, Func $queueForFunctions, Usage $queueForUsage) {
$function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId));
@@ -1116,7 +1044,9 @@ App::post('/v1/functions/:functionId/executions')
$execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', new Document([
'$id' => $executionId,
'$permissions' => !$user->isEmpty() ? [Permission::read(Role::user($user->getId()))] : [],
'functionInternalId' => $function->getInternalId(),
'functionId' => $function->getId(),
'deploymentInternalId' => $deployment->getInternalId(),
'deploymentId' => $deployment->getId(),
'trigger' => 'http', // http / schedule / event
'status' => $async ? 'waiting' : 'processing', // waiting / processing / completed / failed
@@ -1197,7 +1127,7 @@ App::post('/v1/functions/:functionId/executions')
variables: $vars,
timeout: $function->getAttribute('timeout', 0),
image: $runtime['image'],
source: $build->getAttribute('path', ''),
source: $build->getAttribute('outputPath', ''),
entrypoint: $deployment->getAttribute('entrypoint', ''),
);
@@ -1208,6 +1138,13 @@ App::post('/v1/functions/:functionId/executions')
$execution->setAttribute('stdout', $executionResponse['stdout']);
$execution->setAttribute('stderr', $executionResponse['stderr']);
$execution->setAttribute('duration', $executionResponse['duration']);
/**
* Sync execution compute usage from
*/
$queueForUsage
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($executionResponse['duration'] * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($executionResponse['duration'] * 1000))// per function
;
} catch (\Throwable $th) {
$interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt()));
$execution
@@ -1220,13 +1157,6 @@ App::post('/v1/functions/:functionId/executions')
Authorization::skip(fn () => $dbForProject->updateDocument('executions', $executionId, $execution));
// TODO revise this later using route label
$usage
->setParam('functionId', $function->getId())
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration')); // ms
$roles = Authorization::getRoles();
$isPrivilegedUser = Auth::isPrivilegedUser($roles);
$isAppUser = Auth::isAppUser($roles);
@@ -1381,7 +1311,8 @@ App::post('/v1/functions/:functionId/variables')
->param('value', null, new Text(8192), 'Variable value. Max length: 8192 chars.', false)
->inject('response')
->inject('dbForProject')
->action(function (string $functionId, string $key, string $value, Response $response, Database $dbForProject) {
->inject('dbForConsole')
->action(function (string $functionId, string $key, string $value, Response $response, Database $dbForProject, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
if ($function->isEmpty()) {
@@ -1410,8 +1341,22 @@ App::post('/v1/functions/:functionId/variables')
throw new Exception(Exception::VARIABLE_ALREADY_EXISTS);
}
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$dbForProject->deleteCachedDocument('functions', $function->getId());
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($variable, Response::MODEL_VARIABLE);
@@ -1497,7 +1442,8 @@ App::put('/v1/functions/:functionId/variables/:variableId')
->param('value', null, new Text(8192), 'Variable value. Max length: 8192 chars.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $functionId, string $variableId, string $key, ?string $value, Response $response, Database $dbForProject) {
->inject('dbForConsole')
->action(function (string $functionId, string $variableId, string $key, ?string $value, Response $response, Database $dbForProject, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
@@ -1526,8 +1472,22 @@ App::put('/v1/functions/:functionId/variables/:variableId')
throw new Exception(Exception::VARIABLE_ALREADY_EXISTS);
}
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$dbForProject->deleteCachedDocument('functions', $function->getId());
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$response->dynamic($variable, Response::MODEL_VARIABLE);
});
@@ -1547,7 +1507,8 @@ App::delete('/v1/functions/:functionId/variables/:variableId')
->param('variableId', '', new UID(), 'Variable unique ID.', false)
->inject('response')
->inject('dbForProject')
->action(function (string $functionId, string $variableId, Response $response, Database $dbForProject) {
->inject('dbForConsole')
->action(function (string $functionId, string $variableId, Response $response, Database $dbForProject, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
if ($function->isEmpty()) {
@@ -1564,7 +1525,22 @@ App::delete('/v1/functions/:functionId/variables/:variableId')
}
$dbForProject->deleteDocument('variables', $variable->getId());
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$dbForProject->deleteCachedDocument('functions', $function->getId());
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$response->noContent();
});
-9
View File
@@ -207,7 +207,6 @@ App::get('/v1/projects')
$cursor = Query::getByType($queries, Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE);
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
$projectId = $cursor->getValue();
$cursorDocument = $dbForConsole->getDocument('projects', $projectId);
@@ -538,8 +537,6 @@ App::delete('/v1/projects/:projectId')
$response->noContent();
});
// Webhooks
App::post('/v1/projects/:projectId/webhooks')
->desc('Create Webhook')
->groups(['api', 'projects'])
@@ -792,8 +789,6 @@ App::delete('/v1/projects/:projectId/webhooks/:webhookId')
$response->noContent();
});
// Keys
App::post('/v1/projects/:projectId/keys')
->desc('Create Key')
->groups(['api', 'projects'])
@@ -994,8 +989,6 @@ App::delete('/v1/projects/:projectId/keys/:keyId')
$response->noContent();
});
// Platforms
App::post('/v1/projects/:projectId/platforms')
->desc('Create Platform')
->groups(['api', 'projects'])
@@ -1197,8 +1190,6 @@ App::delete('/v1/projects/:projectId/platforms/:platformId')
$response->noContent();
});
// Domains
App::post('/v1/projects/:projectId/domains')
->desc('Create Domain')
->groups(['api', 'projects'])
+23 -31
View File
@@ -160,38 +160,15 @@ Server::setResource('pools', function ($register) {
return $register->get('pools');
}, ['register']);
/**
* Get Console DB
*
* @returns Cache
*/
function getCache(): Cache
{
global $register;
$pools = $register->get('pools');
/** @var Group $pools */
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource();
}
return new Cache(new Sharding($adapters));
}
/**
* Get Functions Storage Device
* @param string $projectId of the project
* @return Device
*/
Server::setResource('getFunctionsDevice', function (string $projectId) {
return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId);
Server::setResource('deviceFunctions', function () {
return function (string $projectId) {
return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId);
};
});
/**
@@ -199,8 +176,10 @@ Server::setResource('getFunctionsDevice', function (string $projectId) {
* @param string $projectId of the project
* @return Device
*/
Server::setResource('getFilesDevice', function (string $projectId) {
return getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId);
Server::setResource('deviceFiles', function () {
return function (string $projectId) {
return getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId);
};
});
/**
@@ -208,8 +187,21 @@ Server::setResource('getFilesDevice', function (string $projectId) {
* @param string $projectId of the project
* @return Device
*/
Server::setResource('getBuildsDevice', function (string $projectId) {
return getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId);
Server::setResource('deviceBuilds', function () {
return function (string $projectId) {
return getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId);
};
});
/**
* Get cache Device
* @param string $projectId of the project
* @return Device
*/
Server::setResource('deviceCache', function () {
return function (string $projectId) {
return getDevice(APP_STORAGE_CACHE . '/app-' . $projectId);
};
});
$pools = $register->get('pools');
+17 -13
View File
@@ -38,17 +38,18 @@ class Builds extends Action
$this
->desc('Builds worker')
->inject('message')
->inject('dbForConsole')
->inject('dbForProject')
->inject('queueForEvents')
->inject('queueForFunctions')
->inject('queueForUsage')
->callback(fn($message, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage) => $this->action($message, $dbForProject, $queueForEvents, $queueForFunctions, $queueForUsage));
->callback(fn($message, Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage) => $this->action($message, $dbForConsole, $dbForProject, $queueForEvents, $queueForFunctions, $queueForUsage));
}
/**
* @throws Exception|\Throwable
*/
public function action(Message $message, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage): void
public function action(Message $message, Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage): void
{
$payload = $message->getPayload() ?? [];
@@ -66,6 +67,7 @@ class Builds extends Action
case BUILD_TYPE_RETRY:
Console::info('Creating build for deployment: ' . $deployment->getId());
$this->buildDeployment(
dbForConsole: $dbForConsole,
dbForProject: $dbForProject,
queueForEvents: $queueForEvents,
queueForFunctions: $queueForFunctions,
@@ -86,7 +88,7 @@ class Builds extends Action
* @throws \Throwable
* @throws Structure
*/
private function buildDeployment(Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Document $deployment, Document $project, Document $function): void
private function buildDeployment(Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Document $deployment, Document $project, Document $function): void
{
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
$function = $dbForProject->getDocument('functions', $function->getId());
@@ -126,10 +128,10 @@ class Builds extends Action
'$id' => $buildId,
'$permissions' => [],
'startTime' => $startTime,
'deploymentInternalId' => $deployment->getInternalId(),
'deploymentId' => $deployment->getId(),
'status' => 'processing',
'path' => '',
'size' => 0,
'outputPath' => '',
'runtime' => $function->getAttribute('runtime'),
'source' => $deployment->getAttribute('path'),
'sourceType' => $device,
@@ -139,6 +141,7 @@ class Builds extends Action
]));
$deployment->setAttribute('buildId', $buildId);
$deployment->setAttribute('buildInternalId', $build->getInternalId());
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
} else {
$build = $dbForProject->getDocument('builds', $buildId);
@@ -211,16 +214,18 @@ class Builds extends Action
commands: [
'sh', '-c',
'tar -zxf /tmp/code.tar.gz -C /usr/code && \
cd /usr/local/src/ && ./build.sh'
cd /usr/local/src/ && ./build.sh'
]
);
$endTime = new \DateTime();
$endTime->setTimestamp($response['endTimeUnix']);
/** Update the build document */
$build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime'])));
$build->setAttribute('endTime', DateTime::format($endTime));
$build->setAttribute('duration', \intval($response['duration']));
$build->setAttribute('status', $response['status']);
$build->setAttribute('path', $response['path']);
$build->setAttribute('size', $response['size']);
$build->setAttribute('outputPath', $response['outputPath']);
$build->setAttribute('stderr', $response['stderr']);
$build->setAttribute('stdout', $response['stdout']);
@@ -229,18 +234,16 @@ class Builds extends Action
Console::success("Build id: $buildId created");
$function->setAttribute('scheduleUpdatedAt', DateTime::now());
/** Set auto deploy */
if ($deployment->getAttribute('activate') === true) {
$function->setAttribute('deploymentInternalId', $deployment->getInternalId());
$function->setAttribute('deployment', $deployment->getId());
$function = $dbForProject->updateDocument('functions', $function->getId(), $function);
}
/** Update function schedule */
$dbForConsole = getConsoleDB();
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt'));
$schedule->setAttribute('resourceUpdatedAt', DateTime::now());
$schedule
->setAttribute('schedule', $function->getAttribute('schedule'))
@@ -251,6 +254,7 @@ class Builds extends Action
} catch (\Throwable $th) {
$endTime = DateTime::now();
$interval = (new \DateTime($endTime))->diff(new \DateTime($startTime));
$build->setAttribute('endTime', $endTime);
$build->setAttribute('duration', $interval->format('%s') + 0);
$build->setAttribute('status', 'failed');
$build->setAttribute('stderr', $th->getMessage());
+40 -40
View File
@@ -35,18 +35,18 @@ class Deletes extends Action
->inject('message')
->inject('dbForConsole')
->inject('getProjectDB')
->inject('getFilesDevice')
->inject('getFunctionsDevice')
->inject('getBuildsDevice')
->inject('getCacheDevice')
->callback(fn($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice));
->inject('deviceFiles')
->inject('deviceFunctions')
->inject('deviceBuilds')
->inject('deviceCache')
->callback(fn($message, $dbForConsole, $getProjectDB, $deviceFiles, $deviceFunctions, $deviceBuilds, $deviceCache) => $this->action($message, $dbForConsole, $getProjectDB, $deviceFiles, $deviceFunctions, $deviceBuilds, $deviceCache));
}
/**
* @throws Exception
* @throws \Throwable
*/
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice): void
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $deviceFiles, callable $deviceFunctions, callable $deviceBuilds, callable $deviceCache): void
{
$payload = $message->getPayload() ?? [];
@@ -71,13 +71,13 @@ class Deletes extends Action
$this->deleteCollection($getProjectDB, $document, $project);
break;
case DELETE_TYPE_PROJECTS:
$this->deleteProject($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $document);
$this->deleteProject($dbForConsole, $getProjectDB, $deviceFiles, $deviceFunctions, $deviceBuilds, $deviceCache, $document);
break;
case DELETE_TYPE_FUNCTIONS:
$this->deleteFunction($getProjectDB, $getFunctionsDevice, $getBuildsDevice, $document, $project);
$this->deleteFunction($getProjectDB, $deviceFunctions, $deviceBuilds, $document, $project);
break;
case DELETE_TYPE_DEPLOYMENTS:
$this->deleteDeployment($getProjectDB, $getFunctionsDevice, $getBuildsDevice, $document, $project);
$this->deleteDeployment($getProjectDB, $deviceFunctions, $deviceBuilds, $document, $project);
break;
case DELETE_TYPE_USERS:
$this->deleteUser($getProjectDB, $document, $project);
@@ -86,7 +86,7 @@ class Deletes extends Action
$this->deleteMemberships($getProjectDB, $document, $project);
break;
case DELETE_TYPE_BUCKETS:
$this->deleteBucket($getProjectDB, $getFilesDevice, $document, $project);
$this->deleteBucket($getProjectDB, $deviceFiles, $document, $project);
break;
default:
if (\str_starts_with($document->getCollection(), 'database_')) {
@@ -134,10 +134,10 @@ class Deletes extends Action
break;
case DELETE_TYPE_CACHE_BY_RESOURCE:
$this->deleteCacheByResource($getProjectDB, $resource);
$this->deleteCacheByResource($dbForConsole, $getProjectDB, $resource);
break;
case DELETE_TYPE_CACHE_BY_TIMESTAMP:
$this->deleteCacheByDate($getProjectDB);
$this->deleteCacheByDate($dbForConsole, $getProjectDB, $datetime);
break;
case DELETE_TYPE_SCHEDULES:
$this->deleteSchedules($dbForConsole, $getProjectDB, $datetime);
@@ -190,9 +190,9 @@ class Deletes extends Action
* @param string $resource
* @throws Exception
*/
protected function deleteCacheByResource(callable $getProjectDB, string $resource): void
protected function deleteCacheByResource(Database $dbForConsole, callable $getProjectDB, string $resource): void
{
$this->deleteCacheFiles($getProjectDB, [
$this->deleteCacheFiles($dbForConsole, $getProjectDB, [
Query::equal('resource', [$resource]),
]);
}
@@ -200,10 +200,10 @@ class Deletes extends Action
/**
* @throws Exception
*/
protected function deleteCacheByDate(callable $getProjectDB): void
protected function deleteCacheByDate(Database $dbForConsole, callable $getProjectDB, string $datetime): void
{
$this->deleteCacheFiles($getProjectDB, [
Query::lessThan('accessedAt', $this->args['datetime']),
$this->deleteCacheFiles($dbForConsole, $getProjectDB, [
Query::lessThan('accessedAt', $datetime),
]);
}
@@ -323,14 +323,14 @@ class Deletes extends Action
/**
* @param Database $dbForConsole
* @param callable $getProjectDB
* @param callable $getFilesDevice
* @param callable $getFunctionsDevice
* @param callable $getBuildsDevice
* @param callable $getCacheDevice
* @param callable $deviceFiles
* @param callable $deviceFunctions
* @param callable $deviceBuilds
* @param callable $deviceCache
* @param Document $document project document
* @throws Authorization
*/
protected function deleteProject(Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Document $document): void
protected function deleteProject(Database $dbForConsole, callable $getProjectDB, callable $deviceFiles, callable $deviceFunctions, callable $deviceBuilds, callable $deviceCache, Document $document): void
{
$projectId = $document->getId();
@@ -344,7 +344,7 @@ class Deletes extends Action
}
// Delete project tables
$dbForProject = $getProjectDB($projectId, $document);
$dbForProject = $getProjectDB($document);
while (true) {
$collections = $dbForProject->listCollections();
@@ -367,10 +367,10 @@ class Deletes extends Action
}
// Delete all storage directories
$uploads = $getFilesDevice($projectId);
$functions = $getFunctionsDevice($projectId);
$builds = $getBuildsDevice($projectId);
$cache = $getCacheDevice($projectId);
$uploads = $deviceFiles($projectId);
$functions = $deviceFunctions($projectId);
$builds = $deviceBuilds($projectId);
$cache = $deviceCache($projectId);
$uploads->delete($uploads->getRoot(), true);
$functions->delete($functions->getRoot(), true);
@@ -540,13 +540,13 @@ class Deletes extends Action
/**
* @param callable $getProjectDB
* @param callable $getFunctionsDevice
* @param callable $getBuildsDevice
* @param callable $deviceFunctions
* @param callable $deviceBuilds
* @param Document $document function document
* @param Document $project
* @throws Exception
*/
protected function deleteFunction(callable $getProjectDB, callable $getFunctionsDevice, callable $getBuildsDevice, Document $document, Document $project): void
protected function deleteFunction(callable $getProjectDB, callable $deviceFunctions, callable $deviceBuilds, Document $document, Document $project): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
@@ -564,7 +564,7 @@ class Deletes extends Action
* Delete Deployments
*/
Console::info("Deleting deployments for function " . $functionId);
$storageFunctions = $getFunctionsDevice($projectId);
$storageFunctions = $deviceFunctions($projectId);
$deploymentIds = [];
$this->deleteByGroup('deployments', [
Query::equal('resourceId', [$functionId])
@@ -581,7 +581,7 @@ class Deletes extends Action
* Delete builds
*/
Console::info("Deleting builds for function " . $functionId);
$storageBuilds = $getBuildsDevice($projectId);
$storageBuilds = $deviceBuilds($projectId);
foreach ($deploymentIds as $deploymentId) {
$this->deleteByGroup('builds', [
Query::equal('deploymentId', [$deploymentId])
@@ -607,13 +607,13 @@ class Deletes extends Action
/**
* @param callable $getProjectDB
* @param callable $getFunctionsDevice
* @param callable $getBuildsDevice
* @param callable $deviceFunctions
* @param callable $deviceBuilds
* @param Document $document deployment document
* @param Document $project
* @throws Exception
*/
protected function deleteDeployment(callable $getProjectDB, callable $getFunctionsDevice, callable $getBuildsDevice, Document $document, Document $project): void
protected function deleteDeployment(callable $getProjectDB, callable $deviceFunctions, callable $deviceBuilds, Document $document, Document $project): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
@@ -624,7 +624,7 @@ class Deletes extends Action
* Delete deployment files
*/
Console::info("Deleting deployment files for deployment " . $deploymentId);
$storageFunctions = $getFunctionsDevice($projectId);
$storageFunctions = $deviceFunctions($projectId);
if ($storageFunctions->delete($document->getAttribute('path', ''), true)) {
Console::success('Deleted deployment files: ' . $document->getAttribute('path', ''));
} else {
@@ -635,7 +635,7 @@ class Deletes extends Action
* Delete builds
*/
Console::info("Deleting builds for deployment " . $deploymentId);
$storageBuilds = $getBuildsDevice($projectId);
$storageBuilds = $deviceBuilds($projectId);
$this->deleteByGroup('builds', [
Query::equal('deploymentId', [$deploymentId])
], $dbForProject, function (Document $document) use ($storageBuilds) {
@@ -832,19 +832,19 @@ class Deletes extends Action
/**
* @param callable $getProjectDB
* @param callable $getFilesDevice
* @param callable $deviceFiles
* @param Document $document
* @param Document $project
* @return void
*/
protected function deleteBucket(callable $getProjectDB, callable $getFilesDevice, Document $document, Document $project): void
protected function deleteBucket(callable $getProjectDB, callable $deviceFiles, Document $document, Document $project): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$dbForProject->deleteCollection('bucket_' . $document->getInternalId());
$device = $getFilesDevice($projectId);
$device = $deviceFiles($projectId);
$device->deletePath($document->getId());
}
+10 -5
View File
@@ -40,14 +40,15 @@ class Functions extends Action
->inject('message')
->inject('dbForProject')
->inject('queueForFunctions')
->inject('queueForEvents')
->inject('queueForUsage')
->callback(fn($message, $dbForProject, $queueForFunctions, $queueForUsage) => $this->action($message, $dbForProject, $queueForFunctions, $queueForUsage));
->callback(fn($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForUsage) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForUsage));
}
/**
* @throws Exception|\Throwable
*/
public function action(Message $message, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage): void
public function action(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage): void
{
$payload = $message->getPayload() ?? [];
@@ -100,6 +101,7 @@ class Functions extends Action
$this->execute(
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
queueForEvents: $queueForEvents,
queueForUsage: $queueForUsage,
project: $project,
function: $function,
@@ -125,6 +127,7 @@ class Functions extends Action
$this->execute(
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
queueForEvents: $queueForEvents,
queueForUsage: $queueForUsage,
project: $project,
function: $function,
@@ -156,6 +159,7 @@ class Functions extends Action
private function execute(
Database $dbForProject,
Func $queueForFunctions,
Event $queueForEvents,
Usage $queueForUsage,
Document $project,
Document $function,
@@ -299,8 +303,9 @@ class Functions extends Action
/** Trigger Webhook */
$executionModel = new Execution();
$executionUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME);
$executionUpdate
$queueForEvents
->setQueue(Event::WEBHOOK_QUEUE_NAME)
->setClass(Event::WEBHOOK_CLASS_NAME)
->setProject($project)
->setUser($user)
->setEvent('functions.[functionId].executions.[executionId].update')
@@ -311,7 +316,7 @@ class Functions extends Action
/** Trigger Functions */
$queueForFunctions
->from($executionUpdate)
->from($queueForEvents)
->trigger();
/** Trigger realtime event */