Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'refactor-usage-sn' of github.com:appwrite/appwrite into refactor-hamster-usage

Browse Source 
 Conflicts:
	src/Appwrite/Platform/Tasks/CalcTierStats.php
This commit is contained in:
shimon
2023-12-21 14:25:06 +02:00
parent eff578bb36 9c510dace8
commit 4e24e3cdc2
22 changed files with 325 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Appwrite/Hooks/Hooks.php
+26
View File
@@ -0,0 +1,26 @@
<?php
namespace Appwrite\Hooks;
class Hooks
{
/**
* @var callable[] $hooks
*/
private static array $hooks = [];
public static function add(string $name, callable $action)
{
self::$hooks[$name] = $action;
}
/**
* @param mixed[] $params
*/
public function trigger(string $name, array $params = [])
{
if (isset(self::$hooks[$name])) {
call_user_func_array(self::$hooks[$name], $params);
}
}
}
src/Appwrite/Platform/Tasks/CreateInfMetric.php
+5 -5
View File
@@ -91,6 +91,8 @@ class CreateInfMetric extends Action
->get($db)
->reclaim();
}
Console::log('Finished project ' . $project->getId() . ' ' . $project->getInternalId());
}
$sum = \count($projects);
@@ -118,11 +120,9 @@ class CreateInfMetric extends Action
try {
$id = \md5("_inf_{$metric}");
$dbForProject->deleteDocument('stats', $id);
$dbForProject->deleteDocument('stats_v2', $id);
echo "_inf_{$metric} , $value \n";
$dbForProject->createDocument('stats', new Document([
$dbForProject->createDocument('stats_v2', new Document([
'$id' => $id,
'metric' => $metric,
'period' => 'inf',
@@ -144,7 +144,7 @@ class CreateInfMetric extends Action
protected function getFromMetric(database $dbForProject, string $metric): int|float
{
return $dbForProject->sum('stats', 'value', [
return $dbForProject->sum('stats_v2', 'value', [
Query::equal('metric', [
$metric,
]),
src/Appwrite/Platform/Tasks/Hamster.php
+1 -1
View File
@@ -240,7 +240,7 @@ class Hamster extends Action
$limit = $periodValue['limit'];
$period = $periodValue['period'];
$requestDocs = $dbForProject->find('stats', [
$requestDocs = $dbForProject->find('stats_v2', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
src/Appwrite/Platform/Tasks/Maintenance.php
+38 -5
View File
@@ -44,18 +44,51 @@ class Maintenance extends Action
$time = DateTime::now();
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
$this->notifyDeleteExecutionLogs($queueForDeletes);
$this->notifyDeleteAbuseLogs($queueForDeletes);
$this->notifyDeleteAuditLogs($queueForDeletes);
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
$this->foreachProject($dbForConsole, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
$queueForDeletes->setProject($project);
$this->notifyDeleteExecutionLogs($queueForDeletes);
$this->notifyDeleteAbuseLogs($queueForDeletes);
$this->notifyDeleteAuditLogs($queueForDeletes);
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
$this->notifyDeleteExpiredSessions($queueForDeletes);
});
$this->notifyDeleteConnections($queueForDeletes);
$this->notifyDeleteExpiredSessions($queueForDeletes);
$this->renewCertificates($dbForConsole, $queueForCertificates);
$this->notifyDeleteCache($cacheRetention, $queueForDeletes);
$this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
}, $interval);
}
protected function foreachProject(Database $dbForConsole, callable $callback): void
{
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
$count = 0;
$chunk = 0;
$limit = 50;
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
$chunk++;
/** @var string[] $projectIds */
$sum = count($projects);
foreach ($projects as $project) {
$callback($project);
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
}
private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void
{
($queueForDeletes)
src/Appwrite/Platform/Workers/Deletes.php
+50 -97
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Auth\Auth;
use Executor\Executor;
use Throwable;
use Utopia\Abuse\Abuse;
@@ -45,17 +46,17 @@ class Deletes extends Action
->inject('getFunctionsDevice')
->inject('getBuildsDevice')
->inject('getCacheDevice')
->inject('getProjectAbuseRetention')
->inject('getProjectExecutionRetention')
->inject('getProjectAuditRetention')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $getProjectAbuseRetention, $getProjectExecutionRetention, $getProjectAuditRetention));
->inject('abuseRetention')
->inject('executionRetention')
->inject('auditRetention')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $abuseRetention, $executionRetention, $auditRetention));
}
/**
* @throws Exception
* @throws Throwable
*/
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, callable $getProjectAbuseRetention, callable $getProjectExecutionRetention, callable $getProjectAuditRetention): void
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention): void
{
$payload = $message->getPayload() ?? [];
@@ -117,12 +118,12 @@ class Deletes extends Action
break;
case DELETE_TYPE_EXECUTIONS:
$this->deleteExecutionLogs($dbForConsole, $getProjectDB, $getProjectExecutionRetention);
$this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
break;
case DELETE_TYPE_AUDIT:
if (!empty($datetime)) {
$this->deleteAuditLogs($dbForConsole, $getProjectDB, $getProjectAuditRetention);
if (!$project->isEmpty()) {
$this->deleteAuditLogs($project, $getProjectDB, $auditRetention);
}
if (!$document->isEmpty()) {
@@ -130,7 +131,7 @@ class Deletes extends Action
}
break;
case DELETE_TYPE_ABUSE:
$this->deleteAbuseLogs($dbForConsole, $getProjectDB, $getProjectAbuseRetention);
$this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention);
break;
case DELETE_TYPE_REALTIME:
@@ -138,10 +139,10 @@ class Deletes extends Action
break;
case DELETE_TYPE_SESSIONS:
$this->deleteExpiredSessions($dbForConsole, $getProjectDB);
$this->deleteExpiredSessions($project, $getProjectDB);
break;
case DELETE_TYPE_USAGE:
$this->deleteUsageStats($dbForConsole, $getProjectDB, $hourlyUsageRetentionDatetime);
$this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
break;
case DELETE_TYPE_CACHE_BY_RESOURCE:
$this->deleteCacheByResource($project, $getProjectDB, $resource);
@@ -340,16 +341,14 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteUsageStats(Database $dbForConsole, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
private function deleteUsageStats(Document $project, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
{
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $hourlyUsageRetentionDatetime) {
$dbForProject = $getProjectDB($project);
// Delete Usage stats
$this->deleteByGroup('stats', [
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::equal('period', ['1h']),
], $dbForProject);
});
$dbForProject = $getProjectDB($project);
// Delete Usage stats
$this->deleteByGroup('stats_v2', [
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::equal('period', ['1h']),
], $dbForProject);
}
/**
@@ -546,16 +545,13 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, callable $getProjectExecutionRetention): void
private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void
{
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectExecutionRetention) {
$dbForProject = $getProjectDB($project);
$datetime = $getProjectExecutionRetention($project);
// Delete Executions
$this->deleteByGroup('executions', [
Query::lessThan('$createdAt', $datetime)
], $dbForProject);
});
$dbForProject = $getProjectDB($project);
// Delete Executions
$this->deleteByGroup('executions', [
Query::lessThan('$createdAt', $datetime)
], $dbForProject);
}
/**
@@ -564,20 +560,16 @@ class Deletes extends Action
* @return void
* @throws Exception|Throwable
*/
private function deleteExpiredSessions(Database $dbForConsole, callable $getProjectDB): void
private function deleteExpiredSessions(Document $project, callable $getProjectDB): void
{
$dbForProject = $getProjectDB($project);
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($dbForConsole, $getProjectDB) {
$dbForProject = $getProjectDB($project);
$project = $dbForConsole->getDocument('projects', $project->getId());
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
// Delete Sessions
$this->deleteByGroup('sessions', [
Query::lessThan('$createdAt', $expired)
], $dbForProject);
});
// Delete Sessions
$this->deleteByGroup('sessions', [
Query::lessThan('$createdAt', $expired)
], $dbForProject);
}
/**
@@ -601,19 +593,16 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAbuseRetention): void
private function deleteAbuseLogs(Document $project, callable $getProjectDB, string $abuseRetention): void
{
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAbuseRetention) {
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$datetime = $getProjectAbuseRetention($project);
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
$abuse = new Abuse($timeLimit);
$status = $abuse->cleanup($datetime);
if (!$status) {
throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
}
});
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
$abuse = new Abuse($timeLimit);
$status = $abuse->cleanup($abuseRetention);
if (!$status) {
throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
}
}
/**
@@ -623,18 +612,15 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, callable $getProjectAuditRetention): void
private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void
{
$this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $getProjectAuditRetention) {
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$datetime = $getProjectAuditRetention($project);
$audit = new Audit($dbForProject);
$status = $audit->cleanup($datetime);
if (!$status) {
throw new Exception('Failed to delete Audit logs for project' . $projectId);
}
});
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$audit = new Audit($dbForProject);
$status = $audit->cleanup($auditRetention);
if (!$status) {
throw new Exception('Failed to delete Audit logs for project' . $projectId);
}
}
/**
@@ -869,39 +855,6 @@ class Deletes extends Action
}
}
/**
* @param Database $dbForConsole
* @param callable $callback
* @throws Exception
*/
private function deleteForProjectIds(database $dbForConsole, callable $callback): void
{
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
$count = 0;
$chunk = 0;
$limit = 50;
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
$chunk++;
/** @var string[] $projectIds */
$sum = count($projects);
Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects');
foreach ($projects as $project) {
$callback($project);
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
}
/**
* @param string $collection collectionID
* @param array $queries
src/Appwrite/Platform/Workers/Functions.php
+10 -10
View File
@@ -421,6 +421,16 @@ class Functions extends Action
$error = $th->getMessage();
$errorCode = $th->getCode();
} finally {
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->trigger()
;
}
if ($function->getAttribute('logging')) {
@@ -472,15 +482,5 @@ class Functions extends Action
if (!empty($error)) {
throw new Exception($error, $errorCode);
}
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->trigger()
;
}
}
src/Appwrite/Platform/Workers/Messaging.php
+1 -1
View File
@@ -6,7 +6,7 @@ use Exception;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\DSN\DSN;
use Utopia\Messaging\Messages\Sms;
use Utopia\Messaging\Messages\SMS;
use Utopia\Messaging\Adapters\SMS\Mock;
use Utopia\Messaging\Adapters\SMS\Msg91;
use Utopia\Messaging\Adapters\SMS\Telesign;
src/Appwrite/Platform/Workers/Usage.php
+12 -12
View File
@@ -121,8 +121,8 @@ class Usage extends Action
}
break;
case $document->getCollection() === 'databases': // databases
$collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
$collections = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
if (!empty($collections['value'])) {
$metrics[] = [
'key' => METRIC_COLLECTIONS,
@@ -140,7 +140,7 @@ class Usage extends Action
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
$documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
if (!empty($documents['value'])) {
$metrics[] = [
@@ -155,8 +155,8 @@ class Usage extends Action
break;
case $document->getCollection() === 'buckets':
$files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
$files = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
if (!empty($files['value'])) {
$metrics[] = [
@@ -174,13 +174,13 @@ class Usage extends Action
break;
case $document->getCollection() === 'functions':
$deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
$deployments = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
if (!empty($deployments['value'])) {
$metrics[] = [
src/Appwrite/Platform/Workers/UsageHook.php
+3 -3
View File
@@ -67,7 +67,7 @@ class UsageHook extends Usage
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats', new Document([
$dbForProject->createDocument('stats_v2', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
@@ -78,14 +78,14 @@ class UsageHook extends Usage
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
'stats_v2',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats',
'stats_v2',
$id,
'value',
$value