Merge branch '1.7.x' into feat-sites

This commit is contained in:
Matej Bačo committed 2025-02-19 12:54:05 +01:00
commit 3932b8bdc1
72 files changed
+3033 -855

No files matched your search

+158
View File
@@ -0,0 +1,158 @@
<?php
namespace Appwrite\Auth;
use Ahc\Jwt\JWT;
use Ahc\Jwt\JWTException;
use Appwrite\Extend\Exception;
use Utopia\Config\Config;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\System\System;
class Key
{
public function __construct(
protected string $projectId,
protected string $type,
protected string $role,
protected array $scopes,
protected string $name,
protected bool $expired = false,
protected array $disabledMetrics = [],
) {
}
public function getProjectId(): string
{
return $this->projectId;
}
public function getType(): string
{
return $this->type;
}
public function getRole(): string
{
return $this->role;
}
public function getScopes(): array
{
return $this->scopes;
}
public function getName(): string
{
return $this->name;
}
public function isExpired(): bool
{
return $this->expired;
}
public function getDisabledMetrics(): array
{
return $this->disabledMetrics;
}
/**
* Decode the given secret key into a Key object, containing the project ID, type, role, scopes, and name.
* Can be a stored API key or a dynamic key (JWT).
*
* @param Document $project
* @param string $key
* @return Key
* @throws Exception
*/
public static function decode(
Document $project,
string $key
): Key {
if (\str_contains($key, '_')) {
[$type, $secret] = \explode('_', $key, 2);
} else {
$type = API_KEY_STANDARD;
$secret = $key;
}
$role = Auth::USER_ROLE_APPS;
$roles = Config::getParam('roles', []);
$scopes = $roles[Auth::USER_ROLE_APPS]['scopes'] ?? [];
$expired = false;
$guestKey = new Key(
$project->getId(),
$type,
Auth::USER_ROLE_GUESTS,
$roles[Auth::USER_ROLE_GUESTS]['scopes'] ?? [],
'UNKNOWN'
);
switch ($type) {
case API_KEY_DYNAMIC:
$jwtObj = new JWT(
key: System::getEnv('_APP_OPENSSL_KEY_V1'),
algo: 'HS256',
maxAge: 86400,
leeway: 0
);
try {
$payload = $jwtObj->decode($secret);
} catch (JWTException) {
$expired = true;
}
$name = $payload['name'] ?? 'Dynamic Key';
$projectId = $payload['projectId'] ?? '';
$disabledMetrics = $payload['disabledMetrics'] ?? [];
$scopes = \array_merge($payload['scopes'] ?? [], $scopes);
if ($projectId !== $project->getId()) {
return $guestKey;
}
return new Key(
$projectId,
$type,
$role,
$scopes,
$name,
$expired,
$disabledMetrics
);
case API_KEY_STANDARD:
$key = $project->find(
key: 'secret',
find: $key,
subject: 'keys'
);
if (!$key) {
return $guestKey;
}
$expire = $key->getAttribute('expire');
if (!empty($expire) && $expire < DateTime::formatTz(DateTime::now())) {
$expired = true;
}
$name = $key->getAttribute('name', 'UNKNOWN');
$scopes = \array_merge($key->getAttribute('scopes', []), $scopes);
return new Key(
$project->getId(),
$type,
$role,
$scopes,
$name,
$expired
);
default:
return $guestKey;
}
}
}
+11
View File
@@ -27,11 +27,22 @@ class Event
public const SITES_QUEUE_NAME = 'v1-sites';
public const SITES_CLASS_NAME = 'SitesV1';
/** remove */
public const USAGE_QUEUE_NAME = 'v1-usage';
public const USAGE_CLASS_NAME = 'UsageV1';
public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump';
public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1';
/** /remove */
public const STATS_RESOURCES_QUEUE_NAME = 'v1-stats-resources';
public const STATS_RESOURCES_CLASS_NAME = 'StatsResourcesV1';
public const STATS_USAGE_QUEUE_NAME = 'v1-stats-usage';
public const STATS_USAGE_CLASS_NAME = 'StatsUsageV1';
public const STATS_USAGE_DUMP_QUEUE_NAME = 'v1-stats-usage-dump';
public const STATS_USAGE_DUMP_CLASS_NAME = 'StatsUsageDumpV1';
public const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
public const WEBHOOK_CLASS_NAME = 'WebhooksV1';
+29
View File
@@ -0,0 +1,29 @@
<?php
namespace Appwrite\Event;
use Utopia\Queue\Publisher;
class StatsResources extends Event
{
public function __construct(protected Publisher $publisher)
{
parent::__construct($publisher);
$this
->setQueue(Event::STATS_RESOURCES_QUEUE_NAME)
->setClass(Event::STATS_RESOURCES_CLASS_NAME);
}
/**
* Prepare the payload for the usage event.
*
* @return array
*/
protected function preparePayload(): array
{
return [
'project' => $this->project
];
}
}
+86
View File
@@ -0,0 +1,86 @@
<?php
namespace Appwrite\Event;
use Utopia\Database\Document;
use Utopia\Queue\Publisher;
class StatsUsage extends Event
{
protected array $metrics = [];
protected array $reduce = [];
protected array $disabled = [];
public function __construct(protected Publisher $publisher)
{
parent::__construct($publisher);
$this
->setQueue(Event::STATS_USAGE_QUEUE_NAME)
->setClass(Event::STATS_USAGE_CLASS_NAME);
}
/**
* Add reduce.
*
* @param Document $document
* @return self
*/
public function addReduce(Document $document): self
{
$this->reduce[] = $document;
return $this;
}
/**
* Add metric.
*
* @param string $key
* @param int $value
* @return self
*/
public function addMetric(string $key, int $value): self
{
$this->metrics[] = [
'key' => $key,
'value' => $value,
];
return $this;
}
/**
* Set disabled metrics.
*
* @param string $key
* @return self
*/
public function disableMetric(string $key): self
{
$this->disabled[] = $key;
return $this;
}
/**
* Prepare the payload for the event
*
* @return array
*/
protected function preparePayload(): array
{
return [
'project' => $this->getProject(),
'reduce' => $this->reduce,
'metrics' => \array_filter($this->metrics, function ($metric) {
foreach ($this->disabled as $disabledMetric) {
if (\str_ends_with($metric['key'], $disabledMetric)) {
return false;
}
}
return true;
}),
];
}
}
+44
View File
@@ -0,0 +1,44 @@
<?php
namespace Appwrite\Event;
use Utopia\Queue\Publisher;
class StatsUsageDump extends Event
{
protected array $stats;
public function __construct(protected Publisher $publisher)
{
parent::__construct($publisher);
$this
->setQueue(Event::STATS_USAGE_DUMP_QUEUE_NAME)
->setClass(Event::STATS_USAGE_DUMP_CLASS_NAME);
}
/**
* Add Stats.
*
* @param array $stats
* @return self
*/
public function setStats(array $stats): self
{
$this->stats = $stats;
return $this;
}
/**
* Prepare the payload for the usage dump event.
*
* @return array
*/
protected function preparePayload(): array
{
return [
'stats' => $this->stats,
];
}
}
+1
View File
@@ -92,6 +92,7 @@ abstract class Migration
'1.5.11' => 'V20',
'1.6.0' => 'V21',
'1.6.1' => 'V21',
'1.6.2' => 'V22',
];
/**
+83
View File
@@ -0,0 +1,83 @@
<?php
namespace Appwrite\Migration\Version;
use Appwrite\Migration\Migration;
use Exception;
use Throwable;
use Utopia\CLI\Console;
use Utopia\Database\Database;
class V22 extends Migration
{
/**
* @throws Throwable
*/
public function execute(): void
{
/**
* Disable SubQueries for Performance.
*/
foreach (['subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships', 'subQueryVariables', 'subQueryChallenges', 'subQueryProjectVariables', 'subQueryTargets', 'subQueryTopicTargets'] as $name) {
Database::addFilter(
$name,
fn () => null,
fn () => []
);
}
Console::info('Migrating Collections');
$this->migrateCollections();
}
/**
* Migrate Collections.
*
* @return void
* @throws Exception|Throwable
*/
private function migrateCollections(): void
{
$internalProjectId = $this->project->getInternalId();
$collectionType = match ($internalProjectId) {
'console' => 'console',
default => 'projects',
};
$collections = $this->collections[$collectionType];
foreach ($collections as $collection) {
$id = $collection['$id'];
Console::log("Migrating Collection \"{$id}\"");
$this->projectDB->setNamespace("_$internalProjectId");
switch ($id) {
case 'installations':
// Create personalAccessToken attribute
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'personalAccessToken');
} catch (Throwable $th) {
Console::warning("'personalAccessToken' from {$id}: {$th->getMessage()}");
}
// Create personalAccessTokenExpiry attribute
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'personalAccessTokenExpiry');
} catch (Throwable $th) {
Console::warning("'personalAccessTokenExpiry' from {$id}: {$th->getMessage()}");
}
// Create personalRefreshToken attribute
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'personalRefreshToken');
} catch (Throwable $th) {
Console::warning("'personalRefreshToken' from {$id}: {$th->getMessage()}");
}
break;
}
usleep(50000);
}
}
}
+90
View File
@@ -0,0 +1,90 @@
<?php
namespace Appwrite\Platform;
use Swoole\Coroutine as Co;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Platform\Action as UtopiaAction;
class Action extends UtopiaAction
{
/**
* Log Error Callback
*
* @var callable
*/
protected mixed $logError;
/**
* Foreach Document
* Call provided callback for each document in the collection
*
* @param string $projectId
* @param string $collection
* @param array $queries
* @param callable $callback
*
* @return void
*/
protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null, int $limit = 1000, bool $concurrent = false): void
{
$results = [];
$sum = $limit;
$latestDocument = null;
while ($sum === $limit) {
$newQueries = $queries;
try {
if ($latestDocument !== null) {
array_unshift($newQueries, Query::cursorAfter($latestDocument));
}
$newQueries[] = Query::limit($limit);
$database->disableValidation();
$results = $database->find($collection, $newQueries);
$database->enableValidation();
} catch (\Exception $e) {
if (!empty($this->logError)) {
call_user_func_array($this->logError, [$e, "CLI", "fetch_documents_namespace_{$database->getNamespace()}_collection{$collection}"]);
}
}
if (empty($results)) {
return;
}
$sum = count($results);
if ($concurrent) {
$callables = [];
$errors = [];
foreach ($results as $document) {
if (is_callable($callback)) {
$callables[] = Co\go(function () use ($document, $callback, &$errors) {
try {
$callback($document);
} catch (\Throwable $error) {
$errors[] = $error;
}
});
}
}
Co::join($callables);
if (!empty($errors)) {
throw new \Error("Errors found in concurrent foreachDocument: " . \json_encode($errors));
}
} else {
foreach ($results as $document) {
if (is_callable($callback)) {
$callback($document);
}
}
}
$latestDocument = $results[array_key_last($results)];
}
}
}
@@ -5,7 +5,7 @@ namespace Appwrite\Platform\Modules\Functions\Workers;
use Ahc\Jwt\JWT;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Utopia\Response\Model\Deployment;
use Appwrite\Vcs\Comment;
@@ -51,12 +51,14 @@ class Builds extends Action
->inject('dbForPlatform')
->inject('queueForEvents')
->inject('queueForFunctions')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('cache')
->inject('dbForProject')
->inject('deviceForFunctions')
->inject('isResourceBlocked')
->inject('log')
->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log));
->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) =>
$this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log));
}
/**
@@ -65,7 +67,7 @@ class Builds extends Action
* @param Database $dbForPlatform
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param Usage $queueForUsage
* @param StatsUsage $queueForStatsUsage
* @param Cache $cache
* @param Database $dbForProject
* @param Device $deviceForFunctions
@@ -73,7 +75,7 @@ class Builds extends Action
* @return void
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void
public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void
{
$payload = $message->getPayload() ?? [];
@@ -94,7 +96,7 @@ class Builds extends Action
case BUILD_TYPE_RETRY:
Console::info('Creating build for deployment: ' . $deployment->getId());
$github = new GitHub($cache);
$this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log);
$this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log);
break;
default:
@@ -106,7 +108,7 @@ class Builds extends Action
* @param Device $deviceForFunctions
* @param Func $queueForFunctions
* @param Event $queueForEvents
* @param Usage $queueForUsage
* @param StatsUsage $queueForStatsUsage
* @param Database $dbForPlatform
* @param Database $dbForProject
* @param GitHub $github
@@ -120,7 +122,7 @@ class Builds extends Action
*
* @throws Exception
*/
protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void
protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void
{
$resourceKey = match($resource->getCollection()) {
'functions' => 'functionId',
@@ -134,24 +136,33 @@ class Builds extends Action
$resource = $dbForProject->getDocument($resource->getCollection(), $resource->getId());
if ($resource->isEmpty()) {
throw new \Exception('Function not found', 404);
throw new \Exception('Resource not found');
}
// TODO: Sites support
if ($isResourceBlocked($project, RESOURCE_TYPE_FUNCTIONS, $resource->getId())) {
throw new \Exception('Resource is blocked');
}
$log->addTag('deploymentId', $deployment->getId());
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
if ($deployment->isEmpty()) {
throw new \Exception('Deployment not found', 404);
throw new \Exception('Deployment not found');
}
// todo: figure out a better way, entrypoint is not required for sites
if ($resource->getCollection() === 'functions' && empty($deployment->getAttribute('entrypoint', ''))) {
throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500);
throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".');
}
$version = $this->getVersion($resource);
$runtime = $this->getRuntime($resource, $version);
$spec = Config::getParam('runtime-specifications')[$resource->getAttribute('specifications', APP_COMPUTE_SPECIFICATION_DEFAULT)];
$spec = Config::getParam('runtime-specifications')[$resource->getAttribute('specification', APP_COMPUTE_SPECIFICATION_DEFAULT)];
if (\is_null($runtime)) {
throw new \Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
}
// Realtime preparation
$allEvents = Event::generateEvents("{$resource->getCollection()}.[{$resourceKey}].deployments.[deploymentId].update", [
@@ -604,7 +615,7 @@ class Builds extends Action
$build = $dbForProject->getDocument('builds', $build->getId());
if ($build->isEmpty()) {
throw new \Exception('Build not found', 404);
throw new \Exception('Build not found');
}
if ($build->getAttribute('status') === 'canceled') {
@@ -813,8 +824,8 @@ class Builds extends Action
$queue
->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project
->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsSuccess']), 1) // per function
->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeSuccess']), (int)$build->getAttribute('duration', 0) * 1000);
->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function
->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000);
break;
case 'failed':
$queue
@@ -825,15 +836,15 @@ class Builds extends Action
break;
}
$queue
$queueForStatsUsage
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT)))
->addMetric(str_replace($key, $resource->getInternalId(), $metrics['builds']), 1) // per function
->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsStorage']), $build->getAttribute('size', 0))
->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsCompute']), (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsMbSeconds']), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT)))
->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT)))
->setProject($project)
->trigger();
}
+2
View File
@@ -13,6 +13,7 @@ use Appwrite\Platform\Tasks\ScheduleMessages;
use Appwrite\Platform\Tasks\SDKs;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\SSL;
use Appwrite\Platform\Tasks\StatsResources;
use Appwrite\Platform\Tasks\Upgrade;
use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
@@ -38,6 +39,7 @@ class Tasks extends Service
->addAction(Upgrade::getName(), new Upgrade())
->addAction(Vars::getName(), new Vars())
->addAction(Version::getName(), new Version())
->addAction(StatsResources::getName(), new StatsResources())
;
}
}
+11 -2
View File
@@ -10,8 +10,13 @@ use Appwrite\Platform\Workers\Functions;
use Appwrite\Platform\Workers\Mails;
use Appwrite\Platform\Workers\Messaging;
use Appwrite\Platform\Workers\Migrations;
use Appwrite\Platform\Workers\StatsResources;
use Appwrite\Platform\Workers\StatsUsage;
use Appwrite\Platform\Workers\StatsUsageDump;
/** remove */
use Appwrite\Platform\Workers\Usage;
use Appwrite\Platform\Workers\UsageDump;
/** /remove */
use Appwrite\Platform\Workers\Webhooks;
use Utopia\Platform\Service;
@@ -29,10 +34,14 @@ class Workers extends Service
->addAction(Mails::getName(), new Mails())
->addAction(Messaging::getName(), new Messaging())
->addAction(Webhooks::getName(), new Webhooks())
->addAction(StatsUsageDump::getName(), new StatsUsageDump())
->addAction(StatsUsage::getName(), new StatsUsage())
->addAction(Migrations::getName(), new Migrations())
->addAction(StatsResources::getName(), new StatsResources())
/** Remove */
->addAction(UsageDump::getName(), new UsageDump())
->addAction(Usage::getName(), new Usage())
->addAction(Migrations::getName(), new Migrations())
/** /remove */
;
}
}
@@ -0,0 +1,81 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Event\StatsResources as EventStatsResources;
use Appwrite\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\System\System;
/**
* Usage count
*
* Runs every hour, schedules project
* for aggregating resource count
*/
class StatsResources extends Action
{
/**
* Log Error Callback
*
* @var callable
*/
protected mixed $logError;
/**
* Console DB
*
* @var Database
*/
protected Database $dbForPlatform;
public static function getName()
{
return 'stats-resources';
}
public function __construct()
{
$this
->desc('Schedules projects for usage count')
->inject('dbForPlatform')
->inject('logError')
->inject('queueForStatsResources')
->callback([$this, 'action']);
}
public function action(Database $dbForPlatform, callable $logError, EventStatsResources $queue): void
{
$this->logError = $logError;
$this->dbForPlatform = $dbForPlatform;
Console::title("Stats resources V1");
Console::success('Stats resources: started');
$interval = (int) System::getEnv('_APP_STATS_RESOURCES_INTERVAL', '3600');
Console::loop(function () use ($queue) {
Authorization::disable();
Authorization::setDefaultStatus(false);
$last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours'));
/**
* For each project that were accessed in last 24 hours
*/
$this->foreachDocument($this->dbForPlatform, 'projects', [
Query::greaterThanEqual('accessedAt', DateTime::format($last24Hours))
], function ($project) use ($queue) {
$queue
->setProject($project)
->trigger();
Console::success('project: ' . $project->getId() . '(' . $project->getInternalId() . ')' . ' queued');
});
}, $interval);
Console::log("Stats resources: exited");
}
}
+59 -13
View File
@@ -6,15 +6,33 @@ use Appwrite\Auth\Auth;
use Exception;
use Throwable;
use Utopia\Audit\Audit;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Authorization;
use Utopia\Database\Exception\Structure;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\System\System;
class Audits extends Action
{
private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development
private const BATCH_SIZE_PRODUCTION = 5_000;
private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds
private int $lastTriggeredTime = 0;
private array $logs = [];
private function getBatchSize(): int
{
return System::getEnv('_APP_ENV', 'development') === 'development'
? self::BATCH_SIZE_DEVELOPMENT
: self::BATCH_SIZE_PRODUCTION;
}
public static function getName(): string
{
return 'audits';
@@ -30,6 +48,8 @@ class Audits extends Action
->inject('message')
->inject('dbForProject')
->callback(fn ($message, $dbForProject) => $this->action($message, $dbForProject));
$this->lastTriggeredTime = time();
}
@@ -44,13 +64,14 @@ class Audits extends Action
*/
public function action(Message $message, Database $dbForProject): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
Console::info('Aggregating audit logs');
$event = $payload['event'] ?? '';
$auditPayload = $payload['payload'] ?? '';
$mode = $payload['mode'] ?? '';
@@ -63,23 +84,48 @@ class Audits extends Action
$userEmail = $user->getAttribute('email', '');
$userType = $user->getAttribute('type', Auth::ACTIVITY_TYPE_USER);
$audit = new Audit($dbForProject);
$audit->log(
userId: $user->getInternalId(),
// Pass first, most verbose event pattern
event: $event,
resource: $resource,
userAgent: $userAgent,
ip: $ip,
location: '',
data: [
// Create event data
$eventData = [
'userId' => $user->getInternalId(),
'event' => $event,
'resource' => $resource,
'userAgent' => $userAgent,
'ip' => $ip,
'location' => '',
'data' => [
'userId' => $user->getId(),
'userName' => $userName,
'userEmail' => $userEmail,
'userType' => $userType,
'mode' => $mode,
'data' => $auditPayload,
]
);
],
'timestamp' => DateTime::formatTz(DateTime::now())
];
$this->logs[] = $eventData;
// Check if we should process the batch by checking both for the batch size and the elapsed time
$batchSize = $this->getBatchSize();
$shouldProcessBatch = count($this->logs) >= $batchSize;
if (!$shouldProcessBatch && count($this->logs) > 0) {
$shouldProcessBatch = (time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL;
}
if ($shouldProcessBatch) {
Console::log('Processing batch with ' . count($this->logs) . ' events');
$audit = new Audit($dbForProject);
try {
$audit->logBatch($this->logs);
Console::success('Audit logs processed successfully');
} catch (Throwable $e) {
Console::error('Error processing audit logs: ' . $e->getMessage());
} finally {
// Clear the pending events after successful batch processing
$this->logs = [];
$this->lastTriggeredTime = time();
}
}
}
}
+13 -27
View File
@@ -577,39 +577,25 @@ class Databases extends Action
*/
protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
{
$count = 0;
$chunk = 0;
$limit = 50;
$sum = $limit;
$start = \microtime(true);
$executionStart = \microtime(true);
try {
$documents = $database->deleteDocuments($collection, $queries);
} catch (\Throwable $th) {
Console::error('Failed to delete documents for collection ' . $collection . ': ' . $th->getMessage());
return;
}
while ($sum === $limit) {
$chunk++;
$results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
$sum = count($results);
Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents');
foreach ($results as $document) {
if ($database->deleteDocument($document->getCollection(), $document->getId())) {
Console::success('Deleted document "' . $document->getId() . '" successfully');
if (\is_callable($callback)) {
$callback($document);
}
} else {
Console::warning('Failed to delete document: ' . $document->getId());
}
$count++;
if (\is_callable($callback)) {
foreach ($documents as $document) {
$callback($document);
}
}
$executionEnd = \microtime(true);
$end = \microtime(true);
$count = \count($documents);
Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
Console::info("Deleted {$count} documents by group in " . ($end - $start) . " seconds");
}
protected function trigger(
+33 -59
View File
@@ -7,6 +7,7 @@ use Appwrite\Certificates\Adapter as CertificatesAdapter;
use Appwrite\Extend\Exception;
use Executor\Executor;
use Throwable;
use Utopia\Abuse\Adapters\TimeLimit\Database as AbuseDatabase;
use Utopia\Audit\Audit;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
@@ -510,7 +511,8 @@ class Deletes extends Action
$projectCollectionIds = [
...\array_keys(Config::getParam('collections', [])['projects']),
Audit::COLLECTION
Audit::COLLECTION,
AbuseDatabase::COLLECTION,
];
$limit = \count($projectCollectionIds) + 25;
@@ -1039,65 +1041,39 @@ class Deletes extends Action
$this->deleteRuntimes($getProjectDB, $document, $project);
}
/**
* @param Document $document to be deleted
* @param Database $database to delete it from
* @param callable|null $callback to perform after document is deleted
* @return void
*/
private function deleteById(Document $document, Database $database, callable $callback = null): void
{
if ($database->deleteDocument($document->getCollection(), $document->getId())) {
Console::success('Deleted document "' . $document->getId() . '" successfully');
if (is_callable($callback)) {
$callback($document);
}
} else {
Console::error('Failed to delete document: ' . $document->getId());
}
}
/**
* @param string $collection collectionID
* @param array $queries
* @param Database $database
* @param callable|null $callback
* @param ?callable $callback
* @return void
* @throws Exception
*/
protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
{
$count = 0;
$chunk = 0;
$limit = 50;
$sum = $limit;
protected function deleteByGroup(
string $collection,
array $queries,
Database $database,
?callable $callback = null
): void {
$start = \microtime(true);
$executionStart = \microtime(true);
try {
$documents = $database->deleteDocuments($collection, $queries);
} catch (\Throwable $th) {
Console::error('Failed to delete documents for collection ' . $collection . ': ' . $th->getMessage());
return;
}
while ($sum === $limit) {
$chunk++;
try {
$results = $database->find($collection, [Query::limit($limit), ...$queries]);
} catch (DatabaseException $e) {
Console::error('Failed to find documents for collection ' . $collection . ': ' . $e->getMessage());
return;
}
$sum = count($results);
Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents');
foreach ($results as $document) {
$this->deleteById($document, $database, $callback);
$count++;
if (\is_callable($callback)) {
foreach ($documents as $document) {
$callback($document);
}
}
$executionEnd = \microtime(true);
$end = \microtime(true);
$count = \count($documents);
Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
Console::info("Deleted {$count} documents by group in " . ($end - $start) . " seconds");
}
/**
@@ -1111,25 +1087,23 @@ class Deletes extends Action
protected function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
{
$count = 0;
$chunk = 0;
$limit = 50;
$results = [];
$limit = 1000;
$sum = $limit;
$cursor = null;
$executionStart = \microtime(true);
$start = \microtime(true);
while ($sum === $limit) {
$chunk++;
$mergedQueries = \array_merge([Query::limit($limit)], $queries);
if ($cursor instanceof Document) {
$mergedQueries[] = Query::cursorAfter($cursor);
$queries = \array_merge([Query::limit($limit)], $queries);
if ($cursor !== null) {
$queries[] = Query::cursorAfter($cursor);
}
$results = $database->find($collection, $mergedQueries);
$results = $database->find($collection, $queries);
$sum = count($results);
$sum = \count($results);
if ($sum > 0) {
$cursor = $results[$sum - 1];
@@ -1144,9 +1118,9 @@ class Deletes extends Action
}
}
$executionEnd = \microtime(true);
$end = \microtime(true);
Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
Console::info("Listed {$count} documents by group in " . ($end - $start) . " seconds");
}
/**
+10 -10
View File
@@ -5,7 +5,7 @@ namespace Appwrite\Platform\Workers;
use Ahc\Jwt\JWT;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Utopia\Response\Model\Execution;
use Exception;
@@ -46,13 +46,13 @@ class Functions extends Action
->inject('dbForProject')
->inject('queueForFunctions')
->inject('queueForEvents')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('log')
->inject('isResourceBlocked')
->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForUsage, $log, $isResourceBlocked));
->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked));
}
public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log, callable $isResourceBlocked): void
public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void
{
$payload = $message->getPayload() ?? [];
@@ -137,7 +137,7 @@ class Functions extends Action
log: $log,
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
queueForUsage: $queueForUsage,
queueForStatsUsage: $queueForStatsUsage,
queueForEvents: $queueForEvents,
project: $project,
function: $function,
@@ -177,7 +177,7 @@ class Functions extends Action
log: $log,
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
queueForUsage: $queueForUsage,
queueForStatsUsage: $queueForStatsUsage,
queueForEvents: $queueForEvents,
project: $project,
function: $function,
@@ -199,7 +199,7 @@ class Functions extends Action
log: $log,
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
queueForUsage: $queueForUsage,
queueForStatsUsage: $queueForStatsUsage,
queueForEvents: $queueForEvents,
project: $project,
function: $function,
@@ -285,7 +285,7 @@ class Functions extends Action
* @param Log $log
* @param Database $dbForProject
* @param Func $queueForFunctions
* @param Usage $queueForUsage
* @param StatsUsage $queueForStatsUsage
* @param Event $queueForEvents
* @param Document $project
* @param Document $function
@@ -309,7 +309,7 @@ class Functions extends Action
Log $log,
Database $dbForProject,
Func $queueForFunctions,
Usage $queueForUsage,
StatsUsage $queueForStatsUsage,
Event $queueForEvents,
Document $project,
Document $function,
@@ -554,7 +554,7 @@ class Functions extends Action
$errorCode = $th->getCode();
} finally {
/** Trigger usage queue */
$queueForUsage
$queueForStatsUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
+12 -12
View File
@@ -2,7 +2,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Messaging\Status as MessageStatus;
use Swoole\Runtime;
use Utopia\CLI\Console;
@@ -70,8 +70,8 @@ class Messaging extends Action
->inject('log')
->inject('dbForProject')
->inject('deviceForFiles')
->inject('queueForUsage')
->callback(fn (Message $message, Document $project, Log $log, Database $dbForProject, Device $deviceForFiles, Usage $queueForUsage) => $this->action($message, $project, $log, $dbForProject, $deviceForFiles, $queueForUsage));
->inject('queueForStatsUsage')
->callback(fn (Message $message, Document $project, Log $log, Database $dbForProject, Device $deviceForFiles, StatsUsage $queueForStatsUsage) => $this->action($message, $project, $log, $dbForProject, $deviceForFiles, $queueForStatsUsage));
}
/**
@@ -80,7 +80,7 @@ class Messaging extends Action
* @param Log $log
* @param Database $dbForProject
* @param Device $deviceForFiles
* @param Usage $queueForUsage
* @param StatsUsage $queueForStatsUsage
* @return void
* @throws \Exception
*/
@@ -90,7 +90,7 @@ class Messaging extends Action
Log $log,
Database $dbForProject,
Device $deviceForFiles,
Usage $queueForUsage
StatsUsage $queueForStatsUsage
): void {
Runtime::setHookFlags(SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_TCP);
$payload = $message->getPayload() ?? [];
@@ -111,7 +111,7 @@ class Messaging extends Action
case MESSAGE_SEND_TYPE_EXTERNAL:
$message = $dbForProject->getDocument('messages', $payload['messageId']);
$this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $queueForUsage);
$this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $queueForStatsUsage);
break;
default:
throw new \Exception('Unknown message type: ' . $type);
@@ -123,7 +123,7 @@ class Messaging extends Action
Document $message,
Device $deviceForFiles,
Document $project,
Usage $queueForUsage
StatsUsage $queueForStatsUsage
): void {
$topicIds = $message->getAttribute('topics', []);
$targetIds = $message->getAttribute('targets', []);
@@ -229,8 +229,8 @@ class Messaging extends Action
/**
* @var array<array> $results
*/
$results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
$results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) {
return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) {
if (\array_key_exists($providerId, $providers)) {
$provider = $providers[$providerId];
} else {
@@ -257,8 +257,8 @@ class Messaging extends Action
$adapter->getMaxMessagesPerRequest()
);
return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) {
return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) {
$deliveredTotal = 0;
$deliveryErrors = [];
$messageData = clone $message;
@@ -298,7 +298,7 @@ class Messaging extends Action
$deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage();
} finally {
$errorTotal = \count($deliveryErrors);
$queueForUsage
$queueForStatsUsage
->setProject($project)
->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal))
->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal)
+9 -19
View File
@@ -188,28 +188,21 @@ class Migrations extends Action
}
/**
* @throws \Utopia\Database\Exception
* @throws Authorization
* @throws Conflict
* @throws Restricted
* @throws Structure
*/
protected function removeAPIKey(Document $apiKey): void
{
$this->dbForPlatform->deleteDocument('keys', $apiKey->getId());
}
/**
* @throws Authorization
* @throws Structure
* @throws \Utopia\Database\Exception
* @throws Exception
*/
protected function generateAPIKey(Document $project): string
{
$jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 86400, 0);
$apiKey = $jwt->encode([
'projectId' => $project->getId(),
'disabledMetrics' => [
METRIC_DATABASES_OPERATIONS_READS,
METRIC_DATABASES_OPERATIONS_WRITES,
METRIC_NETWORK_REQUESTS,
METRIC_NETWORK_INBOUND,
METRIC_NETWORK_OUTBOUND,
],
'scopes' => [
'users.read',
'users.write',
@@ -222,12 +215,9 @@ class Migrations extends Action
'functions.read',
'functions.write',
'databases.read',
'databases.write',
'collections.read',
'collections.write',
'documents.read',
'documents.write'
]
],
]);
return API_KEY_DYNAMIC . '_' . $apiKey;
@@ -0,0 +1,340 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Platform\Action;
use Exception;
use Throwable;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Queue\Message;
class StatsResources extends Action
{
/**
* Date format for different periods
*/
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
'inf' => '0000-00-00 00:00'
];
/**
* @var array $documents
*
* Array of documents to batch write
*
*/
protected array $documents = [];
public static function getName(): string
{
return 'stats-resources';
}
/**
* @throws Exception
*/
public function __construct()
{
$this
->desc('Stats resources worker')
->inject('message')
->inject('project')
->inject('getProjectDB')
->inject('getLogsDB')
->inject('dbForPlatform')
->inject('logError')
->callback([$this, 'action']);
}
/**
* @param Message $message
* @param Document $project
* @param callable $getProjectDB
* @return void
* @throws \Utopia\Database\Exception
* @throws Exception
*/
public function action(Message $message, Document $project, callable $getProjectDB, callable $getLogsDB, Database $dbForPlatform, callable $logError): void
{
$this->logError = $logError;
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
if (empty($project->getAttribute('database'))) {
var_dump($payload);
return;
}
// Reset documents for each job
$this->documents = [];
$this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project);
}
protected function countForProject(Database $dbForPlatform, callable $getLogsDB, callable $getProjectDB, Document $project): void
{
Console::info('Begining count for: ' . $project->getId());
$dbForLogs = null;
$dbForProject = null;
try {
/** @var \Utopia\Database\Database $dbForLogs */
$dbForLogs = call_user_func($getLogsDB, $project);
/** @var \Utopia\Database\Database $dbForProject */
$dbForProject = call_user_func($getProjectDB, $project);
} catch (Throwable $th) {
Console::error('Unable to get database');
Console::error($th->getMessage());
return;
}
try {
$region = $project->getAttribute('region');
$platforms = $dbForPlatform->count('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()])
]);
$webhooks = $dbForPlatform->count('webhooks', [
Query::equal('projectInternalId', [$project->getInternalId()])
]);
$keys = $dbForPlatform->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()])
]);
$databases = $dbForProject->count('databases');
$buckets = $dbForProject->count('buckets');
$users = $dbForProject->count('users');
$last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00');
$usersMAU = $dbForProject->count('users', [
Query::greaterThanEqual('accessedAt', $last30Days)
]);
$last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours'))->format('Y-m-d h:m:00');
$usersDAU = $dbForProject->count('users', [
Query::greaterThanEqual('accessedAt', $last24Hours)
]);
$last7Days = (new \DateTime())->sub(\DateInterval::createFromDateString('7 days'))->format('Y-m-d 00:00:00');
$usersWAU = $dbForProject->count('users', [
Query::greaterThanEqual('accessedAt', $last7Days)
]);
$teams = $dbForProject->count('teams');
$functions = $dbForProject->count('functions');
$messages = $dbForProject->count('messages');
$providers = $dbForProject->count('providers');
$topics = $dbForProject->count('topics');
$metrics = [
METRIC_DATABASES => $databases,
METRIC_BUCKETS => $buckets,
METRIC_USERS => $users,
METRIC_FUNCTIONS => $functions,
METRIC_TEAMS => $teams,
METRIC_MESSAGES => $messages,
METRIC_MAU => $usersMAU,
METRIC_DAU => $usersDAU,
METRIC_WAU => $usersWAU,
METRIC_WEBHOOKS => $webhooks,
METRIC_PLATFORMS => $platforms,
METRIC_PROVIDERS => $providers,
METRIC_TOPICS => $topics,
METRIC_KEYS => $keys,
];
foreach ($metrics as $metric => $value) {
$this->createStatsDocuments($region, $metric, $value);
}
try {
$this->countForBuckets($dbForProject, $dbForLogs, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]);
}
try {
$this->countImageTransformations($dbForProject, $dbForLogs, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]);
}
try {
$this->countForDatabase($dbForProject, $dbForLogs, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]);
}
try {
$this->countForFunctions($dbForProject, $dbForLogs, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_functions_{$project->getId()}"]);
}
$this->writeDocuments($dbForLogs, $project);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]);
}
Console::info('End of count for: ' . $project->getId());
}
protected function countForBuckets(Database $dbForProject, Database $dbForLogs, string $region)
{
$totalFiles = 0;
$totalStorage = 0;
$this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $dbForLogs, $region, &$totalFiles, &$totalStorage) {
$files = $dbForProject->count('bucket_' . $bucket->getInternalId());
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES);
$this->createStatsDocuments($region, $metric, $files);
$storage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeActual');
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE);
$this->createStatsDocuments($region, $metric, $storage);
$totalStorage += $storage;
$totalFiles += $files;
});
$this->createStatsDocuments($region, METRIC_FILES, $totalFiles);
$this->createStatsDocuments($region, METRIC_FILES_STORAGE, $totalStorage);
}
/**
* Need separate function to count per period data
*/
protected function countImageTransformations(Database $dbForProject, Database $dbForLogs, string $region)
{
$totalImageTransformations = 0;
$last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00');
$this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $last30Days, $region, &$totalImageTransformations) {
$imageTransformations = $dbForProject->count('bucket_' . $bucket->getInternalId(), [
Query::greaterThanEqual('transformedAt', $last30Days),
]);
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED);
$this->createStatsDocuments($region, $metric, $imageTransformations);
$totalImageTransformations += $imageTransformations;
});
$this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalImageTransformations);
}
protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region)
{
$totalCollections = 0;
$totalDocuments = 0;
$this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $dbForLogs, $region, &$totalCollections, &$totalDocuments) {
$collections = $dbForProject->count('database_' . $database->getInternalId());
$metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS);
$this->createStatsDocuments($region, $metric, $collections);
$documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region);
$totalDocuments += $documents;
$totalCollections += $collections;
});
$this->createStatsDocuments($region, METRIC_COLLECTIONS, $totalCollections);
$this->createStatsDocuments($region, METRIC_DOCUMENTS, $totalDocuments);
}
protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int
{
$databaseDocuments = 0;
$this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $dbForLogs, $database, $region, &$totalCollections, &$databaseDocuments) {
$documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
$metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS);
$this->createStatsDocuments($region, $metric, $documents);
$databaseDocuments += $documents;
});
$metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS);
$this->createStatsDocuments($region, $metric, $databaseDocuments);
return $databaseDocuments;
}
protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region)
{
$deploymentsStorage = $dbForProject->sum('deployments', 'size');
$buildsStorage = $dbForProject->sum('builds', 'size');
$this->createStatsDocuments($region, METRIC_DEPLOYMENTS_STORAGE, $deploymentsStorage);
$this->createStatsDocuments($region, METRIC_BUILDS_STORAGE, $buildsStorage);
$deployments = $dbForProject->count('deployments');
$builds = $dbForProject->count('builds');
$this->createStatsDocuments($region, METRIC_DEPLOYMENTS, $deployments);
$this->createStatsDocuments($region, METRIC_BUILDS, $builds);
$this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForLogs, $region) {
$functionDeploymentsStorage = $dbForProject->sum('deployments', 'size', [
Query::equal('resourceInternalId', [$function->getInternalId()]),
Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]),
]);
$this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $functionDeploymentsStorage);
$functionDeployments = $dbForProject->count('deployments', [
Query::equal('resourceInternalId', [$function->getInternalId()]),
Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]),
]);
$this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $functionDeployments);
/**
* As deployments and builds have 1-1 relationship,
* the count for one should match the other
*/
$this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS), $functionDeployments);
$functionBuildsStorage = 0;
$this->foreachDocument($dbForProject, 'deployments', [
Query::equal('resourceInternalId', [$function->getInternalId()]),
Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]),
], function (Document $deployment) use ($dbForProject, &$functionBuildsStorage): void {
$build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''));
$functionBuildsStorage += $build->getAttribute('size', 0);
});
$this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionBuildsStorage);
});
}
protected function createStatsDocuments(string $region, string $metric, int $value)
{
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : \date($format, \time());
$id = \md5("{$time}_{$period}_{$metric}");
$this->documents[] = new Document([
'$id' => $id,
'metric' => $metric,
'period' => $period,
'region' => $region,
'value' => $value,
'time' => $time,
]);
}
}
protected function writeDocuments(Database $dbForLogs, Document $project): void
{
$dbForLogs->createOrUpdateDocuments(
'stats',
$this->documents
);
$this->documents = [];
Console::success('Stats written to logs db for project: ' . $project->getId() . '(' . $project->getInternalId() . ')');
}
}
@@ -0,0 +1,246 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Event\StatsUsageDump;
use Exception;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\System\System;
class StatsUsage extends Action
{
private array $stats = [];
private int $lastTriggeredTime = 0;
private int $keys = 0;
private const INFINITY_PERIOD = '_inf_';
private const KEYS_THRESHOLD = 10000;
public static function getName(): string
{
return 'stats-usage';
}
/**
* @throws Exception
*/
public function __construct()
{
$this
->desc('Stats usage worker')
->inject('message')
->inject('getProjectDB')
->inject('queueForStatsUsageDump')
->callback([$this, 'action']);
$this->lastTriggeredTime = time();
}
/**
* @param Message $message
* @param callable $getProjectDB
* @param StatsUsageDump $queueForStatsUsageDump
* @return void
* @throws \Utopia\Database\Exception
* @throws Exception
*/
public function action(Message $message, callable $getProjectDB, StatsUsageDump $queueForStatsUsageDump): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
//Todo Figure out way to preserve keys when the container is being recreated @shimonewman
$aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
$project = new Document($payload['project'] ?? []);
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
if (empty($document)) {
continue;
}
$this->reduce(
project: $project,
document: new Document($document),
metrics: $payload['metrics'],
getProjectDB: $getProjectDB
);
}
$this->stats[$projectId]['project'] = $project;
$this->stats[$projectId]['receivedAt'] = DateTime::now();
foreach ($payload['metrics'] ?? [] as $metric) {
$this->keys++;
if (!isset($this->stats[$projectId]['keys'][$metric['key']])) {
$this->stats[$projectId]['keys'][$metric['key']] = $metric['value'];
continue;
}
$this->stats[$projectId]['keys'][$metric['key']] += $metric['value'];
}
// If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
if (
$this->keys >= self::KEYS_THRESHOLD ||
(time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0)
) {
Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
$queueForStatsUsageDump
->setStats($this->stats)
->trigger();
$this->stats = [];
$this->keys = 0;
$this->lastTriggeredTime = time();
}
}
/**
* On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
* When we remove a parent document we need to deduct his children aggregation from the project scope.
* @param Document $project
* @param Document $document
* @param array $metrics
* @param callable $getProjectDB
* @return void
*/
private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void
{
$dbForProject = $getProjectDB($project);
try {
switch (true) {
case $document->getCollection() === 'users': // users
$sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
if (!empty($sessions)) {
$metrics[] = [
'key' => METRIC_SESSIONS,
'value' => ($sessions * -1),
];
}
break;
case $document->getCollection() === 'databases': // databases
$collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
if (!empty($collections['value'])) {
$metrics[] = [
'key' => METRIC_COLLECTIONS,
'value' => ($collections['value'] * -1),
];
}
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
$metrics[] = [
'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
'value' => ($documents['value'] * -1),
];
}
break;
case $document->getCollection() === 'buckets':
$files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
if (!empty($files['value'])) {
$metrics[] = [
'key' => METRIC_FILES,
'value' => ($files['value'] * -1),
];
}
if (!empty($storage['value'])) {
$metrics[] = [
'key' => METRIC_FILES_STORAGE,
'value' => ($storage['value'] * -1),
];
}
break;
case $document->getCollection() === 'functions':
$deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
if (!empty($deployments['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS,
'value' => ($deployments['value'] * -1),
];
}
if (!empty($deploymentsStorage['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS_STORAGE,
'value' => ($deploymentsStorage['value'] * -1),
];
}
if (!empty($builds['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS,
'value' => ($builds['value'] * -1),
];
}
if (!empty($buildsStorage['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_STORAGE,
'value' => ($buildsStorage['value'] * -1),
];
}
if (!empty($buildsCompute['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_COMPUTE,
'value' => ($buildsCompute['value'] * -1),
];
}
if (!empty($executions['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS,
'value' => ($executions['value'] * -1),
];
}
if (!empty($executionsCompute['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS_COMPUTE,
'value' => ($executionsCompute['value'] * -1),
];
}
break;
default:
break;
}
} catch (\Throwable $e) {
console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}");
}
}
}
@@ -0,0 +1,353 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Extend\Exception;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\Registry\Registry;
use Utopia\System\System;
class StatsUsageDump extends Action
{
public const METRIC_COLLECTION_LEVEL_STORAGE = 4;
public const METRIC_DATABASE_LEVEL_STORAGE = 3;
public const METRIC_PROJECT_LEVEL_STORAGE = 2;
protected array $stats = [];
protected Registry $register;
/**
* Metrics to skip writing to logsDB
* As these metrics are calculated separately
* by logs DB
* @var array
*/
protected array $skipBaseMetrics = [
METRIC_DATABASES => true,
METRIC_BUCKETS => true,
METRIC_USERS => true,
METRIC_FUNCTIONS => true,
METRIC_TEAMS => true,
METRIC_MESSAGES => true,
METRIC_MAU => true,
METRIC_WEBHOOKS => true,
METRIC_PLATFORMS => true,
METRIC_PROVIDERS => true,
METRIC_TOPICS => true,
METRIC_KEYS => true,
METRIC_FILES => true,
METRIC_FILES_STORAGE => true,
METRIC_DEPLOYMENTS_STORAGE => true,
METRIC_BUILDS_STORAGE => true,
METRIC_DEPLOYMENTS => true,
METRIC_BUILDS => true,
METRIC_COLLECTIONS => true,
METRIC_DOCUMENTS => true,
];
/**
* Skip metrics associated with parent IDs
* these need to be checked individually with `str_ends_with`
*/
protected array $skipParentIdMetrics = [
'.files',
'.files.storage',
'.collections',
'.documents',
'.deployments',
'.deployments.storage',
'.builds',
'.builds.storage',
];
/**
* @var callable
*/
protected mixed $getLogsDB;
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
'inf' => '0000-00-00 00:00'
];
public static function getName(): string
{
return 'stats-usage-dump';
}
/**
* @throws \Exception
*/
public function __construct()
{
$this
->inject('message')
->inject('getProjectDB')
->inject('getLogsDB')
->inject('register')
->callback([$this, 'action']);
}
/**
* @param Message $message
* @param callable $getProjectDB
* @param callable $getLogsDB
* @param Registry $register
* @return void
* @throws Exception
* @throws \Throwable
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, callable $getProjectDB, callable $getLogsDB, Registry $register): void
{
$this->getLogsDB = $getLogsDB;
$this->register = $register;
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
foreach ($payload['stats'] ?? [] as $stats) {
$project = new Document($stats['project'] ?? []);
$numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
$receivedAt = $stats['receivedAt'] ?? 'NONE';
if ($numberOfKeys === 0) {
continue;
}
console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys);
try {
/** @var \Utopia\Database\Database $dbForProject */
$dbForProject = $getProjectDB($project);
foreach ($stats['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
if (str_contains($key, METRIC_DATABASES_STORAGE)) {
try {
$this->handleDatabaseStorage($key, $dbForProject, $project);
} catch (\Exception $e) {
console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage());
}
continue;
}
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
$document = new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => System::getEnv('_APP_REGION', 'default'),
]);
$documentClone = new Document($document->getArrayCopy());
$dbForProject->createOrUpdateDocumentsWithIncrease(
'stats',
'value',
[$document]
);
$this->writeToLogsDB($project, $documentClone);
}
}
} catch (\Exception $e) {
console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage());
}
}
}
private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project): void
{
$data = explode('.', $key);
$start = microtime(true);
$updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) {
$id = \md5("{$time}_{$period}_{$key}");
$document = new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => System::getEnv('_APP_REGION', 'default'),
]);
$documentClone = new Document($document->getArrayCopy());
$dbForProject->createOrUpdateDocumentsWithIncrease(
'stats',
'value',
[$document]
);
$this->writeToLogsDB($project, $documentClone);
};
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
$value = 0;
$previousValue = 0;
try {
$previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0);
} catch (\Exception $e) {
// No previous value
}
switch (count($data)) {
// Collection Level
case self::METRIC_COLLECTION_LEVEL_STORAGE:
Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']');
$databaseInternalId = $data[0];
$collectionInternalId = $data[1];
try {
$value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId);
} catch (\Exception $e) {
// Collection not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
// Compare with previous value
$diff = $value - $previousValue;
if ($diff === 0) {
break;
}
// Update Collection
$updateMetric($dbForProject, $project, $diff, $key, $period, $time);
// Update Database
$databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE);
$updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time);
// Update Project
$projectKey = METRIC_DATABASES_STORAGE;
$updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time);
break;
// Database Level
case self::METRIC_DATABASE_LEVEL_STORAGE:
Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']');
$databaseInternalId = $data[0];
$collections = [];
try {
$collections = $dbForProject->find('database_' . $databaseInternalId);
} catch (\Exception $e) {
// Database not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
foreach ($collections as $collection) {
try {
$value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId());
} catch (\Exception $e) {
// Collection not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
}
$diff = $value - $previousValue;
if ($diff === 0) {
break;
}
// Update Database
$databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE);
$updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time);
// Update Project
$projectKey = METRIC_DATABASES_STORAGE;
$updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time);
break;
// Project Level
case self::METRIC_PROJECT_LEVEL_STORAGE:
Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']');
// Get all project databases
$databases = $dbForProject->find('database');
// Recalculate all databases
foreach ($databases as $database) {
$collections = $dbForProject->find('database_' . $database->getInternalId());
foreach ($collections as $collection) {
try {
$value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
} catch (\Exception $e) {
// Collection not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
}
}
$diff = $value - $previousValue;
// Update Project
$projectKey = METRIC_DATABASES_STORAGE;
$updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time);
break;
}
}
$end = microtime(true);
console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds');
}
protected function writeToLogsDB(Document $project, Document $document): void
{
if (!System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', false)) {
Console::log('Dual Writing is disabled. Skipping...');
return;
}
if (array_key_exists($document->getAttribute('metric'), $this->skipBaseMetrics)) {
return;
}
foreach ($this->skipParentIdMetrics as $skipMetric) {
if (str_ends_with($document->getAttribute('metric'), $skipMetric)) {
return;
}
}
/** @var \Utopia\Database\Database $dbForLogs*/
$dbForLogs = call_user_func($this->getLogsDB, $project);
try {
$dbForLogs->createOrUpdateDocumentsWithIncrease(
'stats',
'value',
[$document]
);
Console::success('Usage logs pushed to Logs DB');
} catch (\Throwable $th) {
Console::error($th->getMessage());
}
$this->register->get('pools')->get('logs')->reclaim();
}
}
+9 -9
View File
@@ -3,7 +3,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Event\Mail;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Template\Template;
use Exception;
use Utopia\Database\Database;
@@ -35,9 +35,9 @@ class Webhooks extends Action
->inject('project')
->inject('dbForPlatform')
->inject('queueForMails')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('log')
->callback(fn (Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForMails, $queueForUsage, $log));
->callback(fn (Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForMails, $queueForStatsUsage, $log));
}
/**
@@ -49,7 +49,7 @@ class Webhooks extends Action
* @return void
* @throws Exception
*/
public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log): void
public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, Log $log): void
{
$this->errors = [];
$payload = $message->getPayload() ?? [];
@@ -66,7 +66,7 @@ class Webhooks extends Action
foreach ($project->getAttribute('webhooks', []) as $webhook) {
if (array_intersect($webhook->getAttribute('events', []), $events)) {
$this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForUsage);
$this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForStatsUsage);
}
}
@@ -85,7 +85,7 @@ class Webhooks extends Action
* @param Mail $queueForMails
* @return void
*/
private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage): void
private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage): void
{
if ($webhook->getAttribute('enabled') !== true) {
return;
@@ -168,7 +168,7 @@ class Webhooks extends Action
$dbForPlatform->purgeCachedDocument('projects', $project->getId());
$this->errors[] = $logs;
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_WEBHOOKS_FAILED, 1)
->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_FAILED), 1)
;
@@ -178,13 +178,13 @@ class Webhooks extends Action
$webhook->setAttribute('attempts', 0); // Reset attempts on success
$dbForPlatform->updateDocument('webhooks', $webhook->getId(), $webhook);
$dbForPlatform->purgeCachedDocument('projects', $project->getId());
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_WEBHOOKS_SENT, 1)
->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_SENT), 1)
;
}
$queueForUsage
$queueForStatsUsage
->setProject($project)
->trigger();
}
@@ -323,9 +323,9 @@ class Project extends Model
}
/**
* Get Collection
* Filter document structure
*
* @return string
* @return Document
*/
public function filter(Document $document): Document
{