Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '1.7.x' into feat-tokens-module

Browse Source
This commit is contained in:
Chirag Aggarwal
2025-04-17 06:19:18 +00:00
parent 3f133cab74 d8e5a97266
commit b2dffe4057
191 changed files with 6060 additions and 6609 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Appwrite/Event/Realtime.php
+46 -11
View File
@@ -7,10 +7,17 @@ use Utopia\Database\Document;
class Realtime extends Event
{
protected array $subscribers = [];
public function __construct()
{
}
/**
* Get Realtime payload for this event.
*
* @return array
*/
public function getRealtimePayload(): array
{
$payload = [];
@@ -24,6 +31,28 @@ class Realtime extends Event
return $payload;
}
/**
* Set subscribers for this realtime event.
*
* @param array $subscribers
* @return array
*/
public function setSubscribers(array $subscribers): self
{
$this->subscribers = $subscribers;
return $this;
}
/**
* Get subscribers for this realtime event.
*
* @return array
*/
public function getSubscribers(): array
{
return $this->subscribers;
}
/**
* Execute Event.
*
@@ -53,17 +82,23 @@ class Realtime extends Event
bucket: $bucket,
);
RealtimeAdapter::send(
projectId: $target['projectId'] ?? $this->getProject()->getId(),
payload: $this->getRealtimePayload(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles'],
options: [
'permissionsChanged' => $target['permissionsChanged'],
'userId' => $this->getParam('userId')
]
);
$projectIds = !empty($this->getSubscribers())
? $this->getSubscribers()
: [$target['projectId'] ?? $this->getProject()->getId()];
foreach ($projectIds as $projectId) {
RealtimeAdapter::send(
projectId: $projectId,
payload: $this->getRealtimePayload(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles'],
options: [
'permissionsChanged' => $target['permissionsChanged'],
'userId' => $this->getParam('userId')
]
);
}
return true;
}
src/Appwrite/Event/Usage.php
-78
View File
@@ -1,78 +0,0 @@
<?php
namespace Appwrite\Event;
use Utopia\Database\Document;
use Utopia\Queue\Publisher;
class Usage extends Event
{
protected array $metrics = [];
protected array $reduce = [];
public function __construct(protected Publisher $publisher)
{
parent::__construct($publisher);
$this
->setQueue(Event::USAGE_QUEUE_NAME)
->setClass(Event::USAGE_CLASS_NAME);
}
/**
* Add reduce.
*
* @param Document $document
* @return self
*/
public function addReduce(Document $document): self
{
$this->reduce[] = $document;
return $this;
}
/**
* Add metric.
*
* @param string $key
* @param int $value
* @return self
*/
public function addMetric(string $key, int $value): self
{
$this->metrics[] = [
'key' => $key,
'value' => $value,
];
return $this;
}
/**
* Prepare the payload for the usage event.
*
* @return array
*/
protected function preparePayload(): array
{
return [
'project' => $this->project,
'reduce' => $this->reduce,
'metrics' => $this->metrics,
];
}
/**
* Sends metrics to the usage worker.
*
* @return string|bool
*/
public function trigger(): string|bool
{
parent::trigger();
$this->metrics = [];
return true;
}
}
src/Appwrite/Event/UsageDump.php
-44
View File
@@ -1,44 +0,0 @@
<?php
namespace Appwrite\Event;
use Utopia\Queue\Publisher;
class UsageDump extends Event
{
protected array $stats;
public function __construct(protected Publisher $publisher)
{
parent::__construct($publisher);
$this
->setQueue(Event::USAGE_DUMP_QUEUE_NAME)
->setClass(Event::USAGE_DUMP_CLASS_NAME);
}
/**
* Add Stats.
*
* @param array $stats
* @return self
*/
public function setStats(array $stats): self
{
$this->stats = $stats;
return $this;
}
/**
* Prepare the payload for the usage dump event.
*
* @return array
*/
protected function preparePayload(): array
{
return [
'stats' => $this->stats,
];
}
}
src/Appwrite/Migration/Migration.php
+1
View File
@@ -93,6 +93,7 @@ abstract class Migration
'1.6.0' => 'V21',
'1.6.1' => 'V21',
'1.6.2' => 'V22',
'1.7.0' => 'V23',
];
/**
src/Appwrite/Migration/Version/V19.php
+1 -2
View File
@@ -10,7 +10,6 @@ use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception;
use Utopia\Database\Query;
use Utopia\System\System;
class V19 extends Migration
{
@@ -731,7 +730,7 @@ class V19 extends Migration
if (empty($document->getAttribute('scheduleId', null))) {
$schedule = $this->consoleDB->createDocument('schedules', new Document([
'region' => System::getEnv('_APP_REGION', 'default'), // Todo replace with projects region
'region' => $project->getAttribute('region'),
'resourceType' => 'function',
'resourceId' => $document->getId(),
'resourceInternalId' => $document->getInternalId(),
src/Appwrite/Migration/Version/V23.php
+69
View File
@@ -0,0 +1,69 @@
<?php
namespace Appwrite\Migration\Version;
use Appwrite\Migration\Migration;
use Exception;
use Throwable;
use Utopia\CLI\Console;
use Utopia\Database\Database;
class V23 extends Migration
{
/**
* @throws Throwable
*/
public function execute(): void
{
/**
* Disable SubQueries for Performance.
*/
foreach (['subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships', 'subQueryVariables', 'subQueryChallenges', 'subQueryProjectVariables', 'subQueryTargets', 'subQueryTopicTargets'] as $name) {
Database::addFilter(
$name,
fn () => null,
fn () => []
);
}
Console::info('Migrating Collections');
$this->migrateCollections();
}
/**
* Migrate Collections.
*
* @return void
* @throws Exception|Throwable
*/
private function migrateCollections(): void
{
$internalProjectId = $this->project->getInternalId();
$collectionType = match ($internalProjectId) {
'console' => 'console',
default => 'projects',
};
$collections = $this->collections[$collectionType];
foreach ($collections as $collection) {
$id = $collection['$id'];
Console::log("Migrating Collection \"{$id}\"");
$this->projectDB->setNamespace("_$internalProjectId");
switch ($id) {
case 'memberships':
// Create roles index
try {
$this->createIndexFromCollection($this->projectDB, $id, '_key_roles');
} catch (Throwable $th) {
Console::warning("'_key_roles' from {$id}: {$th->getMessage()}");
}
break;
}
usleep(50000);
}
}
}
src/Appwrite/Platform/Services/Workers.php
-8
View File
@@ -14,10 +14,6 @@ use Appwrite\Platform\Workers\Migrations;
use Appwrite\Platform\Workers\StatsResources;
use Appwrite\Platform\Workers\StatsUsage;
use Appwrite\Platform\Workers\StatsUsageDump;
/** remove */
use Appwrite\Platform\Workers\Usage;
use Appwrite\Platform\Workers\UsageDump;
/** /remove */
use Appwrite\Platform\Workers\Webhooks;
use Utopia\Platform\Service;
@@ -40,10 +36,6 @@ class Workers extends Service
->addAction(StatsUsage::getName(), new StatsUsage())
->addAction(Migrations::getName(), new Migrations())
->addAction(StatsResources::getName(), new StatsResources())
/** Remove */
->addAction(UsageDump::getName(), new UsageDump())
->addAction(Usage::getName(), new Usage())
/** /remove */
;
}
}
src/Appwrite/Platform/Tasks/Maintenance.php
+13 -32
View File
@@ -24,12 +24,13 @@ class Maintenance extends Action
$this
->desc('Schedules maintenance tasks and publishes them to our queues')
->inject('dbForPlatform')
->inject('console')
->inject('queueForCertificates')
->inject('queueForDeletes')
->callback(fn (Database $dbForPlatform, Certificate $queueForCertificates, Delete $queueForDeletes) => $this->action($dbForPlatform, $queueForCertificates, $queueForDeletes));
->callback(fn (Database $dbForPlatform, Document $console, Certificate $queueForCertificates, Delete $queueForDeletes) => $this->action($dbForPlatform, $console, $queueForCertificates, $queueForDeletes));
}
public function action(Database $dbForPlatform, Certificate $queueForCertificates, Delete $queueForDeletes): void
public function action(Database $dbForPlatform, Document $console, Certificate $queueForCertificates, Delete $queueForDeletes): void
{
Console::title('Maintenance V1');
Console::success(APP_NAME . ' maintenance process v1 has started');
@@ -41,19 +42,26 @@ class Maintenance extends Action
$cacheRetention = (int) System::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
$schedulesDeletionRetention = (int) System::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForPlatform, $queueForDeletes, $queueForCertificates) {
Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForPlatform, $console, $queueForDeletes, $queueForCertificates) {
$time = DateTime::now();
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
$this->foreachProject($dbForPlatform, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
$dbForPlatform->foreach('projects', function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
$queueForDeletes
->setType(DELETE_TYPE_MAINTENANCE)
->setProject($project)
->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly))
->trigger();
}, [
Query::limit(100),
]);
});
$queueForDeletes
->setType(DELETE_TYPE_MAINTENANCE)
->setProject($console)
->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly))
->trigger();
$this->notifyDeleteConnections($queueForDeletes);
$this->renewCertificates($dbForPlatform, $queueForCertificates);
@@ -62,33 +70,6 @@ class Maintenance extends Action
}, $interval, $delay);
}
protected function foreachProject(Database $dbForPlatform, callable $callback): void
{
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
$count = 0;
$chunk = 0;
$limit = 50;
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$projects = $dbForPlatform->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
$chunk++;
/** @var string[] $projectIds */
$sum = count($projects);
foreach ($projects as $project) {
$callback($project);
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
}
private function notifyDeleteConnections(Delete $queueForDeletes): void
{
$queueForDeletes
src/Appwrite/Platform/Tasks/ScheduleBase.php
+16 -2
View File
@@ -103,8 +103,15 @@ abstract class ScheduleBase extends Action
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
// Temporarly accepting both 'fra' and 'default'
// When all migrated, only use _APP_REGION with 'default' as default value
$regions = [System::getEnv('_APP_REGION', 'default')];
if (!in_array('default', $regions)) {
$regions[] = 'default';
}
$results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [
Query::equal('region', [System::getEnv('_APP_REGION', 'default')]),
Query::equal('region', $regions),
Query::equal('resourceType', [static::getSupportedResource()]),
Query::equal('active', [true]),
]));
@@ -153,8 +160,15 @@ abstract class ScheduleBase extends Action
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
// Temporarly accepting both 'fra' and 'default'
// When all migrated, only use _APP_REGION with 'default' as default value
$regions = [System::getEnv('_APP_REGION', 'default')];
if (!in_array('default', $regions)) {
$regions[] = 'default';
}
$results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [
Query::equal('region', [System::getEnv('_APP_REGION', 'default')]),
Query::equal('region', $regions),
Query::equal('resourceType', [static::getSupportedResource()]),
Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate),
]));
src/Appwrite/Platform/Workers/Audits.php
+43 -23
View File
@@ -7,8 +7,6 @@ use Exception;
use Throwable;
use Utopia\Audit\Audit;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Authorization;
use Utopia\Database\Exception\Structure;
@@ -18,15 +16,16 @@ use Utopia\System\System;
class Audits extends Action
{
private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development
private const BATCH_SIZE_PRODUCTION = 5_000;
private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds
protected const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development
protected const BATCH_SIZE_PRODUCTION = 5_000;
protected const BATCH_AGGREGATION_INTERVAL = 60; // in seconds
private int $lastTriggeredTime = 0;
private array $logs = [];
private function getBatchSize(): int
protected function getBatchSize(): int
{
return System::getEnv('_APP_ENV', 'development') === 'development'
? self::BATCH_SIZE_DEVELOPMENT
@@ -46,8 +45,9 @@ class Audits extends Action
$this
->desc('Audits worker')
->inject('message')
->inject('dbForProject')
->callback(fn ($message, $dbForProject) => $this->action($message, $dbForProject));
->inject('getProjectDB')
->inject('project')
->callback([$this, 'action']);
$this->lastTriggeredTime = time();
}
@@ -55,14 +55,15 @@ class Audits extends Action
/**
* @param Message $message
* @param Database $dbForProject
* @param callable $getProjectDB
* @param Document $project
* @return void
* @throws Throwable
* @throws \Utopia\Database\Exception
* @throws Authorization
* @throws Structure
*/
public function action(Message $message, Database $dbForProject): void
public function action(Message $message, callable $getProjectDB, Document $project): void
{
$payload = $message->getPayload() ?? [];
@@ -73,7 +74,11 @@ class Audits extends Action
Console::info('Aggregating audit logs');
$event = $payload['event'] ?? '';
$auditPayload = $payload['payload'] ?? '';
$auditPayload = '';
if ($project->getId() === 'console') {
$auditPayload = $payload['payload'] ?? '';
}
$mode = $payload['mode'] ?? '';
$resource = $payload['resource'] ?? '';
$userAgent = $payload['userAgent'] ?? '';
@@ -100,30 +105,45 @@ class Audits extends Action
'mode' => $mode,
'data' => $auditPayload,
],
'timestamp' => DateTime::formatTz(DateTime::now())
'timestamp' => date("Y-m-d H:i:s", $message->getTimestamp()),
];
$this->logs[] = $eventData;
if (isset($this->logs[$project->getInternalId()])) {
$this->logs[$project->getInternalId()]['logs'][] = $eventData;
} else {
$this->logs[$project->getInternalId()] = [
'project' => new Document([
'$id' => $project->getId(),
'$internalId' => $project->getInternalId(),
'database' => $project->getAttribute('database'),
]),
'logs' => [$eventData]
];
}
// Check if we should process the batch by checking both for the batch size and the elapsed time
$batchSize = $this->getBatchSize();
$shouldProcessBatch = count($this->logs) >= $batchSize;
if (!$shouldProcessBatch && count($this->logs) > 0) {
$shouldProcessBatch = (time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL;
$shouldProcessBatch = \count($this->logs) >= $batchSize;
if (!$shouldProcessBatch && \count($this->logs) > 0) {
$shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL;
}
if ($shouldProcessBatch) {
Console::log('Processing batch with ' . count($this->logs) . ' events');
$audit = new Audit($dbForProject);
try {
$audit->logBatch($this->logs);
Console::success('Audit logs processed successfully');
foreach ($this->logs as $internalId => $projectLogs) {
$dbForProject = $getProjectDB($projectLogs['project']);
Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events');
$audit = new Audit($dbForProject);
$audit->logBatch($projectLogs['logs']);
Console::success('Audit logs processed successfully');
unset($this->logs[$internalId]);
}
} catch (Throwable $e) {
Console::error('Error processing audit logs: ' . $e->getMessage());
} finally {
// Clear the pending events after successful batch processing
$this->logs = [];
$this->lastTriggeredTime = time();
}
}
src/Appwrite/Platform/Workers/Builds.php
+59 -75
View File
@@ -5,8 +5,9 @@ namespace Appwrite\Platform\Workers;
use Ahc\Jwt\JWT;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Realtime;
use Appwrite\Event\StatsUsage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Event\Webhook;
use Appwrite\Utopia\Response\Model\Deployment;
use Appwrite\Vcs\Comment;
use Exception;
@@ -49,15 +50,18 @@ class Builds extends Action
->inject('project')
->inject('dbForPlatform')
->inject('queueForEvents')
->inject('queueForWebhooks')
->inject('queueForFunctions')
->inject('queueForRealtime')
->inject('queueForStatsUsage')
->inject('cache')
->inject('dbForProject')
->inject('deviceForFunctions')
->inject('isResourceBlocked')
->inject('log')
->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) =>
$this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log));
->inject('executor')
->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log, Executor $executor) =>
$this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log, $executor));
}
/**
@@ -65,16 +69,19 @@ class Builds extends Action
* @param Document $project
* @param Database $dbForPlatform
* @param Event $queueForEvents
* @param Webhook $queueForWebhooks
* @param Func $queueForFunctions
* @param Realtime $queueForRealtime
* @param StatsUsage $queueForStatsUsage
* @param Cache $cache
* @param Database $dbForProject
* @param Device $deviceForFunctions
* @param Log $log
* @param Executor $executor
* @return void
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void
public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log, Executor $executor): void
{
$payload = $message->getPayload() ?? [];
@@ -95,7 +102,7 @@ class Builds extends Action
case BUILD_TYPE_RETRY:
Console::info('Creating build for deployment: ' . $deployment->getId());
$github = new GitHub($cache);
$this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log);
$this->buildDeployment($deviceForFunctions, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log, $executor);
break;
default:
@@ -105,7 +112,9 @@ class Builds extends Action
/**
* @param Device $deviceForFunctions
* @param Webhook $queueForWebhooks
* @param Func $queueForFunctions
* @param Realtime $queueForRealtime
* @param Event $queueForEvents
* @param StatsUsage $queueForStatsUsage
* @param Database $dbForPlatform
@@ -116,14 +125,13 @@ class Builds extends Action
* @param Document $deployment
* @param Document $template
* @param Log $log
* @param Executor $executor
* @return void
* @throws \Utopia\Database\Exception
* @throws Exception
*/
protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void
protected function buildDeployment(Device $deviceForFunctions, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log, Executor $executor): void
{
$executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
$functionId = $function->getId();
$log->addTag('functionId', $function->getId());
@@ -158,10 +166,7 @@ class Builds extends Action
}
// Realtime preparation
$allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [
'functionId' => $function->getId(),
'deploymentId' => $deployment->getId()
]);
$event = "functions.[functionId].deployments.[deploymentId].update";
$startTime = DateTime::now();
$durationStart = \microtime(true);
@@ -375,21 +380,16 @@ class Builds extends Action
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
/**
* Send realtime Event
* Trigger Realtime Event
*/
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
event: $allEvents[0],
payload: $build,
project: $project
);
Realtime::send(
projectId: 'console',
payload: $build->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles']
);
$queueForRealtime
->setProject($project)
->setSubscribers(['console'])
->setEvent($event)
->setParam('functionId', $function->getId())
->setParam('deploymentId', $deployment->getId())
->setPayload($build->getArrayCopy())
->trigger();
}
$tmpPath = '/tmp/builds/' . $buildId;
@@ -436,40 +436,34 @@ class Builds extends Action
$this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform);
}
/** Trigger Webhook */
$deploymentModel = new Deployment();
$deploymentUpdate =
$queueForEvents
->setQueue(Event::WEBHOOK_QUEUE_NAME)
->setClass(Event::WEBHOOK_CLASS_NAME)
->setProject($project)
->setEvent('functions.[functionId].deployments.[deploymentId].update')
->setParam('functionId', $function->getId())
->setParam('deploymentId', $deployment->getId())
->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules())));
$deploymentUpdate->trigger();
/** Trigger Webhook */
$queueForWebhooks
->from($deploymentUpdate)
->trigger();
/** Trigger Functions */
$queueForFunctions
->from($deploymentUpdate)
->trigger();
/** Trigger Realtime */
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
event: $allEvents[0],
payload: $build,
project: $project
);
Realtime::send(
projectId: 'console',
payload: $build->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles']
);
/** Trigger Realtime Event */
$queueForRealtime
->setProject($project)
->setSubscribers(['console'])
->setEvent($event)
->setParam('functionId', $function->getId())
->setParam('deploymentId', $deployment->getId())
->setPayload($build->getArrayCopy())
->trigger();
$vars = [];
@@ -562,12 +556,12 @@ class Builds extends Action
$err = $error;
}
}),
Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err, &$isCanceled) {
Co\go(function () use ($executor, $project, $function, $deployment, &$response, &$build, $dbForProject, $event, &$err, $queueForRealtime, &$isCanceled) {
try {
$executor->getLogs(
deploymentId: $deployment->getId(),
projectId: $project->getId(),
callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $allEvents, $project, &$isCanceled) {
callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $event, $project, $function, $deployment, $queueForRealtime, &$isCanceled) {
if ($isCanceled) {
return;
}
@@ -592,21 +586,16 @@ class Builds extends Action
$build = $dbForProject->updateDocument('builds', $build->getId(), $build);
/**
* Send realtime Event
* Trigger Realtime Event
*/
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
event: $allEvents[0],
payload: $build,
project: $project
);
Realtime::send(
projectId: 'console',
payload: $build->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles']
);
$queueForRealtime
->setProject($project)
->setSubscribers(['console'])
->setEvent($event)
->setParam('functionId', $function->getId())
->setParam('deploymentId', $deployment->getId())
->setPayload($build->getArrayCopy())
->trigger();
}
}
);
@@ -694,21 +683,16 @@ class Builds extends Action
}
} finally {
/**
* Send realtime Event
* Trigger Realtime Event
*/
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
event: $allEvents[0],
payload: $build,
project: $project
);
Realtime::send(
projectId: 'console',
payload: $build->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles']
);
$queueForRealtime
->setProject($project)
->setSubscribers(['console'])
->setEvent($event)
->setParam('functionId', $function->getId())
->setParam('deploymentId', $deployment->getId())
->setPayload($build->getArrayCopy())
->trigger();
/** Trigger usage queue */
if ($build->getAttribute('status') === 'ready') {
src/Appwrite/Platform/Workers/Certificates.php
+28 -43
View File
@@ -6,7 +6,8 @@ use Appwrite\Certificates\Adapter as CertificatesAdapter;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Mail;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Event\Realtime;
use Appwrite\Event\Webhook;
use Appwrite\Network\Validator\CNAME;
use Appwrite\Template\Template;
use Appwrite\Utopia\Response\Model\Rule;
@@ -46,12 +47,14 @@ class Certificates extends Action
->inject('dbForPlatform')
->inject('queueForMails')
->inject('queueForEvents')
->inject('queueForWebhooks')
->inject('queueForFunctions')
->inject('queueForRealtime')
->inject('log')
->inject('certificates')
->callback(
fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates) =>
$this->action($message, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates)
fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates) =>
$this->action($message, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates)
);
}
@@ -60,14 +63,16 @@ class Certificates extends Action
* @param Database $dbForPlatform
* @param Mail $queueForMails
* @param Event $queueForEvents
* @param Webhook $queueForWebhooks
* @param Func $queueForFunctions
* @param Realtime $queueForRealtime
* @param Log $log
* @param CertificatesAdapter $certificates
* @return void
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates): void
public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates): void
{
$payload = $message->getPayload() ?? [];
@@ -81,7 +86,7 @@ class Certificates extends Action
$log->addTag('domain', $domain->get());
$this->execute($domain, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates, $skipRenewCheck);
$this->execute($domain, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck);
}
/**
@@ -90,13 +95,14 @@ class Certificates extends Action
* @param Mail $queueForMails
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param Realtime $queueForRealtime
* @param CertificatesAdapter $certificates
* @param bool $skipRenewCheck
* @return void
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
private function execute(Domain $domain, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void
private function execute(Domain $domain, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void
{
/**
* 1. Read arguments and validate domain
@@ -186,7 +192,7 @@ class Certificates extends Action
$certificate->setAttribute('updated', DateTime::now());
// Save all changes we made to certificate document into database
$this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForFunctions);
$this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime);
}
}
@@ -199,13 +205,14 @@ class Certificates extends Action
* @param Database $dbForPlatform Database connection for console
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param Realtime $queueForRealtime
* @return void
* @throws \Utopia\Database\Exception
* @throws Authorization
* @throws Conflict
* @throws Structure
*/
private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions): void
private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime): void
{
// Check if update or insert required
$certificateDocument = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain])]);
@@ -219,7 +226,7 @@ class Certificates extends Action
}
$certificateId = $certificate->getId();
$this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForFunctions);
$this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime);
}
/**
@@ -338,7 +345,7 @@ class Certificates extends Action
*
* @return void
*/
private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions): void
private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime): void
{
// TODO: @christyjacob remove once we migrate the rules in 1.7.x
if (System::getEnv('_APP_RULES_FORMAT') === 'md5') {
@@ -367,50 +374,28 @@ class Certificates extends Action
return;
}
/** Trigger Webhook */
$ruleModel = new Rule();
$queueForEvents
->setQueue(Event::WEBHOOK_QUEUE_NAME)
->setClass(Event::WEBHOOK_CLASS_NAME)
->setProject($project)
->setEvent('rules.[ruleId].update')
->setParam('ruleId', $rule->getId())
->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules())))
->trigger();
->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules())));
/** Trigger Webhook */
$queueForWebhooks
->from($queueForEvents)
->trigger();
/** Trigger Functions */
$queueForFunctions
->setProject($project)
->setEvent('rules.[ruleId].update')
->setParam('ruleId', $rule->getId())
->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules())))
->from($queueForEvents)
->trigger();
/** Trigger realtime event */
$allEvents = Event::generateEvents('rules.[ruleId].update', [
'ruleId' => $rule->getId(),
]);
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
event: $allEvents[0],
payload: $rule,
project: $project
);
Realtime::send(
projectId: 'console',
payload: $rule->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles']
);
Realtime::send(
projectId: $project->getId(),
payload: $rule->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles']
);
/** Trigger Realtime Events */
$queueForRealtime
->from($queueForEvents)
->setSubscribers(['console', $projectId])
->trigger();
}
}
}
src/Appwrite/Platform/Workers/Databases.php
+66 -71
View File
@@ -2,8 +2,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Event\Event;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Event\Realtime;
use Exception;
use Utopia\CLI\Console;
use Utopia\Database\Database;
@@ -37,8 +36,9 @@ class Databases extends Action
->inject('project')
->inject('dbForPlatform')
->inject('dbForProject')
->inject('queueForRealtime')
->inject('log')
->callback(fn (Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log) => $this->action($message, $project, $dbForPlatform, $dbForProject, $log));
->callback(fn (Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime, Log $log) => $this->action($message, $project, $dbForPlatform, $dbForProject, $queueForRealtime, $log));
}
/**
@@ -46,16 +46,17 @@ class Databases extends Action
* @param Document $project
* @param Database $dbForPlatform
* @param Database $dbForProject
* @param Realtime $queueForRealtime
* @param Log $log
* @return void
* @throws \Exception
*/
public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Log $log): void
public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime, Log $log): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new \Exception('Missing payload');
throw new Exception('Missing payload');
}
$type = $payload['type'];
@@ -75,11 +76,11 @@ class Databases extends Action
match (\strval($type)) {
DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject),
DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject),
DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject),
DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject),
DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject),
DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject),
default => throw new \Exception('No database operation for type: ' . \strval($type)),
DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime),
DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime),
DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime),
DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime),
default => throw new Exception('No database operation for type: ' . \strval($type)),
};
}
@@ -90,13 +91,14 @@ class Databases extends Action
* @param Document $project
* @param Database $dbForPlatform
* @param Database $dbForProject
* @param Realtime $queueForRealtime
* @return void
* @throws Authorization
* @throws Conflict
* @throws \Exception
* @throws \Throwable
*/
private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject): void
private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime): void
{
if ($collection->isEmpty()) {
throw new Exception('Missing collection');
@@ -106,12 +108,7 @@ class Databases extends Action
}
$projectId = $project->getId();
$events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update', [
'databaseId' => $database->getId(),
'collectionId' => $collection->getId(),
'attributeId' => $attribute->getId()
]);
$event = "databases.[databaseId].collections.[collectionId].attributes.[attributeId].update";
/**
* TODO @christyjacob4 verify if this is still the case
* Fetch attribute from the database, since with Resque float values are loosing informations.
@@ -169,7 +166,7 @@ class Databases extends Action
break;
default:
if (!$dbForProject->createAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters)) {
throw new \Exception('Failed to create Attribute');
throw new Exception('Failed to create Attribute');
}
}
@@ -200,7 +197,7 @@ class Databases extends Action
throw $e;
} finally {
$this->trigger($database, $collection, $attribute, $project, $projectId, $events);
$this->trigger($database, $collection, $project, $event, $queueForRealtime, $attribute);
if (! $relatedCollection->isEmpty()) {
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId());
@@ -217,13 +214,14 @@ class Databases extends Action
* @param Document $project
* @param Database $dbForPlatform
* @param Database $dbForProject
* @param Realtime $queueForRealtime
* @return void
* @throws Authorization
* @throws Conflict
* @throws \Exception
* @throws \Throwable
**/
private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject): void
private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime): void
{
if ($collection->isEmpty()) {
throw new Exception('Missing collection');
@@ -233,15 +231,9 @@ class Databases extends Action
}
$projectId = $project->getId();
$events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [
'databaseId' => $database->getId(),
'collectionId' => $collection->getId(),
'attributeId' => $attribute->getId()
]);
$event = 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete';
$collectionId = $collection->getId();
$key = $attribute->getAttribute('key', '');
$status = $attribute->getAttribute('status', '');
$type = $attribute->getAttribute('type', '');
$project = $dbForPlatform->getDocument('projects', $projectId);
$options = $attribute->getAttribute('options', []);
@@ -312,7 +304,7 @@ class Databases extends Action
throw $e;
} finally {
$this->trigger($database, $collection, $attribute, $project, $projectId, $events);
$this->trigger($database, $collection, $project, $event, $queueForRealtime, $attribute);
}
// The underlying database removes/rebuilds indexes when attribute is removed
@@ -358,7 +350,7 @@ class Databases extends Action
}
if ($exists) { // Delete the duplicate if created, else update in db
$this->deleteIndex($database, $collection, $index, $project, $dbForPlatform, $dbForProject);
$this->deleteIndex($database, $collection, $index, $project, $dbForPlatform, $dbForProject, $queueForRealtime);
} else {
$dbForProject->updateDocument('indexes', $index->getId(), $index);
}
@@ -381,6 +373,7 @@ class Databases extends Action
* @param Document $project
* @param Database $dbForPlatform
* @param Database $dbForProject
* @param Realtime $queueForRealtime
* @return void
* @throws Authorization
* @throws Conflict
@@ -388,7 +381,7 @@ class Databases extends Action
* @throws DatabaseException
* @throws \Throwable
*/
private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject): void
private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime): void
{
if ($collection->isEmpty()) {
throw new Exception('Missing collection');
@@ -398,12 +391,7 @@ class Databases extends Action
}
$projectId = $project->getId();
$events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [
'databaseId' => $database->getId(),
'collectionId' => $collection->getId(),
'indexId' => $index->getId()
]);
$event = 'databases.[databaseId].collections.[collectionId].indexes.[indexId].update';
$collectionId = $collection->getId();
$key = $index->getAttribute('key', '');
$type = $index->getAttribute('type', '');
@@ -430,7 +418,7 @@ class Databases extends Action
throw $e;
} finally {
$this->trigger($database, $collection, $index, $project, $projectId, $events);
$this->trigger($database, $collection, $project, $event, $queueForRealtime, null, $index);
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
}
}
@@ -442,6 +430,7 @@ class Databases extends Action
* @param Document $project
* @param Database $dbForPlatform
* @param Database $dbForProject
* @param Realtime $queueForRealtime
* @return void
* @throws Authorization
* @throws Conflict
@@ -449,7 +438,7 @@ class Databases extends Action
* @throws DatabaseException
* @throws \Throwable
*/
private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject): void
private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject, Realtime $queueForRealtime): void
{
if ($collection->isEmpty()) {
throw new Exception('Missing collection');
@@ -459,12 +448,7 @@ class Databases extends Action
}
$projectId = $project->getId();
$events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [
'databaseId' => $database->getId(),
'collectionId' => $collection->getId(),
'indexId' => $index->getId()
]);
$event = 'databases.[databaseId].collections.[collectionId].indexes.[indexId].delete';
$key = $index->getAttribute('key');
$status = $index->getAttribute('status', '');
$project = $dbForPlatform->getDocument('projects', $projectId);
@@ -490,7 +474,7 @@ class Databases extends Action
throw $e;
} finally {
$this->trigger($database, $collection, $index, $project, $projectId, $events);
$this->trigger($database, $collection, $project, $event, $queueForRealtime, null, $index);
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collection->getId());
}
}
@@ -532,7 +516,6 @@ class Databases extends Action
$collectionId = $collection->getId();
$collectionInternalId = $collection->getInternalId();
$databaseId = $database->getId();
$databaseInternalId = $database->getInternalId();
$dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId());
@@ -568,21 +551,22 @@ class Databases extends Action
/**
* @param string $collection collectionID
* @param string $collectionId
* @param array $queries
* @param Database $database
* @param callable|null $callback
* @return void
* @throws Exception
*/
protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
protected function deleteByGroup(string $collectionId, array $queries, Database $database, callable $callback = null): void
{
$start = \microtime(true);
try {
$documents = $database->deleteDocuments($collection, $queries);
$documents = $database->deleteDocuments($collectionId, $queries);
} catch (\Throwable $th) {
Console::error('Failed to delete documents for collection ' . $collection . ': ' . $th->getMessage());
$tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : '';
Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collectionId} {$tenant} :{$th->getMessage()}");
return;
}
@@ -598,31 +582,42 @@ class Databases extends Action
Console::info("Deleted {$count} documents by group in " . ($end - $start) . " seconds");
}
/**
* @param Document $database
* @param Document $collection
* @param Document $project
* @param Realtime $queueForRealtime
* @param Document|null $attribute
* @param Document|null $index
* @return void
*/
protected function trigger(
Document $database,
Document $collection,
Document $attribute,
Document $project,
string $projectId,
array $events
string $event,
Realtime $queueForRealtime,
Document|null $attribute = null,
Document|null $index = null,
): void {
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
event: $events[0],
payload: $attribute,
project: $project,
);
Realtime::send(
projectId: 'console',
payload: $attribute->getArrayCopy(),
events: $events,
channels: $target['channels'],
roles: $target['roles'],
options: [
'projectId' => $projectId,
'databaseId' => $database->getId(),
'collectionId' => $collection->getId()
]
);
$queueForRealtime
->setProject($project)
->setSubscribers(['console'])
->setEvent($event)
->setParam('databaseId', $database->getId())
->setParam('collectionId', $collection->getId());
if ($attribute !== null && !empty($attribute)) {
$queueForRealtime
->setParam('attributeId', $attribute->getId())
->setPayload($attribute->getArrayCopy());
}
if ($index !== null && !empty($index)) {
$queueForRealtime
->setParam('indexId', $index->getId())
->setPayload($index->getArrayCopy());
}
$queueForRealtime->trigger();
}
}
src/Appwrite/Platform/Workers/Deletes.php
+166 -61
View File
@@ -31,6 +31,8 @@ use Utopia\System\System;
class Deletes extends Action
{
protected array $selects = ['$internalId', '$id', '$collection', '$permissions', '$updatedAt'];
public static function getName(): string
{
return 'deletes';
@@ -47,18 +49,19 @@ class Deletes extends Action
->inject('project')
->inject('dbForPlatform')
->inject('getProjectDB')
->inject('timelimit')
->inject('getLogsDB')
->inject('deviceForFiles')
->inject('deviceForFunctions')
->inject('deviceForBuilds')
->inject('deviceForCache')
->inject('certificates')
->inject('executor')
->inject('executionRetention')
->inject('auditRetention')
->inject('log')
->callback(
fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) =>
$this->action($message, $project, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log)
fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Executor $executor, string $executionRetention, string $auditRetention, Log $log) =>
$this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executor, $executionRetention, $auditRetention, $log)
);
}
@@ -66,7 +69,7 @@ class Deletes extends Action
* @throws Exception
* @throws Throwable
*/
public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void
public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Executor $executor, string $executionRetention, string $auditRetention, Log $log): void
{
$payload = $message->getPayload() ?? [];
@@ -91,10 +94,10 @@ class Deletes extends Action
$this->deleteProject($dbForPlatform, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document);
break;
case DELETE_TYPE_FUNCTIONS:
$this->deleteFunction($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project);
$this->deleteFunction($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project, $executor);
break;
case DELETE_TYPE_DEPLOYMENTS:
$this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project);
$this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project, $executor);
break;
case DELETE_TYPE_USERS:
$this->deleteUser($getProjectDB, $document, $project);
@@ -131,7 +134,7 @@ class Deletes extends Action
$this->deleteExpiredSessions($project, $getProjectDB);
break;
case DELETE_TYPE_USAGE:
$this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
$this->deleteUsageStats($project, $getProjectDB, $getLogsDB, $hourlyUsageRetentionDatetime);
break;
case DELETE_TYPE_CACHE_BY_RESOURCE:
$this->deleteCacheByResource($project, $getProjectDB, $resource, $resourceType);
@@ -158,7 +161,7 @@ class Deletes extends Action
$this->deleteExpiredTargets($project, $getProjectDB);
$this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
$this->deleteAuditLogs($project, $getProjectDB, $auditRetention);
$this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
$this->deleteUsageStats($project, $getProjectDB, $getLogsDB, $hourlyUsageRetentionDatetime);
$this->deleteExpiredSessions($project, $getProjectDB);
break;
default:
@@ -180,10 +183,17 @@ class Deletes extends Action
*/
private function deleteSchedules(Database $dbForPlatform, callable $getProjectDB, string $datetime): void
{
// Temporarly accepting both 'fra' and 'default'
// When all migrated, only use _APP_REGION with 'default' as default value
$regions = [System::getEnv('_APP_REGION', 'default')];
if (!in_array('default', $regions)) {
$regions[] = 'default';
}
$this->listByGroup(
'schedules',
[
Query::equal('region', [System::getEnv('_APP_REGION', 'default')]),
Query::equal('region', $regions),
Query::lessThanEqual('resourceUpdatedAt', $datetime),
Query::equal('active', [false]),
],
@@ -248,7 +258,8 @@ class Deletes extends Action
$this->deleteByGroup(
'subscribers',
[
Query::equal('topicInternalId', [$topic->getInternalId()])
Query::equal('topicInternalId', [$topic->getInternalId()]),
Query::orderAsc(),
],
$getProjectDB($project)
);
@@ -269,7 +280,8 @@ class Deletes extends Action
$this->deleteByGroup(
'subscribers',
[
Query::equal('targetInternalId', [$target->getInternalId()])
Query::equal('targetInternalId', [$target->getInternalId()]),
Query::orderAsc(),
],
$dbForProject,
function (Document $subscriber) use ($dbForProject, $target) {
@@ -306,7 +318,8 @@ class Deletes extends Action
$this->deleteByGroup(
'targets',
[
Query::equal('expired', [true])
Query::equal('expired', [true]),
Query::orderAsc(),
],
$getProjectDB($project),
function (Document $target) use ($getProjectDB, $project) {
@@ -320,7 +333,8 @@ class Deletes extends Action
$this->deleteByGroup(
'targets',
[
Query::equal('sessionInternalId', [$session->getInternalId()])
Query::equal('sessionInternalId', [$session->getInternalId()]),
Query::orderAsc(),
],
$getProjectDB($project),
function (Document $target) use ($getProjectDB, $project) {
@@ -347,14 +361,20 @@ class Deletes extends Action
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
);
$query[] = Query::equal('resource', [$resource]);
$queries = [
Query::equal('resource', [$resource])
];
if (!empty($resourceType)) {
$query[] = Query::equal('resourceType', [$resourceType]);
$queries[] = Query::equal('resourceType', [$resourceType]);
}
$queries[] = Query::select($this->selects);
$queries[] = Query::orderAsc();
$this->deleteByGroup(
'cache',
$query,
$queries,
$dbForProject,
function (Document $document) use ($cache, $projectId) {
$path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
@@ -385,13 +405,16 @@ class Deletes extends Action
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
);
$query = [
$queries = [
Query::select([...$this->selects, 'accessedAt']),
Query::lessThan('accessedAt', $datetime),
Query::orderDesc('accessedAt'),
Query::orderDesc(),
];
$this->deleteByGroup(
'cache',
$query,
$queries,
$dbForProject,
function (Document $document) use ($cache, $projectId) {
$path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
@@ -412,14 +435,35 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteUsageStats(Document $project, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
private function deleteUsageStats(Document $project, callable $getProjectDB, callable $getLogsDB, string $hourlyUsageRetentionDatetime): void
{
/** @var Database $dbForProject*/
$dbForProject = $getProjectDB($project);
// Delete Usage stats
$selects = [...$this->selects, 'time'];
// Delete Usage stats from projectDB
$this->deleteByGroup('stats', [
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::select($selects),
Query::equal('period', ['1h']),
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::orderDesc('time'),
Query::orderDesc(),
], $dbForProject);
if ($project->getId() !== 'console') {
/** @var Database $dbForLogs*/
$dbForLogs = call_user_func($getLogsDB, $project);
// Delete Usage stats from logsDB
$this->deleteByGroup('stats', [
Query::select($selects),
Query::equal('period', ['1h']),
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::orderDesc('time'),
Query::orderDesc(),
], $dbForLogs);
}
}
/**
@@ -438,7 +482,8 @@ class Deletes extends Action
$this->deleteByGroup(
'memberships',
[
Query::equal('teamInternalId', [$teamInternalId])
Query::equal('teamInternalId', [$teamInternalId]),
Query::orderAsc()
],
$dbForProject,
function (Document $membership) use ($dbForProject) {
@@ -528,7 +573,13 @@ class Deletes extends Action
if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) {
$dbForProject->deleteCollection($collection->getId());
} else {
$this->deleteByGroup($collection->getId(), [], database: $dbForProject);
$this->deleteByGroup(
$collection->getId(),
[
Query::orderAsc()
],
database: $dbForProject
);
}
} catch (Throwable $e) {
Console::error('Error deleting '.$collection->getId().' '.$e->getMessage());
@@ -548,58 +599,78 @@ class Deletes extends Action
// Delete Platforms
$this->deleteByGroup('platforms', [
Query::equal('projectInternalId', [$projectInternalId])
Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform);
// Delete project and function rules
$this->deleteByGroup('rules', [
Query::equal('projectInternalId', [$projectInternalId])
Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform, function (Document $document) use ($dbForPlatform, $certificates) {
$this->deleteRule($dbForPlatform, $document, $certificates);
});
// Delete Keys
$this->deleteByGroup('keys', [
Query::equal('projectInternalId', [$projectInternalId])
Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform);
// Delete Webhooks
$this->deleteByGroup('webhooks', [
Query::equal('projectInternalId', [$projectInternalId])
Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform);
// Delete VCS Installations
$this->deleteByGroup('installations', [
Query::equal('projectInternalId', [$projectInternalId])
Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform);
// Delete VCS Repositories
$this->deleteByGroup('repositories', [
Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform);
// Delete VCS comments
$this->deleteByGroup('vcsComments', [
Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform);
// Delete Schedules (No projectInternalId in this collection)
// Delete Schedules
$this->deleteByGroup('schedules', [
Query::equal('projectId', [$projectId]),
Query::orderAsc()
], $dbForPlatform);
// Delete metadata table
if ($projectTables) {
$dbForProject->deleteCollection(Database::METADATA);
} elseif ($sharedTablesV1) {
$this->deleteByGroup(Database::METADATA, [], $dbForProject);
$this->deleteByGroup(
Database::METADATA,
[
Query::orderAsc()
],
$dbForProject
);
} elseif ($sharedTablesV2) {
$queries = \array_map(
fn ($id) => Query::notEqual('$id', $id),
$projectCollectionIds
);
$this->deleteByGroup(Database::METADATA, $queries, $dbForProject);
$queries[] = Query::orderAsc();
$this->deleteByGroup(
Database::METADATA,
$queries,
$dbForProject
);
}
// Delete all storage directories
@@ -624,14 +695,16 @@ class Deletes extends Action
// Delete all sessions of this user from the sessions table and update the sessions field of the user record
$this->deleteByGroup('sessions', [
Query::equal('userInternalId', [$userInternalId])
Query::equal('userInternalId', [$userInternalId]),
Query::orderAsc()
], $dbForProject);
$dbForProject->purgeCachedDocument('users', $userId);
// Delete Memberships and decrement team membership counts
$this->deleteByGroup('memberships', [
Query::equal('userInternalId', [$userInternalId])
Query::equal('userInternalId', [$userInternalId]),
Query::orderAsc()
], $dbForProject, function (Document $document) use ($dbForProject) {
if ($document->getAttribute('confirm')) { // Count only confirmed members
$teamId = $document->getAttribute('teamId');
@@ -644,19 +717,22 @@ class Deletes extends Action
// Delete tokens
$this->deleteByGroup('tokens', [
Query::equal('userInternalId', [$userInternalId])
Query::equal('userInternalId', [$userInternalId]),
Query::orderAsc()
], $dbForProject);
// Delete identities
$this->deleteByGroup('identities', [
Query::equal('userInternalId', [$userInternalId])
Query::equal('userInternalId', [$userInternalId]),
Query::orderAsc()
], $dbForProject);
// Delete targets
$this->deleteByGroup(
'targets',
[
Query::equal('userInternalId', [$userInternalId])
Query::equal('userInternalId', [$userInternalId]),
Query::orderAsc()
],
$dbForProject,
function (Document $target) use ($getProjectDB, $project) {
@@ -675,9 +751,13 @@ class Deletes extends Action
private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void
{
$dbForProject = $getProjectDB($project);
// Delete Executions
$this->deleteByGroup('executions', [
Query::lessThan('$createdAt', $datetime)
Query::select([...$this->selects, '$createdAt']),
Query::lessThan('$createdAt', $datetime),
Query::orderDesc('$createdAt'),
Query::orderDesc(),
], $dbForProject);
}
@@ -695,7 +775,10 @@ class Deletes extends Action
// Delete Sessions
$this->deleteByGroup('sessions', [
Query::lessThan('$createdAt', $expired)
Query::select([...$this->selects, '$createdAt']),
Query::lessThan('$createdAt', $expired),
Query::orderDesc('$createdAt'),
Query::orderDesc(),
], $dbForProject);
}
@@ -709,14 +792,16 @@ class Deletes extends Action
{
// Delete Dead Realtime Logs
$this->deleteByGroup('realtime', [
Query::lessThan('timestamp', $datetime)
Query::lessThan('timestamp', $datetime),
Query::orderDesc('timestamp'),
Query::orderAsc(),
], $dbForPlatform);
}
/**
* @param Database $dbForPlatform
* @param callable $getProjectDB
* @param string $datetime
* @param string $auditRetention
* @return void
* @throws Exception
*/
@@ -724,10 +809,14 @@ class Deletes extends Action
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$audit = new Audit($dbForProject);
try {
$audit->cleanup($auditRetention);
$this->deleteByGroup(Audit::COLLECTION, [
Query::select([...$this->selects, 'time']),
Query::lessThan('time', $auditRetention),
Query::orderDesc('time'),
Query::orderAsc(),
], $dbForProject);
} catch (DatabaseException $e) {
Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage());
}
@@ -739,10 +828,11 @@ class Deletes extends Action
* @param Device $deviceForBuilds
* @param Document $document function document
* @param Document $project
* @param Executor $executor
* @return void
* @throws Exception
*/
private function deleteFunction(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void
private function deleteFunction(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project, Executor $executor): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
@@ -754,9 +844,10 @@ class Deletes extends Action
*/
Console::info("Deleting rules for function " . $functionId);
$this->deleteByGroup('rules', [
Query::equal('resourceType', ['function']),
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::equal('resourceInternalId', [$functionInternalId]),
Query::equal('projectInternalId', [$project->getInternalId()])
Query::equal('resourceType', ['function']),
Query::orderAsc()
], $dbForPlatform, function (Document $document) use ($project, $dbForPlatform, $certificates) {
$this->deleteRule($dbForPlatform, $document, $certificates);
});
@@ -766,8 +857,9 @@ class Deletes extends Action
*/
Console::info("Deleting variables for function " . $functionId);
$this->deleteByGroup('variables', [
Query::equal('resourceInternalId', [$functionInternalId]),
Query::equal('resourceType', ['function']),
Query::equal('resourceInternalId', [$functionInternalId])
Query::orderAsc()
], $dbForProject);
/**
@@ -777,7 +869,8 @@ class Deletes extends Action
$deploymentInternalIds = [];
$this->deleteByGroup('deployments', [
Query::equal('resourceInternalId', [$functionInternalId])
Query::equal('resourceInternalId', [$functionInternalId]),
Query::orderAsc()
], $dbForProject, function (Document $document) use ($deviceForFunctions, &$deploymentInternalIds) {
$deploymentInternalIds[] = $document->getInternalId();
$this->deleteDeploymentFiles($deviceForFunctions, $document);
@@ -790,7 +883,8 @@ class Deletes extends Action
foreach ($deploymentInternalIds as $deploymentInternalId) {
$this->deleteByGroup('builds', [
Query::equal('deploymentInternalId', [$deploymentInternalId])
Query::equal('deploymentInternalId', [$deploymentInternalId]),
Query::orderAsc()
], $dbForProject, function (Document $document) use ($deviceForBuilds) {
$this->deleteBuildFiles($deviceForBuilds, $document);
});
@@ -801,7 +895,9 @@ class Deletes extends Action
*/
Console::info("Deleting executions for function " . $functionId);
$this->deleteByGroup('executions', [
Query::equal('functionInternalId', [$functionInternalId])
Query::select($this->selects),
Query::equal('functionInternalId', [$functionInternalId]),
Query::orderAsc()
], $dbForProject);
/**
@@ -812,12 +908,15 @@ class Deletes extends Action
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::equal('resourceInternalId', [$functionInternalId]),
Query::equal('resourceType', ['function']),
Query::orderAsc()
], $dbForPlatform, function (Document $document) use ($dbForPlatform) {
$providerRepositoryId = $document->getAttribute('providerRepositoryId', '');
$projectInternalId = $document->getAttribute('projectInternalId', '');
$this->deleteByGroup('vcsComments', [
Query::equal('providerRepositoryId', [$providerRepositoryId]),
Query::equal('projectInternalId', [$projectInternalId]),
Query::orderAsc()
], $dbForPlatform);
});
@@ -825,7 +924,7 @@ class Deletes extends Action
* Request executor to delete all deployment containers
*/
Console::info("Requesting executor to delete all deployment containers for function " . $functionId);
$this->deleteRuntimes($getProjectDB, $document, $project);
$this->deleteRuntimes($getProjectDB, $document, $project, $executor);
}
/**
@@ -851,7 +950,7 @@ class Deletes extends Action
} else {
Console::error('Failed to delete deployment files: ' . $deploymentPath);
}
} catch (\Throwable $th) {
} catch (Throwable $th) {
Console::error('Failed to delete deployment files: ' . $deploymentPath);
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
@@ -881,7 +980,7 @@ class Deletes extends Action
} else {
Console::error('Failed to delete build files: ' . $buildPath);
}
} catch (\Throwable $th) {
} catch (Throwable $th) {
Console::error('Failed to delete deployment files: ' . $buildPath);
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
@@ -896,10 +995,11 @@ class Deletes extends Action
* @param Device $deviceForBuilds
* @param Document $document
* @param Document $project
* @param Executor $executor
* @return void
* @throws Exception
*/
private function deleteDeployment(callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void
private function deleteDeployment(callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project, Executor $executor): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
@@ -917,7 +1017,8 @@ class Deletes extends Action
Console::info("Deleting builds for deployment " . $deploymentId);
$this->deleteByGroup('builds', [
Query::equal('deploymentInternalId', [$deploymentInternalId])
Query::equal('deploymentInternalId', [$deploymentInternalId]),
Query::orderAsc()
], $dbForProject, function (Document $document) use ($deviceForBuilds) {
$this->deleteBuildFiles($deviceForBuilds, $document);
});
@@ -926,7 +1027,7 @@ class Deletes extends Action
* Request executor to delete all deployment containers
*/
Console::info("Requesting executor to delete deployment container for deployment " . $deploymentId);
$this->deleteRuntimes($getProjectDB, $document, $project);
$this->deleteRuntimes($getProjectDB, $document, $project, $executor);
}
/**
@@ -935,7 +1036,7 @@ class Deletes extends Action
* @param Database $database
* @param ?callable $callback
* @return void
* @throws Exception
* @throws DatabaseException
*/
protected function deleteByGroup(
string $collection,
@@ -945,10 +1046,15 @@ class Deletes extends Action
): void {
$start = \microtime(true);
/**
* deleteDocuments uses a cursor, we need to add a unique order by field or use default
*/
try {
$documents = $database->deleteDocuments($collection, $queries);
} catch (\Throwable $th) {
Console::error('Failed to delete documents for collection ' . $collection . ': ' . $th->getMessage());
} catch (Throwable $th) {
$tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : '';
Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collection} {$tenant} :{$th->getMessage()}");
return;
}
@@ -1077,13 +1183,12 @@ class Deletes extends Action
* @param callable $getProjectDB
* @param ?Document $function
* @param Document $project
* @param Executor $executor
* @return void
* @throws Exception
*/
private function deleteRuntimes(callable $getProjectDB, ?Document $function, Document $project): void
private function deleteRuntimes(callable $getProjectDB, ?Document $function, Document $project, Executor $executor): void
{
$executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
$deleteByFunction = function (Document $function) use ($getProjectDB, $project, $executor) {
$this->listByGroup(
'deployments',
src/Appwrite/Platform/Workers/Functions.php
+31 -34
View File
@@ -5,8 +5,9 @@ namespace Appwrite\Platform\Workers;
use Ahc\Jwt\JWT;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Realtime;
use Appwrite\Event\StatsUsage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Event\Webhook;
use Appwrite\Utopia\Response\Model\Execution;
use Exception;
use Executor\Executor;
@@ -44,15 +45,18 @@ class Functions extends Action
->inject('project')
->inject('message')
->inject('dbForProject')
->inject('queueForWebhooks')
->inject('queueForFunctions')
->inject('queueForRealtime')
->inject('queueForEvents')
->inject('queueForStatsUsage')
->inject('log')
->inject('executor')
->inject('isResourceBlocked')
->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked));
->callback(fn (Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, Executor $executor, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $executor, $isResourceBlocked));
}
public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void
public function action(Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, Executor $executor, callable $isResourceBlocked): void
{
$payload = $message->getPayload() ?? [];
@@ -136,11 +140,14 @@ class Functions extends Action
$this->execute(
log: $log,
dbForProject: $dbForProject,
queueForWebhooks: $queueForWebhooks,
queueForFunctions: $queueForFunctions,
queueForRealtime: $queueForRealtime,
queueForStatsUsage: $queueForStatsUsage,
queueForEvents: $queueForEvents,
project: $project,
function: $function,
executor: $executor,
trigger: 'event',
path: '/',
method: 'POST',
@@ -176,11 +183,14 @@ class Functions extends Action
$this->execute(
log: $log,
dbForProject: $dbForProject,
queueForWebhooks: $queueForWebhooks,
queueForFunctions: $queueForFunctions,
queueForRealtime: $queueForRealtime,
queueForStatsUsage: $queueForStatsUsage,
queueForEvents: $queueForEvents,
project: $project,
function: $function,
executor: $executor,
trigger: 'http',
path: $path,
method: $method,
@@ -198,11 +208,14 @@ class Functions extends Action
$this->execute(
log: $log,
dbForProject: $dbForProject,
queueForWebhooks: $queueForWebhooks,
queueForFunctions: $queueForFunctions,
queueForRealtime: $queueForRealtime,
queueForStatsUsage: $queueForStatsUsage,
queueForEvents: $queueForEvents,
project: $project,
function: $function,
executor: $executor,
trigger: 'schedule',
path: $path,
method: $method,
@@ -284,10 +297,12 @@ class Functions extends Action
* @param Log $log
* @param Database $dbForProject
* @param Func $queueForFunctions
* @param Realtime $queueForRealtime
* @param StatsUsage $queueForStatsUsage
* @param Event $queueForEvents
* @param Document $project
* @param Document $function
* @param Executor $executor
* @param string $trigger
* @param string $path
* @param string $method
@@ -307,11 +322,14 @@ class Functions extends Action
private function execute(
Log $log,
Database $dbForProject,
Webhook $queueForWebhooks,
Func $queueForFunctions,
Realtime $queueForRealtime,
StatsUsage $queueForStatsUsage,
Event $queueForEvents,
Document $project,
Document $function,
Executor $executor,
string $trigger,
string $path,
string $method,
@@ -502,7 +520,6 @@ class Functions extends Action
try {
$version = $function->getAttribute('version', 'v2');
$command = $runtime['startCommand'];
$executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
$command = $version === 'v2' ? '' : 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "' . $command . '"';
$executionResponse = $executor->createExecution(
projectId: $project->getId(),
@@ -564,20 +581,20 @@ class Functions extends Action
;
}
$execution = $dbForProject->updateDocument('executions', $executionId, $execution);
/** Trigger Webhook */
$executionModel = new Execution();
$queueForEvents
->setQueue(Event::WEBHOOK_QUEUE_NAME)
->setClass(Event::WEBHOOK_CLASS_NAME)
->setProject($project)
->setUser($user)
->setEvent('functions.[functionId].executions.[executionId].update')
->setParam('functionId', $function->getId())
->setParam('executionId', $execution->getId())
->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules())))
->setPayload($execution->getArrayCopy(array_keys($executionModel->getRules())));
/** Trigger Webhook */
$queueForWebhooks
->from($queueForEvents)
->trigger();
/** Trigger Functions */
@@ -585,31 +602,11 @@ class Functions extends Action
->from($queueForEvents)
->trigger();
/** Trigger realtime event */
$allEvents = Event::generateEvents('functions.[functionId].executions.[executionId].update', [
'functionId' => $function->getId(),
'executionId' => $execution->getId()
]);
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
event: $allEvents[0],
payload: $execution,
project: $project
);
Realtime::send(
projectId: 'console',
payload: $execution->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles']
);
Realtime::send(
projectId: $project->getId(),
payload: $execution->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles']
);
/** Trigger Realtime Events */
$queueForRealtime
->from($queueForEvents)
->setSubscribers(['console', $project->getId()])
->trigger();
if (!empty($error)) {
throw new Exception($error, $errorCode);
src/Appwrite/Platform/Workers/Migrations.php
+38 -39
View File
@@ -3,8 +3,7 @@
namespace Appwrite\Platform\Workers;
use Ahc\Jwt\JWT;
use Appwrite\Event\Event;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Event\Realtime;
use Exception;
use Utopia\CLI\Console;
use Utopia\Config\Config;
@@ -19,12 +18,14 @@ use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite;
use Utopia\Migration\Exception as MigrationException;
use Utopia\Migration\Source;
use Utopia\Migration\Sources\Appwrite as SourceAppwrite;
use Utopia\Migration\Sources\CSV;
use Utopia\Migration\Sources\Firebase;
use Utopia\Migration\Sources\NHost;
use Utopia\Migration\Sources\Supabase;
use Utopia\Migration\Transfer;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\Storage\Device;
use Utopia\System\System;
class Migrations extends Action
@@ -33,8 +34,13 @@ class Migrations extends Action
protected Database $dbForPlatform;
protected Device $deviceForImports;
protected Document $project;
/**
* @var callable
*/
protected $logError;
public static function getName(): string
@@ -54,15 +60,18 @@ class Migrations extends Action
->inject('dbForProject')
->inject('dbForPlatform')
->inject('logError')
->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError));
->inject('queueForRealtime')
->inject('deviceForImports')
->callback(fn (Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports) => $this->action($message, $project, $dbForProject, $dbForPlatform, $logError, $queueForRealtime, $deviceForImports));
}
/**
* @throws Exception
*/
public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError): void
public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports): void
{
$payload = $message->getPayload() ?? [];
$this->deviceForImports = $deviceForImports;
if (empty($payload)) {
throw new Exception('Missing payload');
@@ -87,7 +96,7 @@ class Migrations extends Action
return;
}
$this->processMigration($migration);
$this->processMigration($migration, $queueForRealtime);
}
/**
@@ -96,7 +105,9 @@ class Migrations extends Action
protected function processSource(Document $migration): Source
{
$source = $migration->getAttribute('source');
$resourceId = $migration->getAttribute('resourceId');
$credentials = $migration->getAttribute('credentials');
$migrationOptions = $migration->getAttribute('options');
return match ($source) {
Firebase::getName() => new Firebase(
@@ -125,6 +136,12 @@ class Migrations extends Action
$credentials['endpoint'] === 'http://localhost/v1' ? 'http://appwrite/v1' : $credentials['endpoint'],
$credentials['apiKey'],
),
CSV::getName() => new CSV(
$resourceId,
$migrationOptions['path'],
$this->deviceForImports,
$this->dbForProject
),
default => throw new \Exception('Invalid source type'),
};
}
@@ -155,34 +172,16 @@ class Migrations extends Action
* @throws \Utopia\Database\Exception
* @throws Exception
*/
protected function updateMigrationDocument(Document $migration, Document $project): Document
protected function updateMigrationDocument(Document $migration, Document $project, Realtime $queueForRealtime): Document
{
/** Trigger Realtime */
$allEvents = Event::generateEvents('migrations.[migrationId].update', [
'migrationId' => $migration->getId(),
]);
$target = Realtime::fromPayload(
event: $allEvents[0],
payload: $migration,
project: $project
);
Realtime::send(
projectId: 'console',
payload: $migration->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles'],
);
Realtime::send(
projectId: $project->getId(),
payload: $migration->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles'],
);
/** Trigger Realtime Events */
$queueForRealtime
->setProject($project)
->setSubscribers(['console', $project->getId()])
->setEvent('migrations.[migrationId].update')
->setParam('migrationId', $migration->getId())
->setPayload($migration->getArrayCopy())
->trigger();
return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration);
}
@@ -234,7 +233,7 @@ class Migrations extends Action
* @throws \Utopia\Database\Exception
* @throws Exception
*/
protected function processMigration(Document $migration): void
protected function processMigration(Document $migration, Realtime $queueForRealtime): void
{
$project = $this->project;
$projectDocument = $this->dbForPlatform->getDocument('projects', $project->getId());
@@ -258,7 +257,7 @@ class Migrations extends Action
$migration->setAttribute('stage', 'processing');
$migration->setAttribute('status', 'processing');
$this->updateMigrationDocument($migration, $projectDocument);
$this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime);
$source = $this->processSource($migration);
$destination = $this->processDestination($migration, $tempAPIKey);
@@ -272,14 +271,14 @@ class Migrations extends Action
/** Start Transfer */
$migration->setAttribute('stage', 'migrating');
$this->updateMigrationDocument($migration, $projectDocument);
$this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime);
$transfer->run(
$migration->getAttribute('resources'),
function () use ($migration, $transfer, $projectDocument) {
function () use ($migration, $transfer, $projectDocument, $queueForRealtime) {
$migration->setAttribute('resourceData', json_encode($transfer->getCache()));
$migration->setAttribute('statusCounters', json_encode($transfer->getStatusCounters()));
$this->updateMigrationDocument($migration, $projectDocument);
$this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime);
},
$migration->getAttribute('resourceId'),
$migration->getAttribute('resourceType')
@@ -316,7 +315,7 @@ class Migrations extends Action
}
$migration->setAttribute('errors', $errorMessages);
$this->updateMigrationDocument($migration, $projectDocument);
$this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime);
return;
}
@@ -357,7 +356,7 @@ class Migrations extends Action
$migration->setAttribute('errors', $errorMessages);
}
} finally {
$this->updateMigrationDocument($migration, $projectDocument);
$this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime);
if ($migration->getAttribute('status', '') === 'failed') {
Console::error('Migration('.$migration->getInternalId().':'.$migration->getId().') failed, Project('.$this->project->getInternalId().':'.$this->project->getId().')');
src/Appwrite/Platform/Workers/StatsResources.php
+48 -10
View File
@@ -77,10 +77,13 @@ class StatsResources extends Action
// Reset documents for each job
$this->documents = [];
$startTime = microtime(true);
$this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project);
$endTime = microtime(true);
$executionTime = $endTime - $startTime;
Console::info('Project: ' . $project->getId() . '(' . $project->getInternalId() . ') aggregated in ' . $executionTime .' seconds');
}
protected function countForProject(Database $dbForPlatform, callable $getLogsDB, callable $getProjectDB, Document $project): void
{
Console::info('Begining count for: ' . $project->getId());
@@ -111,6 +114,13 @@ class StatsResources extends Action
$keys = $dbForPlatform->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()])
]);
$domains = $dbForPlatform->count('rules', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::equal('owner', ['']),
]);
$databases = $dbForProject->count('databases');
$buckets = $dbForProject->count('buckets');
$users = $dbForProject->count('users');
@@ -129,9 +139,20 @@ class StatsResources extends Action
]);
$teams = $dbForProject->count('teams');
$functions = $dbForProject->count('functions');
$messages = $dbForProject->count('messages');
$providers = $dbForProject->count('providers');
$topics = $dbForProject->count('topics');
$targets = $dbForProject->count('targets');
$emailTargets = $dbForProject->count('targets', [
Query::equal('providerType', [MESSAGE_TYPE_EMAIL])
]);
$pushTargets = $dbForProject->count('targets', [
Query::equal('providerType', [MESSAGE_TYPE_PUSH])
]);
$smsTargets = $dbForProject->count('targets', [
Query::equal('providerType', [MESSAGE_TYPE_SMS])
]);
$metrics = [
METRIC_DATABASES => $databases,
@@ -148,6 +169,11 @@ class StatsResources extends Action
METRIC_PROVIDERS => $providers,
METRIC_TOPICS => $topics,
METRIC_KEYS => $keys,
METRIC_DOMAINS => $domains,
METRIC_TARGETS => $targets,
str_replace('{providerType}', MESSAGE_TYPE_EMAIL, METRIC_PROVIDER_TYPE_TARGETS) => $emailTargets,
str_replace('{providerType}', MESSAGE_TYPE_PUSH, METRIC_PROVIDER_TYPE_TARGETS) => $pushTargets,
str_replace('{providerType}', MESSAGE_TYPE_SMS, METRIC_PROVIDER_TYPE_TARGETS) => $smsTargets,
];
foreach ($metrics as $metric => $value) {
@@ -167,7 +193,7 @@ class StatsResources extends Action
}
try {
$this->countForDatabase($dbForProject, $dbForLogs, $region);
$this->countForDatabase($dbForProject, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]);
}
@@ -227,42 +253,54 @@ class StatsResources extends Action
$this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalImageTransformations);
}
protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region)
protected function countForDatabase(Database $dbForProject, string $region)
{
$totalCollections = 0;
$totalDocuments = 0;
$this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $dbForLogs, $region, &$totalCollections, &$totalDocuments) {
$totalDatabaseStorage = 0;
$this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $region, &$totalCollections, &$totalDocuments, &$totalDatabaseStorage) {
$collections = $dbForProject->count('database_' . $database->getInternalId());
$metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS);
$this->createStatsDocuments($region, $metric, $collections);
$documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region);
[$documents, $storage] = $this->countForCollections($dbForProject, $database, $region);
$totalDatabaseStorage += $storage;
$totalDocuments += $documents;
$totalCollections += $collections;
});
$this->createStatsDocuments($region, METRIC_COLLECTIONS, $totalCollections);
$this->createStatsDocuments($region, METRIC_DOCUMENTS, $totalDocuments);
$this->createStatsDocuments($region, METRIC_DATABASES_STORAGE, $totalDatabaseStorage);
}
protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int
protected function countForCollections(Database $dbForProject, Document $database, string $region): array
{
$databaseDocuments = 0;
$this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $dbForLogs, $database, $region, &$totalCollections, &$databaseDocuments) {
$databaseStorage = 0;
$this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $database, $region, &$databaseStorage, &$databaseDocuments) {
$documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
$metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS);
$this->createStatsDocuments($region, $metric, $documents);
$databaseDocuments += $documents;
$collectionStorage = $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
$metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE);
$this->createStatsDocuments($region, $metric, $collectionStorage);
$databaseStorage += $collectionStorage;
});
$metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS);
$this->createStatsDocuments($region, $metric, $databaseDocuments);
return $databaseDocuments;
$metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE);
$this->createStatsDocuments($region, $metric, $databaseStorage);
return [$databaseDocuments, $databaseStorage];
}
protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region)
src/Appwrite/Platform/Workers/StatsUsage.php
+9 -2
View File
@@ -17,13 +17,20 @@ class StatsUsage extends Action
private int $lastTriggeredTime = 0;
private int $keys = 0;
private const INFINITY_PERIOD = '_inf_';
private const KEYS_THRESHOLD = 10000;
private const BATCH_SIZE_DEVELOPMENT = 1;
private const BATCH_SIZE_PRODUCTION = 10_000;
public static function getName(): string
{
return 'stats-usage';
}
private function getBatchSize(): int
{
return System::getEnv('_APP_ENV', 'development') === 'development'
? self::BATCH_SIZE_DEVELOPMENT
: self::BATCH_SIZE_PRODUCTION;
}
/**
* @throws Exception
*/
@@ -86,7 +93,7 @@ class StatsUsage extends Action
// If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
if (
$this->keys >= self::KEYS_THRESHOLD ||
$this->keys >= $this->getBatchSize() ||
(time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0)
) {
Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
src/Appwrite/Platform/Workers/StatsUsageDump.php
+17 -7
View File
@@ -48,6 +48,7 @@ class StatsUsageDump extends Action
METRIC_BUILDS => true,
METRIC_COLLECTIONS => true,
METRIC_DOCUMENTS => true,
METRIC_DATABASES_STORAGE => true,
];
/**
@@ -63,6 +64,7 @@ class StatsUsageDump extends Action
'.deployments.storage',
'.builds',
'.builds.storage',
'.databases.storage'
];
/**
@@ -117,7 +119,7 @@ class StatsUsageDump extends Action
$project = new Document($stats['project'] ?? []);
$numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
$receivedAt = $stats['receivedAt'] ?? 'NONE';
$receivedAt = $stats['receivedAt'] ?? null;
if ($numberOfKeys === 0) {
continue;
}
@@ -134,7 +136,7 @@ class StatsUsageDump extends Action
if (str_contains($key, METRIC_DATABASES_STORAGE)) {
try {
$this->handleDatabaseStorage($key, $dbForProject, $project);
$this->handleDatabaseStorage($key, $dbForProject, $project, $receivedAt);
} catch (\Exception $e) {
console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage());
}
@@ -142,7 +144,11 @@ class StatsUsageDump extends Action
}
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$time = null;
if ($period !== 'inf') {
$time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time());
}
$id = \md5("{$time}_{$period}_{$key}");
$document = new Document([
@@ -171,12 +177,12 @@ class StatsUsageDump extends Action
}
}
private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project): void
private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project, string $receivedAt): void
{
$data = explode('.', $key);
$start = microtime(true);
$updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) {
$updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) use ($receivedAt) {
$id = \md5("{$time}_{$period}_{$key}");
$document = new Document([
@@ -197,7 +203,11 @@ class StatsUsageDump extends Action
};
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$time = null;
if ($period !== 'inf') {
$time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time());
}
$id = \md5("{$time}_{$period}_{$key}");
$value = 0;
@@ -320,7 +330,7 @@ class StatsUsageDump extends Action
protected function writeToLogsDB(Document $project, Document $document): void
{
if (!System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', false)) {
if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') {
Console::log('Dual Writing is disabled. Skipping...');
return;
}
src/Appwrite/Platform/Workers/Usage.php
-292
View File
@@ -1,292 +0,0 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Event\UsageDump;
use Exception;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\System\System;
class Usage extends Action
{
private array $stats = [];
private int $lastTriggeredTime = 0;
private int $aggregationInterval = 20;
private int $keys = 0;
private const INFINITY_PERIOD = '_inf_';
private const KEYS_THRESHOLD = 20000;
public static function getName(): string
{
return 'usage';
}
/**
* @throws Exception
*/
public function __construct()
{
$this
->desc('Usage worker')
->inject('message')
->inject('project')
->inject('getProjectDB')
->inject('queueForUsageDump')
->callback(function (Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump) {
$this->action($message, $project, $getProjectDB, $queueForUsageDump);
});
$this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
$this->lastTriggeredTime = time();
}
/**
* @param Message $message
* @param Document $project
* @param callable $getProjectDB
* @param UsageDump $queueForUsageDump
* @return void
* @throws \Utopia\Database\Exception
* @throws Exception
*/
public function action(Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
if (empty($project->getAttribute('database'))) {
var_dump($payload);
return;
}
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
if (empty($document)) {
continue;
}
$this->reduce(
project: $project,
document: new Document($document),
metrics: $payload['metrics'],
getProjectDB: $getProjectDB
);
}
$this->stats[$projectId]['project'] = [
'$id' => $project->getId(),
'$internalId' => $project->getInternalId(),
'database' => $project->getAttribute('database'),
];
$this->stats[$projectId]['receivedAt'] = DateTime::now();
foreach ($payload['metrics'] ?? [] as $metric) {
$this->keys++;
if (!isset($this->stats[$projectId]['keys'][$metric['key']])) {
$this->stats[$projectId]['keys'][$metric['key']] = $metric['value'];
continue;
}
$this->stats[$projectId]['keys'][$metric['key']] += $metric['value'];
}
// If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
if (
$this->keys >= self::KEYS_THRESHOLD ||
(time() - $this->lastTriggeredTime > $this->aggregationInterval && $this->keys > 0)
) {
Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
$queueForUsageDump
->setStats($this->stats)
->trigger();
$this->stats = [];
$this->keys = 0;
$this->lastTriggeredTime = time();
}
}
/**
* On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
* When we remove a parent document we need to deduct his children aggregation from the project scope.
* @param Document $project
* @param Document $document
* @param array $metrics
* @param callable $getProjectDB
* @return void
*/
private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void
{
$dbForProject = $getProjectDB($project);
try {
switch (true) {
case $document->getCollection() === 'users': // users
$sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
if (!empty($sessions)) {
$metrics[] = [
'key' => METRIC_SESSIONS,
'value' => ($sessions * -1),
];
}
break;
case $document->getCollection() === 'databases': // databases
$collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
if (!empty($collections['value'])) {
$metrics[] = [
'key' => METRIC_COLLECTIONS,
'value' => ($collections['value'] * -1),
];
}
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
$metrics[] = [
'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
'value' => ($documents['value'] * -1),
];
}
break;
case $document->getCollection() === 'buckets':
$files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
if (!empty($files['value'])) {
$metrics[] = [
'key' => METRIC_FILES,
'value' => ($files['value'] * -1),
];
}
if (!empty($storage['value'])) {
$metrics[] = [
'key' => METRIC_FILES_STORAGE,
'value' => ($storage['value'] * -1),
];
}
break;
case $document->getCollection() === 'functions':
$deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS)));
$buildsFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED)));
$buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$buildsComputeSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS)));
$buildsComputeFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED)));
$executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
if (!empty($deployments['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS,
'value' => ($deployments['value'] * -1),
];
}
if (!empty($deploymentsStorage['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS_STORAGE,
'value' => ($deploymentsStorage['value'] * -1),
];
}
if (!empty($builds['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS,
'value' => ($builds['value'] * -1),
];
}
if (!empty($buildsSuccess['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_SUCCESS,
'value' => ($buildsSuccess['value'] * -1),
];
}
if (!empty($buildsFailed['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_FAILED,
'value' => ($buildsFailed['value'] * -1),
];
}
if (!empty($buildsStorage['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_STORAGE,
'value' => ($buildsStorage['value'] * -1),
];
}
if (!empty($buildsCompute['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_COMPUTE,
'value' => ($buildsCompute['value'] * -1),
];
}
if (!empty($buildsComputeSuccess['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_COMPUTE_SUCCESS,
'value' => ($buildsComputeSuccess['value'] * -1),
];
}
if (!empty($buildsComputeFailed['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_COMPUTE_FAILED,
'value' => ($buildsComputeFailed['value'] * -1),
];
}
if (!empty($executions['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS,
'value' => ($executions['value'] * -1),
];
}
if (!empty($executionsCompute['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS_COMPUTE,
'value' => ($executionsCompute['value'] * -1),
];
}
break;
default:
break;
}
} catch (\Throwable $e) {
console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}");
}
}
}
src/Appwrite/Platform/Workers/UsageDump.php
-286
View File
@@ -1,286 +0,0 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Extend\Exception;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\System\System;
const METRIC_COLLECTION_LEVEL_STORAGE = 4;
const METRIC_DATABASE_LEVEL_STORAGE = 3;
const METRIC_PROJECT_LEVEL_STORAGE = 2;
class UsageDump extends Action
{
protected array $stats = [];
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
'inf' => '0000-00-00 00:00'
];
public static function getName(): string
{
return 'usage-dump';
}
/**
* @throws \Exception
*/
public function __construct()
{
$this
->inject('message')
->inject('getProjectDB')
->callback(function (Message $message, callable $getProjectDB) {
$this->action($message, $getProjectDB);
});
}
/**
* @param Message $message
* @param callable $getProjectDB
* @return void
* @throws Exception
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, callable $getProjectDB): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
foreach ($payload['stats'] ?? [] as $stats) {
$project = new Document($stats['project'] ?? []);
/**
* End temp bug fallback
*/
$numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
$receivedAt = $stats['receivedAt'] ?? 'NONE';
if ($numberOfKeys === 0) {
continue;
}
console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys);
try {
$dbForProject = $getProjectDB($project);
foreach ($stats['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
if (str_contains($key, METRIC_DATABASES_STORAGE)) {
try {
$this->handleDatabaseStorage($key, $dbForProject);
} catch (\Exception $e) {
console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage());
}
continue;
}
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => System::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats',
$id,
'value',
$value
);
}
}
}
}
} catch (\Exception $e) {
console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage());
}
}
}
private function handleDatabaseStorage(string $key, Database $dbForProject): void
{
$data = explode('.', $key);
$start = microtime(true);
$updateMetric = function (Database $dbForProject, int $value, string $key, string $period, string|null $time) {
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => System::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats',
$id,
'value',
$value
);
}
}
};
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
$value = 0;
$previousValue = 0;
try {
$previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0);
} catch (\Exception $e) {
// No previous value
}
switch (count($data)) {
// Collection Level
case METRIC_COLLECTION_LEVEL_STORAGE:
Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']');
$databaseInternalId = $data[0];
$collectionInternalId = $data[1];
try {
$value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId);
} catch (\Exception $e) {
// Collection not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
// Compare with previous value
$diff = $value - $previousValue;
if ($diff === 0) {
break;
}
// Update Collection
$updateMetric($dbForProject, $diff, $key, $period, $time);
// Update Database
$databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE);
$updateMetric($dbForProject, $diff, $databaseKey, $period, $time);
// Update Project
$projectKey = METRIC_DATABASES_STORAGE;
$updateMetric($dbForProject, $diff, $projectKey, $period, $time);
break;
// Database Level
case METRIC_DATABASE_LEVEL_STORAGE:
Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']');
$databaseInternalId = $data[0];
$collections = [];
try {
$collections = $dbForProject->find('database_' . $databaseInternalId);
} catch (\Exception $e) {
// Database not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
foreach ($collections as $collection) {
try {
$value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId());
} catch (\Exception $e) {
// Collection not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
}
$diff = $value - $previousValue;
if ($diff === 0) {
break;
}
// Update Database
$databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE);
$updateMetric($dbForProject, $diff, $databaseKey, $period, $time);
// Update Project
$projectKey = METRIC_DATABASES_STORAGE;
$updateMetric($dbForProject, $diff, $projectKey, $period, $time);
break;
// Project Level
case METRIC_PROJECT_LEVEL_STORAGE:
Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']');
// Get all project databases
$databases = $dbForProject->find('database');
// Recalculate all databases
foreach ($databases as $database) {
$collections = $dbForProject->find('database_' . $database->getInternalId());
foreach ($collections as $collection) {
try {
$value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
} catch (\Exception $e) {
// Collection not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
}
}
$diff = $value - $previousValue;
// Update Project
$projectKey = METRIC_DATABASES_STORAGE;
$updateMetric($dbForProject, $diff, $projectKey, $period, $time);
break;
}
}
$end = microtime(true);
console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds');
}
}
src/Appwrite/Utopia/Database/Validator/Queries/Base.php
+17 -2
View File
@@ -24,7 +24,15 @@ class Base extends Queries
public function __construct(string $collection, array $allowedAttributes)
{
$config = Config::getParam('collections', []);
$collections = array_merge($config['projects'], $config['buckets'], $config['databases'], $config['console']);
$collections = array_merge(
$config['projects'],
$config['buckets'],
$config['databases'],
$config['console'],
$config['logs']
);
$collection = $collections[$collection];
// array for constant lookup time
$allowedAttributesLookup = [];
@@ -35,6 +43,7 @@ class Base extends Queries
$attributes = [];
foreach ($collection['attributes'] as $attribute) {
$key = $attribute['$id'];
if (!isset($allowedAttributesLookup[$key])) {
continue;
}
@@ -62,12 +71,18 @@ class Base extends Queries
'array' => false,
]);
$internalId = new Document([
'key' => '$internalId',
'type' => Database::VAR_STRING,
'array' => false,
]);
$validators = [
new Limit(),
new Offset(),
new Cursor(),
new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES),
new Order($attributes),
new Order([...$attributes, $internalId]),
];
parent::__construct($validators);
src/Appwrite/Utopia/Database/Validator/Queries/Memberships.php
+2 -2
View File
@@ -9,12 +9,12 @@ class Memberships extends Base
'teamId',
'invited',
'joined',
'confirm'
'confirm',
'roles',
];
/**
* Expression constructor
*
*/
public function __construct()
{
src/Appwrite/Utopia/Response/Model/UsageBuckets.php
+13
View File
@@ -42,6 +42,19 @@ class UsageBuckets extends Model
'example' => [],
'array' => true
])
->addRule('imageTransformations', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of files transformations per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('imageTransformationsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of files transformations.',
'default' => 0,
'example' => 0,
])
;
}
src/Appwrite/Utopia/Response/Model/UsageProject.php
+13
View File
@@ -197,6 +197,19 @@ class UsageProject extends Model
'example' => [],
'array' => true
])
->addRule('imageTransformations', [
'type' => Response::MODEL_METRIC,
'description' => 'An array of aggregated number of image transformations.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('imageTransformationsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of image transformations.',
'default' => 0,
'example' => 0,
])
;
}
src/Executor/Executor.php
+29 -14
View File
@@ -21,17 +21,19 @@ class Executor
private bool $selfSigned = false;
private string $endpoint;
/**
* @var callable(string, string): string $endpoint
*/
private $endpointSelector;
protected array $headers;
public function __construct(string $endpoint)
/**
* @param callable(string, string): string $endpointSelector
*/
public function __construct(callable $endpointSelector)
{
if (!filter_var($endpoint, FILTER_VALIDATE_URL)) {
throw new Exception('Unsupported endpoint');
}
$this->endpoint = $endpoint;
$this->endpointSelector = $endpointSelector;
$this->headers = [
'content-type' => 'application/json',
'authorization' => 'Bearer ' . System::getEnv('_APP_EXECUTOR_SECRET', ''),
@@ -92,7 +94,8 @@ class Executor
'timeout' => $timeout,
];
$response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout);
$endpoint = $this->selectEndpoint($projectId, $deploymentId);
$response = $this->call($endpoint, self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout);
$status = $response['headers']['status-code'];
if ($status >= 400) {
@@ -123,7 +126,8 @@ class Executor
'timeout' => $timeout
];
$this->call(self::METHOD_GET, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout, $callback);
$endpoint = $this->selectEndpoint($projectId, $deploymentId);
$this->call($endpoint, self::METHOD_GET, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout, $callback);
}
/**
@@ -139,7 +143,8 @@ class Executor
$runtimeId = "$projectId-$deploymentId";
$route = "/runtimes/$runtimeId";
$response = $this->call(self::METHOD_DELETE, $route, [
$endpoint = $this->selectEndpoint($projectId, $deploymentId);
$response = $this->call($endpoint, self::METHOD_DELETE, $route, [
'x-opr-addressing-method' => 'broadcast'
], [], true, 30);
@@ -227,7 +232,8 @@ class Executor
$requestTimeout = $timeout + 15;
}
$response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId, 'content-type' => 'multipart/form-data', 'accept' => 'multipart/form-data' ], $params, true, $requestTimeout);
$endpoint = $this->selectEndpoint($projectId, $deploymentId);
$response = $this->call($endpoint, self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId, 'content-type' => 'multipart/form-data', 'accept' => 'multipart/form-data' ], $params, true, $requestTimeout);
$status = $response['headers']['status-code'];
if ($status >= 400) {
@@ -235,7 +241,11 @@ class Executor
throw new \Exception($message, $status);
}
$response['body']['headers'] = \json_decode($response['body']['headers'] ?? '{}', true);
$headers = $response['body']['headers'] ?? [];
if (is_string($headers)) {
$headers = \json_decode($headers, true);
}
$response['body']['headers'] = $headers;
$response['body']['statusCode'] = \intval($response['body']['statusCode'] ?? 500);
$response['body']['duration'] = \floatval($response['body']['duration'] ?? 0);
$response['body']['startTime'] = \floatval($response['body']['startTime'] ?? \microtime(true));
@@ -256,10 +266,10 @@ class Executor
* @return array|string
* @throws Exception
*/
public function call(string $method, string $path = '', array $headers = [], array $params = [], bool $decode = true, int $timeout = 15, callable $callback = null)
private function call(string $endpoint, string $method, string $path = '', array $headers = [], array $params = [], bool $decode = true, int $timeout = 15, callable $callback = null)
{
$headers = array_merge($this->headers, $headers);
$ch = curl_init($this->endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : ''));
$ch = curl_init($endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : ''));
$responseHeaders = [];
$responseStatus = -1;
$responseType = '';
@@ -422,4 +432,9 @@ class Executor
return $output;
}
private function selectEndpoint(string $projectId, string $deploymentId): string
{
return call_user_func($this->endpointSelector, $projectId, $deploymentId);
}
}