Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

clear unnecessary cli task

Browse Source
This commit is contained in:
shimon
2023-12-21 20:50:26 +02:00
parent 76e05983ff
commit 724c07914d
12 changed files with 0 additions and 898 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
-6
View File
@@ -98,16 +98,10 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-hamster
# Cloud Executabless
RUN chmod +x /usr/local/bin/hamster && \
chmod +x /usr/local/bin/volume-sync && \
chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
chmod +x /usr/local/bin/patch-recreate-repositories-documents && \
chmod +x /usr/local/bin/patch-delete-project-collections && \
chmod +x /usr/local/bin/delete-orphaned-projects && \
chmod +x /usr/local/bin/clear-card-cache && \
chmod +x /usr/local/bin/calc-users-stats && \
chmod +x /usr/local/bin/calc-tier-stats && \
chmod +x /usr/local/bin/get-migration-stats
# Letsencrypt Permissions
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/
bin/calc-tier-stats
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php calc-tier-stats $@
bin/calc-users-stats
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php calc-users-stats $@
bin/clear-card-cache
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php clear-card-cache $@
bin/delete-orphaned-projects
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php delete-orphaned-projects $@
bin/get-migration-stats
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php get-migration-stats $@
bin/hamster
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php hamster $@
src/Appwrite/Platform/Services/Tasks.php
-9
View File
@@ -11,15 +11,11 @@ use Appwrite\Platform\Tasks\Schedule;
use Appwrite\Platform\Tasks\SDKs;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\SSL;
use Appwrite\Platform\Tasks\Hamster;
use Appwrite\Platform\Tasks\Usage;
use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
use Appwrite\Platform\Tasks\VolumeSync;
use Appwrite\Platform\Tasks\CalcTierStats;
use Appwrite\Platform\Tasks\Upgrade;
use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
use Appwrite\Platform\Tasks\GetMigrationStats;
use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
class Tasks extends Service
@@ -32,7 +28,6 @@ class Tasks extends Service
->addAction(Usage::getName(), new Usage())
->addAction(Vars::getName(), new Vars())
->addAction(SSL::getName(), new SSL())
->addAction(Hamster::getName(), new Hamster())
->addAction(Doctor::getName(), new Doctor())
->addAction(Install::getName(), new Install())
->addAction(Upgrade::getName(), new Upgrade())
@@ -42,11 +37,7 @@ class Tasks extends Service
->addAction(SDKs::getName(), new SDKs())
->addAction(VolumeSync::getName(), new VolumeSync())
->addAction(Specs::getName(), new Specs())
->addAction(CalcTierStats::getName(), new CalcTierStats())
->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
->addAction(GetMigrationStats::getName(), new GetMigrationStats())
;
}
}
src/Appwrite/Platform/Tasks/CalcTierStats.php
-359
View File
@@ -1,359 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Exception;
use League\Csv\CannotInsertRecord;
use Utopia\App;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use League\Csv\Writer;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
class CalcTierStats extends Action
{
/*
* Csv cols headers
*/
private array $columns = [
'Project ID',
'Organization ID',
'Organization Members',
'Teams',
'Users',
'Requests',
'Bandwidth',
'Domains',
'Api keys',
'Webhooks',
'Platforms',
'Buckets',
'Files',
'Storage (bytes)',
'Max File Size (bytes)',
'Databases',
'Functions',
'Deployments',
'Executions',
'Migrations',
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
private array $usageStats = [
'project.$all.network.requests' => 'Requests',
'project.$all.network.bandwidth' => 'Bandwidth',
];
public static function getName(): string
{
return 'calc-tier-stats';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('register')
->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
$this->action($pools, $cache, $dbForConsole, $register);
});
}
/**
* @throws \Utopia\Exception
* @throws CannotInsertRecord
*/
public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
{
//docker compose exec -t appwrite calc-tier-stats
Console::title('Cloud free tier stats calculation V1');
Console::success(APP_NAME . ' cloud free tier stats calculation has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/tier_stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 100;
$sum = 100;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
/** Get Project ID */
$stats['Project ID'] = $project->getId();
$stats['Organization ID'] = $project->getAttribute('teamId', null);
/** Get Total Members */
$teamInternalId = $project->getAttribute('teamInternalId', null);
if ($teamInternalId) {
$stats['Organization Members'] = $dbForConsole->count('memberships', [
Query::equal('teamInternalId', [$teamInternalId])
]);
} else {
$stats['Organization Members'] = 0;
}
/** Get Total internal Teams */
try {
$stats['Teams'] = $dbForProject->count('teams', []);
} catch (\Throwable) {
$stats['Teams'] = 0;
}
/** Get Total users */
try {
$stats['Users'] = $dbForProject->count('users', []);
} catch (\Throwable) {
$stats['Users'] = 0;
}
/** Get Usage stats */
$range = '30d';
$periods = [
'30d' => [
'period' => '1d',
'limit' => 30,
]
];
$tmp = [];
$metrics = $this->usageStats;
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
foreach ($metrics as $metric => $name) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$tmp[$metric] = [];
foreach ($requestDocs as $requestDoc) {
if (empty($requestDoc)) {
continue;
}
$tmp[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$tmp[$metric] = array_reverse($tmp[$metric]);
$tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
}
});
foreach ($tmp as $key => $value) {
$stats[$metrics[$key]] = $value;
}
try {
/** Get Domains */
$stats['Domains'] = $dbForConsole->count('rules', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Domains'] = 0;
}
try {
/** Get Api keys */
$stats['Api keys'] = $dbForConsole->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Api keys'] = 0;
}
try {
/** Get Webhooks */
$stats['Webhooks'] = $dbForConsole->count('webhooks', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Webhooks'] = 0;
}
try {
/** Get Platforms */
$stats['Platforms'] = $dbForConsole->count('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Platforms'] = 0;
}
/** Get Files & Buckets */
$filesCount = 0;
$filesSum = 0;
$maxFileSize = 0;
$counter = 0;
try {
$buckets = $dbForProject->find('buckets', []);
foreach ($buckets as $bucket) {
$file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
if (empty($file)) {
continue;
}
$filesSum += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []);
$filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
$maxFileSize = $file->getAttribute('sizeOriginal');
}
$counter++;
}
} catch (\Throwable) {
;
}
$stats['Buckets'] = $counter;
$stats['Files'] = $filesCount;
$stats['Storage (bytes)'] = $filesSum;
$stats['Max File Size (bytes)'] = $maxFileSize;
try {
/** Get Total Functions */
$stats['Databases'] = $dbForProject->count('databases', []);
} catch (\Throwable) {
$stats['Databases'] = 0;
}
/** Get Total Functions */
try {
$stats['Functions'] = $dbForProject->count('functions', []);
} catch (\Throwable) {
$stats['Functions'] = 0;
}
/** Get Total Deployments */
try {
$stats['Deployments'] = $dbForProject->count('deployments', []);
} catch (\Throwable) {
$stats['Deployments'] = 0;
}
/** Get Total Executions */
try {
$stats['Executions'] = $dbForProject->count('executions', []);
} catch (\Throwable) {
$stats['Executions'] = 0;
}
/** Get Total Migrations */
try {
$stats['Migrations'] = $dbForProject->count('migrations', []);
} catch (\Throwable) {
$stats['Migrations'] = 0;
}
$csv->insertOne(array_values($stats));
} catch (\Throwable $th) {
Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$pools
->get('console')
->reclaim();
/** @var PHPMailer $mail */
$mail = $register->get('smtp');
$mail->clearAddresses();
$mail->clearAllRecipients();
$mail->clearReplyTos();
$mail->clearAttachments();
$mail->clearBCCs();
$mail->clearCCs();
try {
/** Addresses */
$mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
$recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
foreach ($recipients as $recipient) {
$mail->addAddress($recipient);
}
/** Attachments */
$mail->addAttachment($this->path);
/** Content */
$mail->Subject = "Cloud Report for {$this->date}";
$mail->Body = "Please find the daily cloud report atttached";
$mail->send();
Console::success('Email has been sent!');
} catch (Exception $e) {
Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
}
}
}
src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php
-161
View File
@@ -1,161 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\App;
use Utopia\Config\Config;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
use Utopia\Validator\Boolean;
class DeleteOrphanedProjects extends Action
{
public static function getName(): string
{
return 'delete-orphaned-projects';
}
public function __construct()
{
$this
->desc('Delete orphaned projects')
->param('commit', false, new Boolean(true), 'Commit project deletion', true)
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('register')
->callback(function (bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
$this->action($commit, $pools, $cache, $dbForConsole, $register);
});
}
public function action(bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
{
Console::title('Delete orphaned projects V1');
Console::success(APP_NAME . ' Delete orphaned projects started');
/** @var array $collections */
$collectionsConfig = Config::getParam('collections', [])['projects'] ?? [];
$collectionsConfig = array_merge([
'audit' => [
'$id' => ID::custom('audit'),
'$collection' => Database::METADATA
],
'abuse' => [
'$id' => ID::custom('abuse'),
'$collection' => Database::METADATA
]
], $collectionsConfig);
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
$projects = [$console];
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$orphans = 1;
$cnt = 0;
$count = 0;
$limit = 30;
$sum = 30;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
$collectionsCreated = 0;
$cnt++;
if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) {
$collectionsCreated = $dbForProject->count(Database::METADATA);
}
$msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')';
if ($collectionsCreated >= count($collectionsConfig)) {
Console::log($msg . ' ignoring....');
continue;
}
Console::log($msg);
if ($collectionsCreated > 0) {
$collections = $dbForProject->find(Database::METADATA, []);
foreach ($collections as $collection) {
if ($commit) {
$dbForProject->deleteCollection($collection->getId());
$dbForConsole->deleteCachedCollection($collection->getId());
}
Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')');
}
}
if ($commit) {
$dbForConsole->deleteDocument('projects', $project->getId());
$dbForConsole->deleteCachedDocument('projects', $project->getId());
if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) {
try {
$dbForProject->deleteCollection(Database::METADATA);
$dbForProject->deleteCachedCollection(Database::METADATA);
} catch (\Throwable $th) {
Console::warning('Metadata collection does not exist');
}
}
}
Console::info('--Deleting project no (' . $project->getInternalId() . ')');
$orphans++;
} catch (\Throwable $th) {
Console::error('Error: ' . $th->getMessage() . ' ' . $th->getTraceAsString());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans');
}
}
src/Appwrite/Platform/Tasks/GetMigrationStats.php
-187
View File
@@ -1,187 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Exception;
use League\Csv\CannotInsertRecord;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use League\Csv\Writer;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
class GetMigrationStats extends Action
{
/*
* Csv cols headers
*/
private array $columns = [
'Project ID',
'$id',
'$createdAt',
'status',
'stage',
'source'
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
public static function getName(): string
{
return 'get-migration-stats';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('register')
->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
$this->action($pools, $cache, $dbForConsole, $register);
});
}
/**
* @throws \Utopia\Exception
* @throws CannotInsertRecord
*/
public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
{
//docker compose exec -t appwrite get-migration-stats
Console::title('Migration stats calculation V1');
Console::success(APP_NAME . ' Migration stats calculation has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/migration_stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 100;
$sum = 100;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
/** Get Project ID */
$stats['Project ID'] = $project->getId();
/** Get Migration details */
$migrations = $dbForProject->find('migrations', [
Query::limit(500)
]);
$migrations = array_map(function ($migration) use ($project) {
return [
$project->getId(),
$migration->getAttribute('$id'),
$migration->getAttribute('$createdAt'),
$migration->getAttribute('status'),
$migration->getAttribute('stage'),
$migration->getAttribute('source'),
];
}, $migrations);
if (!empty($migrations)) {
$csv->insertAll($migrations);
}
} catch (\Throwable $th) {
Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$pools
->get('console')
->reclaim();
/** @var PHPMailer $mail */
$mail = $register->get('smtp');
$mail->clearAddresses();
$mail->clearAllRecipients();
$mail->clearReplyTos();
$mail->clearAttachments();
$mail->clearBCCs();
$mail->clearCCs();
try {
/** Addresses */
$mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
$recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
foreach ($recipients as $recipient) {
$mail->addAddress($recipient);
}
/** Attachments */
$mail->addAttachment($this->path);
/** Content */
$mail->Subject = "Migration Report for {$this->date}";
$mail->Body = "Please find the migration report atttached";
$mail->send();
Console::success('Email has been sent!');
} catch (Exception $e) {
Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
}
}
}
src/Appwrite/Platform/Tasks/Hamster.php
-158
View File
@@ -1,158 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Event\Hamster as EventHamster;
use Exception;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Database\Document;
class Hamster extends Action
{
public static function getName(): string
{
return 'hamster';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->inject('queueForHamster')
->inject('dbForConsole')
->callback(function (EventHamster $queueForHamster, Database $dbForConsole) {
$this->action($queueForHamster, $dbForConsole);
});
}
public function action(EventHamster $queueForHamster, Database $dbForConsole): void
{
Console::title('Cloud Hamster V1');
Console::success(APP_NAME . ' cloud hamster process has started');
$sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default)
$jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes)
$now = new \DateTime();
$now->setTimezone(new \DateTimeZone(date_default_timezone_get()));
$next = new \DateTime($now->format("Y-m-d $jobInitTime"));
$next->setTimezone(new \DateTimeZone(date_default_timezone_get()));
$delay = $next->getTimestamp() - $now->getTimestamp();
/**
* If time passed for the target day.
*/
if ($delay <= 0) {
$next->add(\DateInterval::createFromDateString('1 days'));
$delay = $next->getTimestamp() - $now->getTimestamp();
}
Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']');
Console::loop(function () use ($queueForHamster, $dbForConsole, $sleep) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Queuing Cloud Usage Stats every {$sleep} seconds");
$loopStart = microtime(true);
Console::info('Queuing stats for all projects');
$this->getStatsPerProject($queueForHamster, $dbForConsole, $loopStart);
Console::success('Completed queuing stats for all projects');
Console::info('Queuing stats for all organizations');
$this->getStatsPerOrganization($queueForHamster, $dbForConsole, $loopStart);
Console::success('Completed queuing stats for all organizations');
Console::info('Queuing stats for all users');
$this->getStatsPerUser($queueForHamster, $dbForConsole, $loopStart);
Console::success('Completed queuing stats for all users');
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Cloud Stats took {$loopTook} seconds");
}, $sleep, $delay);
}
protected function calculateByGroup(string $collection, Database $database, callable $callback)
{
$count = 0;
$chunk = 0;
$limit = 50;
$results = [];
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$chunk++;
$results = $database->find($collection, \array_merge([
Query::limit($limit),
Query::offset($count)
]));
$sum = count($results);
Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents');
foreach ($results as $document) {
call_user_func($callback, $database, $document);
$count++;
}
}
$executionEnd = \microtime(true);
Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
}
protected function getStatsPerOrganization(EventHamster $hamster, Database $dbForConsole, float $loopStart)
{
$this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($hamster, $loopStart) {
try {
$organization->setAttribute('$time', $loopStart);
$hamster
->setType(EventHamster::TYPE_ORGANISATION)
->setOrganization($organization)
->trigger();
} catch (Exception $e) {
Console::error($e->getMessage());
}
});
}
private function getStatsPerProject(EventHamster $hamster, Database $dbForConsole, float $loopStart)
{
$this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($hamster, $loopStart) {
try {
$project->setAttribute('$time', $loopStart);
$hamster
->setType(EventHamster::TYPE_PROJECT)
->setProject($project)
->trigger();
} catch (Exception $e) {
Console::error($e->getMessage());
}
});
}
protected function getStatsPerUser(EventHamster $hamster, Database $dbForConsole, float $loopStart)
{
$this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($hamster, $loopStart) {
try {
$user->setAttribute('$time', $loopStart);
$hamster
->setType(EventHamster::TYPE_USER)
->setUser($user)
->trigger();
} catch (Exception $e) {
Console::error($e->getMessage());
}
});
}
}