mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge branch '1.6.x' into 1.7.x
This commit is contained in:
commit
165caef9ab
100 files changed
+3721
-1299
No files matched your search
@@ -0,0 +1,158 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Auth;
|
||||
|
||||
use Ahc\Jwt\JWT;
|
||||
use Ahc\Jwt\JWTException;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\System\System;
|
||||
|
||||
class Key
|
||||
{
|
||||
public function __construct(
|
||||
protected string $projectId,
|
||||
protected string $type,
|
||||
protected string $role,
|
||||
protected array $scopes,
|
||||
protected string $name,
|
||||
protected bool $expired = false,
|
||||
protected array $disabledMetrics = [],
|
||||
) {
|
||||
}
|
||||
|
||||
public function getProjectId(): string
|
||||
{
|
||||
return $this->projectId;
|
||||
}
|
||||
|
||||
public function getType(): string
|
||||
{
|
||||
return $this->type;
|
||||
}
|
||||
|
||||
public function getRole(): string
|
||||
{
|
||||
return $this->role;
|
||||
}
|
||||
|
||||
public function getScopes(): array
|
||||
{
|
||||
return $this->scopes;
|
||||
}
|
||||
|
||||
public function getName(): string
|
||||
{
|
||||
return $this->name;
|
||||
}
|
||||
|
||||
public function isExpired(): bool
|
||||
{
|
||||
return $this->expired;
|
||||
}
|
||||
|
||||
public function getDisabledMetrics(): array
|
||||
{
|
||||
return $this->disabledMetrics;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode the given secret key into a Key object, containing the project ID, type, role, scopes, and name.
|
||||
* Can be a stored API key or a dynamic key (JWT).
|
||||
*
|
||||
* @param Document $project
|
||||
* @param string $key
|
||||
* @return Key
|
||||
* @throws Exception
|
||||
*/
|
||||
public static function decode(
|
||||
Document $project,
|
||||
string $key
|
||||
): Key {
|
||||
if (\str_contains($key, '_')) {
|
||||
[$type, $secret] = \explode('_', $key, 2);
|
||||
} else {
|
||||
$type = API_KEY_STANDARD;
|
||||
$secret = $key;
|
||||
}
|
||||
|
||||
$role = Auth::USER_ROLE_APPS;
|
||||
$roles = Config::getParam('roles', []);
|
||||
$scopes = $roles[Auth::USER_ROLE_APPS]['scopes'] ?? [];
|
||||
$expired = false;
|
||||
|
||||
$guestKey = new Key(
|
||||
$project->getId(),
|
||||
$type,
|
||||
Auth::USER_ROLE_GUESTS,
|
||||
$roles[Auth::USER_ROLE_GUESTS]['scopes'] ?? [],
|
||||
'UNKNOWN'
|
||||
);
|
||||
|
||||
switch ($type) {
|
||||
case API_KEY_DYNAMIC:
|
||||
$jwtObj = new JWT(
|
||||
key: System::getEnv('_APP_OPENSSL_KEY_V1'),
|
||||
algo: 'HS256',
|
||||
maxAge: 86400,
|
||||
leeway: 0
|
||||
);
|
||||
|
||||
try {
|
||||
$payload = $jwtObj->decode($secret);
|
||||
} catch (JWTException) {
|
||||
$expired = true;
|
||||
}
|
||||
|
||||
$name = $payload['name'] ?? 'Dynamic Key';
|
||||
$projectId = $payload['projectId'] ?? '';
|
||||
$disabledMetrics = $payload['disabledMetrics'] ?? [];
|
||||
$scopes = \array_merge($payload['scopes'] ?? [], $scopes);
|
||||
|
||||
if ($projectId !== $project->getId()) {
|
||||
return $guestKey;
|
||||
}
|
||||
|
||||
return new Key(
|
||||
$projectId,
|
||||
$type,
|
||||
$role,
|
||||
$scopes,
|
||||
$name,
|
||||
$expired,
|
||||
$disabledMetrics
|
||||
);
|
||||
case API_KEY_STANDARD:
|
||||
$key = $project->find(
|
||||
key: 'secret',
|
||||
find: $key,
|
||||
subject: 'keys'
|
||||
);
|
||||
|
||||
if (!$key) {
|
||||
return $guestKey;
|
||||
}
|
||||
|
||||
$expire = $key->getAttribute('expire');
|
||||
if (!empty($expire) && $expire < DateTime::formatTz(DateTime::now())) {
|
||||
$expired = true;
|
||||
}
|
||||
|
||||
$name = $key->getAttribute('name', 'UNKNOWN');
|
||||
$scopes = \array_merge($key->getAttribute('scopes', []), $scopes);
|
||||
|
||||
return new Key(
|
||||
$project->getId(),
|
||||
$type,
|
||||
$role,
|
||||
$scopes,
|
||||
$name,
|
||||
$expired
|
||||
);
|
||||
default:
|
||||
return $guestKey;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,10 +20,9 @@ class Slack extends OAuth2
|
||||
* @var array
|
||||
*/
|
||||
protected array $scopes = [
|
||||
'identity.avatar',
|
||||
'identity.basic',
|
||||
'identity.email',
|
||||
'identity.team'
|
||||
'openid',
|
||||
'email',
|
||||
'profile'
|
||||
];
|
||||
|
||||
/**
|
||||
@@ -35,14 +34,15 @@ class Slack extends OAuth2
|
||||
}
|
||||
|
||||
/**
|
||||
* @link https://api.slack.com/authentication/oauth-v2
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getLoginURL(): string
|
||||
{
|
||||
// https://api.slack.com/docs/oauth#step_1_-_sending_users_to_authorize_and_or_install
|
||||
return 'https://slack.com/oauth/authorize?' . \http_build_query([
|
||||
return 'https://slack.com/oauth/v2/authorize?' . \http_build_query([
|
||||
'client_id' => $this->appID,
|
||||
'scope' => \implode(' ', $this->getScopes()),
|
||||
'user_scope' => \implode(' ', $this->getScopes()),
|
||||
'redirect_uri' => $this->callback,
|
||||
'state' => \json_encode($this->state)
|
||||
]);
|
||||
@@ -56,16 +56,15 @@ class Slack extends OAuth2
|
||||
protected function getTokens(string $code): array
|
||||
{
|
||||
if (empty($this->tokens)) {
|
||||
// https://api.slack.com/docs/oauth#step_3_-_exchanging_a_verification_code_for_an_access_token
|
||||
$this->tokens = \json_decode($this->request(
|
||||
'GET',
|
||||
'https://slack.com/api/oauth.access?' . \http_build_query([
|
||||
'https://slack.com/api/oauth.v2.access?' . \http_build_query([
|
||||
'client_id' => $this->appID,
|
||||
'client_secret' => $this->appSecret,
|
||||
'code' => $code,
|
||||
'redirect_uri' => $this->callback
|
||||
])
|
||||
), true);
|
||||
), true)['authed_user'] ?? [];
|
||||
}
|
||||
|
||||
return $this->tokens;
|
||||
@@ -80,13 +79,13 @@ class Slack extends OAuth2
|
||||
{
|
||||
$this->tokens = \json_decode($this->request(
|
||||
'GET',
|
||||
'https://slack.com/api/oauth.access?' . \http_build_query([
|
||||
'https://slack.com/api/oauth.v2.access?' . \http_build_query([
|
||||
'client_id' => $this->appID,
|
||||
'client_secret' => $this->appSecret,
|
||||
'refresh_token' => $refreshToken,
|
||||
'grant_type' => 'refresh_token'
|
||||
])
|
||||
), true);
|
||||
), true)['authed_user'] ?? [];
|
||||
|
||||
if (empty($this->tokens['refresh_token'])) {
|
||||
$this->tokens['refresh_token'] = $refreshToken;
|
||||
@@ -161,9 +160,9 @@ class Slack extends OAuth2
|
||||
if (empty($this->user)) {
|
||||
$user = $this->request(
|
||||
'GET',
|
||||
'https://slack.com/api/users.identity?token=' . \urlencode($accessToken)
|
||||
'https://slack.com/api/users.identity',
|
||||
['Authorization: Bearer ' . \urlencode($accessToken)]
|
||||
);
|
||||
|
||||
$this->user = \json_decode($user, true);
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class Audit extends Event
|
||||
{
|
||||
@@ -12,9 +12,9 @@ class Audit extends Event
|
||||
protected string $ip = '';
|
||||
protected string $hostname = '';
|
||||
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($connection);
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::AUDITS_QUEUE_NAME)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class Build extends Event
|
||||
{
|
||||
@@ -12,9 +12,9 @@ class Build extends Event
|
||||
protected ?Document $deployment = null;
|
||||
protected ?Document $template = null;
|
||||
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($connection);
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::BUILDS_QUEUE_NAME)
|
||||
|
||||
@@ -3,16 +3,16 @@
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class Certificate extends Event
|
||||
{
|
||||
protected bool $skipRenewCheck = false;
|
||||
protected ?Document $domain = null;
|
||||
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($connection);
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::CERTIFICATES_QUEUE_NAME)
|
||||
|
||||
@@ -4,7 +4,7 @@ namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\DSN\DSN;
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class Database extends Event
|
||||
{
|
||||
@@ -13,9 +13,9 @@ class Database extends Event
|
||||
protected ?Document $collection = null;
|
||||
protected ?Document $document = null;
|
||||
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($connection);
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this->setClass(Event::DATABASE_CLASS_NAME);
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class Delete extends Event
|
||||
{
|
||||
@@ -15,9 +15,9 @@ class Delete extends Event
|
||||
protected ?string $hourlyUsageRetentionDatetime = null;
|
||||
|
||||
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($connection);
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::DELETE_QUEUE_NAME)
|
||||
|
||||
@@ -4,8 +4,8 @@ namespace Appwrite\Event;
|
||||
|
||||
use InvalidArgumentException;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Queue\Client;
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
use Utopia\Queue\Queue;
|
||||
|
||||
class Event
|
||||
{
|
||||
@@ -24,11 +24,22 @@ class Event
|
||||
public const FUNCTIONS_QUEUE_NAME = 'v1-functions';
|
||||
public const FUNCTIONS_CLASS_NAME = 'FunctionsV1';
|
||||
|
||||
/** remove */
|
||||
public const USAGE_QUEUE_NAME = 'v1-usage';
|
||||
public const USAGE_CLASS_NAME = 'UsageV1';
|
||||
|
||||
public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump';
|
||||
public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1';
|
||||
/** /remove */
|
||||
|
||||
public const STATS_RESOURCES_QUEUE_NAME = 'v1-stats-resources';
|
||||
public const STATS_RESOURCES_CLASS_NAME = 'StatsResourcesV1';
|
||||
|
||||
public const STATS_USAGE_QUEUE_NAME = 'v1-stats-usage';
|
||||
public const STATS_USAGE_CLASS_NAME = 'StatsUsageV1';
|
||||
|
||||
public const STATS_USAGE_DUMP_QUEUE_NAME = 'v1-stats-usage-dump';
|
||||
public const STATS_USAGE_DUMP_CLASS_NAME = 'StatsUsageDumpV1';
|
||||
|
||||
public const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
|
||||
public const WEBHOOK_CLASS_NAME = 'WebhooksV1';
|
||||
@@ -58,10 +69,10 @@ class Event
|
||||
protected bool $paused = false;
|
||||
|
||||
/**
|
||||
* @param Connection $connection
|
||||
* @param Publisher $publisher
|
||||
* @return void
|
||||
*/
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -345,12 +356,11 @@ class Event
|
||||
}
|
||||
|
||||
/** The getter is required since events like Databases need to override the queue name depending on the project */
|
||||
$client = new Client($this->getQueue(), $this->connection);
|
||||
$queue = new Queue($this->getQueue());
|
||||
|
||||
// Merge the base payload with any trimmed values
|
||||
$payload = array_merge($this->preparePayload(), $this->trimPayload());
|
||||
|
||||
return $client->enqueue($payload);
|
||||
return $this->publisher->enqueue($queue, $payload);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class Func extends Event
|
||||
{
|
||||
@@ -19,9 +19,9 @@ class Func extends Event
|
||||
protected ?Document $function = null;
|
||||
protected ?Document $execution = null;
|
||||
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($connection);
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::FUNCTIONS_QUEUE_NAME)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class Mail extends Event
|
||||
{
|
||||
@@ -15,9 +15,9 @@ class Mail extends Event
|
||||
protected string $bodyTemplate = '';
|
||||
protected array $attachment = [];
|
||||
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($connection);
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::MAILS_QUEUE_NAME)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class Messaging extends Event
|
||||
{
|
||||
@@ -14,9 +14,9 @@ class Messaging extends Event
|
||||
protected ?string $scheduledAt = null;
|
||||
protected ?string $providerType = null;
|
||||
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($connection);
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::MESSAGING_QUEUE_NAME)
|
||||
|
||||
@@ -3,16 +3,16 @@
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class Migration extends Event
|
||||
{
|
||||
protected string $type = '';
|
||||
protected ?Document $migration = null;
|
||||
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($connection);
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::MIGRATIONS_QUEUE_NAME)
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class StatsResources extends Event
|
||||
{
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::STATS_RESOURCES_QUEUE_NAME)
|
||||
->setClass(Event::STATS_RESOURCES_CLASS_NAME);
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare the payload for the usage event.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
protected function preparePayload(): array
|
||||
{
|
||||
return [
|
||||
'project' => $this->project
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class StatsUsage extends Event
|
||||
{
|
||||
protected array $metrics = [];
|
||||
protected array $reduce = [];
|
||||
protected array $disabled = [];
|
||||
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::STATS_USAGE_QUEUE_NAME)
|
||||
->setClass(Event::STATS_USAGE_CLASS_NAME);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add reduce.
|
||||
*
|
||||
* @param Document $document
|
||||
* @return self
|
||||
*/
|
||||
public function addReduce(Document $document): self
|
||||
{
|
||||
$this->reduce[] = $document;
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add metric.
|
||||
*
|
||||
* @param string $key
|
||||
* @param int $value
|
||||
* @return self
|
||||
*/
|
||||
public function addMetric(string $key, int $value): self
|
||||
{
|
||||
$this->metrics[] = [
|
||||
'key' => $key,
|
||||
'value' => $value,
|
||||
];
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set disabled metrics.
|
||||
*
|
||||
* @param string $key
|
||||
* @return self
|
||||
*/
|
||||
public function disableMetric(string $key): self
|
||||
{
|
||||
$this->disabled[] = $key;
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare the payload for the event
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
protected function preparePayload(): array
|
||||
{
|
||||
return [
|
||||
'project' => $this->getProject(),
|
||||
'reduce' => $this->reduce,
|
||||
'metrics' => \array_filter($this->metrics, function ($metric) {
|
||||
foreach ($this->disabled as $disabledMetric) {
|
||||
if (\str_ends_with($metric['key'], $disabledMetric)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}),
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class StatsUsageDump extends Event
|
||||
{
|
||||
protected array $stats;
|
||||
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::STATS_USAGE_DUMP_QUEUE_NAME)
|
||||
->setClass(Event::STATS_USAGE_DUMP_CLASS_NAME);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add Stats.
|
||||
*
|
||||
* @param array $stats
|
||||
* @return self
|
||||
*/
|
||||
public function setStats(array $stats): self
|
||||
{
|
||||
$this->stats = $stats;
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare the payload for the usage dump event.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
protected function preparePayload(): array
|
||||
{
|
||||
return [
|
||||
'stats' => $this->stats,
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -3,16 +3,16 @@
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class Usage extends Event
|
||||
{
|
||||
protected array $metrics = [];
|
||||
protected array $reduce = [];
|
||||
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($connection);
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::USAGE_QUEUE_NAME)
|
||||
|
||||
@@ -2,15 +2,15 @@
|
||||
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class UsageDump extends Event
|
||||
{
|
||||
protected array $stats;
|
||||
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($connection);
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::USAGE_DUMP_QUEUE_NAME)
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
|
||||
namespace Appwrite\Event;
|
||||
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
|
||||
class Webhook extends Event
|
||||
{
|
||||
public function __construct(protected Connection $connection)
|
||||
public function __construct(protected Publisher $publisher)
|
||||
{
|
||||
parent::__construct($connection);
|
||||
parent::__construct($publisher);
|
||||
|
||||
$this
|
||||
->setQueue(Event::WEBHOOK_QUEUE_NAME)
|
||||
|
||||
@@ -92,6 +92,7 @@ abstract class Migration
|
||||
'1.5.11' => 'V20',
|
||||
'1.6.0' => 'V21',
|
||||
'1.6.1' => 'V21',
|
||||
'1.6.2' => 'V22',
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Migration\Version;
|
||||
|
||||
use Appwrite\Migration\Migration;
|
||||
use Exception;
|
||||
use Throwable;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\Database;
|
||||
|
||||
class V22 extends Migration
|
||||
{
|
||||
/**
|
||||
* @throws Throwable
|
||||
*/
|
||||
public function execute(): void
|
||||
{
|
||||
/**
|
||||
* Disable SubQueries for Performance.
|
||||
*/
|
||||
foreach (['subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships', 'subQueryVariables', 'subQueryChallenges', 'subQueryProjectVariables', 'subQueryTargets', 'subQueryTopicTargets'] as $name) {
|
||||
Database::addFilter(
|
||||
$name,
|
||||
fn () => null,
|
||||
fn () => []
|
||||
);
|
||||
}
|
||||
|
||||
Console::info('Migrating Collections');
|
||||
$this->migrateCollections();
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate Collections.
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception|Throwable
|
||||
*/
|
||||
private function migrateCollections(): void
|
||||
{
|
||||
$internalProjectId = $this->project->getInternalId();
|
||||
$collectionType = match ($internalProjectId) {
|
||||
'console' => 'console',
|
||||
default => 'projects',
|
||||
};
|
||||
|
||||
$collections = $this->collections[$collectionType];
|
||||
foreach ($collections as $collection) {
|
||||
$id = $collection['$id'];
|
||||
|
||||
Console::log("Migrating Collection \"{$id}\"");
|
||||
|
||||
$this->projectDB->setNamespace("_$internalProjectId");
|
||||
|
||||
switch ($id) {
|
||||
case 'installations':
|
||||
// Create personalAccessToken attribute
|
||||
try {
|
||||
$this->createAttributeFromCollection($this->projectDB, $id, 'personalAccessToken');
|
||||
} catch (Throwable $th) {
|
||||
Console::warning("'personalAccessToken' from {$id}: {$th->getMessage()}");
|
||||
}
|
||||
|
||||
// Create personalAccessTokenExpiry attribute
|
||||
try {
|
||||
$this->createAttributeFromCollection($this->projectDB, $id, 'personalAccessTokenExpiry');
|
||||
} catch (Throwable $th) {
|
||||
Console::warning("'personalAccessTokenExpiry' from {$id}: {$th->getMessage()}");
|
||||
}
|
||||
|
||||
// Create personalRefreshToken attribute
|
||||
try {
|
||||
$this->createAttributeFromCollection($this->projectDB, $id, 'personalRefreshToken');
|
||||
} catch (Throwable $th) {
|
||||
Console::warning("'personalRefreshToken' from {$id}: {$th->getMessage()}");
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
usleep(50000);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform;
|
||||
|
||||
use Swoole\Coroutine as Co;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Platform\Action as UtopiaAction;
|
||||
|
||||
class Action extends UtopiaAction
|
||||
{
|
||||
/**
|
||||
* Log Error Callback
|
||||
*
|
||||
* @var callable
|
||||
*/
|
||||
protected mixed $logError;
|
||||
|
||||
/**
|
||||
* Foreach Document
|
||||
* Call provided callback for each document in the collection
|
||||
*
|
||||
* @param string $projectId
|
||||
* @param string $collection
|
||||
* @param array $queries
|
||||
* @param callable $callback
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null, int $limit = 1000, bool $concurrent = false): void
|
||||
{
|
||||
$results = [];
|
||||
$sum = $limit;
|
||||
$latestDocument = null;
|
||||
|
||||
while ($sum === $limit) {
|
||||
$newQueries = $queries;
|
||||
try {
|
||||
if ($latestDocument !== null) {
|
||||
array_unshift($newQueries, Query::cursorAfter($latestDocument));
|
||||
}
|
||||
$newQueries[] = Query::limit($limit);
|
||||
$database->disableValidation();
|
||||
$results = $database->find($collection, $newQueries);
|
||||
$database->enableValidation();
|
||||
} catch (\Exception $e) {
|
||||
if (!empty($this->logError)) {
|
||||
call_user_func_array($this->logError, [$e, "CLI", "fetch_documents_namespace_{$database->getNamespace()}_collection{$collection}"]);
|
||||
}
|
||||
}
|
||||
|
||||
if (empty($results)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$sum = count($results);
|
||||
|
||||
if ($concurrent) {
|
||||
$callables = [];
|
||||
$errors = [];
|
||||
|
||||
foreach ($results as $document) {
|
||||
if (is_callable($callback)) {
|
||||
$callables[] = Co\go(function () use ($document, $callback, &$errors) {
|
||||
try {
|
||||
$callback($document);
|
||||
} catch (\Throwable $error) {
|
||||
$errors[] = $error;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Co::join($callables);
|
||||
|
||||
if (!empty($errors)) {
|
||||
throw new \Error("Errors found in concurrent foreachDocument: " . \json_encode($errors));
|
||||
}
|
||||
} else {
|
||||
foreach ($results as $document) {
|
||||
if (is_callable($callback)) {
|
||||
$callback($document);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$latestDocument = $results[array_key_last($results)];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,6 @@ use Appwrite\Platform\Tasks\Doctor;
|
||||
use Appwrite\Platform\Tasks\Install;
|
||||
use Appwrite\Platform\Tasks\Maintenance;
|
||||
use Appwrite\Platform\Tasks\Migrate;
|
||||
use Appwrite\Platform\Tasks\QueueCount;
|
||||
use Appwrite\Platform\Tasks\QueueRetry;
|
||||
use Appwrite\Platform\Tasks\ScheduleExecutions;
|
||||
use Appwrite\Platform\Tasks\ScheduleFunctions;
|
||||
@@ -14,6 +13,7 @@ use Appwrite\Platform\Tasks\ScheduleMessages;
|
||||
use Appwrite\Platform\Tasks\SDKs;
|
||||
use Appwrite\Platform\Tasks\Specs;
|
||||
use Appwrite\Platform\Tasks\SSL;
|
||||
use Appwrite\Platform\Tasks\StatsResources;
|
||||
use Appwrite\Platform\Tasks\Upgrade;
|
||||
use Appwrite\Platform\Tasks\Vars;
|
||||
use Appwrite\Platform\Tasks\Version;
|
||||
@@ -29,7 +29,6 @@ class Tasks extends Service
|
||||
->addAction(Install::getName(), new Install())
|
||||
->addAction(Maintenance::getName(), new Maintenance())
|
||||
->addAction(Migrate::getName(), new Migrate())
|
||||
->addAction(QueueCount::getName(), new QueueCount())
|
||||
->addAction(QueueRetry::getName(), new QueueRetry())
|
||||
->addAction(SDKs::getName(), new SDKs())
|
||||
->addAction(SSL::getName(), new SSL())
|
||||
@@ -40,6 +39,7 @@ class Tasks extends Service
|
||||
->addAction(Upgrade::getName(), new Upgrade())
|
||||
->addAction(Vars::getName(), new Vars())
|
||||
->addAction(Version::getName(), new Version())
|
||||
->addAction(StatsResources::getName(), new StatsResources())
|
||||
;
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,13 @@ use Appwrite\Platform\Workers\Functions;
|
||||
use Appwrite\Platform\Workers\Mails;
|
||||
use Appwrite\Platform\Workers\Messaging;
|
||||
use Appwrite\Platform\Workers\Migrations;
|
||||
use Appwrite\Platform\Workers\StatsResources;
|
||||
use Appwrite\Platform\Workers\StatsUsage;
|
||||
use Appwrite\Platform\Workers\StatsUsageDump;
|
||||
/** remove */
|
||||
use Appwrite\Platform\Workers\Usage;
|
||||
use Appwrite\Platform\Workers\UsageDump;
|
||||
/** /remove */
|
||||
use Appwrite\Platform\Workers\Webhooks;
|
||||
use Utopia\Platform\Service;
|
||||
|
||||
@@ -31,10 +36,14 @@ class Workers extends Service
|
||||
->addAction(Mails::getName(), new Mails())
|
||||
->addAction(Messaging::getName(), new Messaging())
|
||||
->addAction(Webhooks::getName(), new Webhooks())
|
||||
->addAction(StatsUsageDump::getName(), new StatsUsageDump())
|
||||
->addAction(StatsUsage::getName(), new StatsUsage())
|
||||
->addAction(Migrations::getName(), new Migrations())
|
||||
->addAction(StatsResources::getName(), new StatsResources())
|
||||
/** Remove */
|
||||
->addAction(UsageDump::getName(), new UsageDump())
|
||||
->addAction(Usage::getName(), new Usage())
|
||||
->addAction(Migrations::getName(), new Migrations())
|
||||
|
||||
/** /remove */
|
||||
;
|
||||
}
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Tasks;
|
||||
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Queue\Client;
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Validator\Text;
|
||||
use Utopia\Validator\WhiteList;
|
||||
|
||||
class QueueCount extends Action
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'queue-count';
|
||||
}
|
||||
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->desc('Return the number of from a specific queue identified by the name parameter with a specific type')
|
||||
->param('name', '', new Text(100), 'Queue name')
|
||||
->param('type', '', new WhiteList([
|
||||
'success',
|
||||
'failed',
|
||||
'processing',
|
||||
]), 'Queue type')
|
||||
->inject('queue')
|
||||
->callback(fn ($name, $type, $queue) => $this->action($name, $type, $queue));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $name The name of the queue to count the jobs from
|
||||
* @param string $type The type of jobs to count
|
||||
* @param Connection $queue
|
||||
*/
|
||||
public function action(string $name, string $type, Connection $queue): void
|
||||
{
|
||||
if (!$name) {
|
||||
Console::error('Missing required parameter $name');
|
||||
return;
|
||||
}
|
||||
|
||||
$queueClient = new Client($name, $queue);
|
||||
|
||||
$count = match ($type) {
|
||||
'success' => $queueClient->countSuccessfulJobs(),
|
||||
'failed' => $queueClient->countFailedJobs(),
|
||||
'processing' => $queueClient->countProcessingJobs(),
|
||||
default => 0
|
||||
};
|
||||
|
||||
Console::log("Queue: '{$name}' has {$count} {$type} jobs.");
|
||||
}
|
||||
}
|
||||
@@ -4,8 +4,8 @@ namespace Appwrite\Platform\Tasks;
|
||||
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Queue\Client;
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Queue\Publisher;
|
||||
use Utopia\Queue\Queue;
|
||||
use Utopia\Validator\Text;
|
||||
use Utopia\Validator\Wildcard;
|
||||
|
||||
@@ -23,33 +23,24 @@ class QueueRetry extends Action
|
||||
->desc('Retry failed jobs from a specific queue identified by the name parameter')
|
||||
->param('name', '', new Text(100), 'Queue name')
|
||||
->param('limit', 0, new Wildcard(), 'jobs limit', true)
|
||||
->inject('queue')
|
||||
->callback(fn ($name, $limit, $queue) => $this->action($name, $limit, $queue));
|
||||
->inject('publisher')
|
||||
->callback(fn ($name, $limit, $publisher) => $this->action($name, $limit, $publisher));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $name The name of the queue to retry jobs from
|
||||
* @param mixed $limit
|
||||
* @param Connection $queue
|
||||
* @param Publisher $publisher
|
||||
*/
|
||||
public function action(string $name, mixed $limit, Connection $queue): void
|
||||
public function action(string $name, mixed $limit, Publisher $publisher): void
|
||||
{
|
||||
|
||||
if (!$name) {
|
||||
Console::error('Missing required parameter $name');
|
||||
return;
|
||||
}
|
||||
|
||||
$limit = (int)$limit;
|
||||
$queueClient = new Client($name, $queue);
|
||||
|
||||
if ($queueClient->countFailedJobs() === 0) {
|
||||
Console::error('No failed jobs found.');
|
||||
return;
|
||||
}
|
||||
|
||||
Console::log('Retrying failed jobs...');
|
||||
|
||||
$queueClient->retry($limit);
|
||||
$publisher->retry(new Queue($name), $limit);
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,7 @@ class ScheduleExecutions extends ScheduleBase
|
||||
|
||||
protected function enqueueResources(Group $pools, Database $dbForPlatform, callable $getProjectDB): void
|
||||
{
|
||||
$queue = $pools->get('queue')->pop();
|
||||
$queue = $pools->get('publisher')->pop();
|
||||
$connection = $queue->getResource();
|
||||
$queueForFunctions = new Func($connection);
|
||||
$intervalEnd = (new \DateTime())->modify('+' . self::ENQUEUE_TIMER . ' seconds');
|
||||
|
||||
@@ -73,7 +73,7 @@ class ScheduleFunctions extends ScheduleBase
|
||||
\go(function () use ($delay, $scheduleKeys, $pools, $dbForPlatform) {
|
||||
\sleep($delay); // in seconds
|
||||
|
||||
$queue = $pools->get('queue')->pop();
|
||||
$queue = $pools->get('publisher')->pop();
|
||||
$connection = $queue->getResource();
|
||||
|
||||
foreach ($scheduleKeys as $scheduleKey) {
|
||||
|
||||
@@ -41,7 +41,7 @@ class ScheduleMessages extends ScheduleBase
|
||||
}
|
||||
|
||||
\go(function () use ($schedule, $pools, $dbForPlatform) {
|
||||
$queue = $pools->get('queue')->pop();
|
||||
$queue = $pools->get('publisher')->pop();
|
||||
$connection = $queue->getResource();
|
||||
$queueForMessaging = new Messaging($connection);
|
||||
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Tasks;
|
||||
|
||||
use Appwrite\Event\StatsResources as EventStatsResources;
|
||||
use Appwrite\Platform\Action;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\System\System;
|
||||
|
||||
/**
|
||||
* Usage count
|
||||
*
|
||||
* Runs every hour, schedules project
|
||||
* for aggregating resource count
|
||||
*/
|
||||
class StatsResources extends Action
|
||||
{
|
||||
/**
|
||||
* Log Error Callback
|
||||
*
|
||||
* @var callable
|
||||
*/
|
||||
protected mixed $logError;
|
||||
|
||||
/**
|
||||
* Console DB
|
||||
*
|
||||
* @var Database
|
||||
*/
|
||||
protected Database $dbForPlatform;
|
||||
|
||||
public static function getName()
|
||||
{
|
||||
return 'stats-resources';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->desc('Schedules projects for usage count')
|
||||
->inject('dbForPlatform')
|
||||
->inject('logError')
|
||||
->inject('queueForStatsResources')
|
||||
->callback([$this, 'action']);
|
||||
}
|
||||
|
||||
public function action(Database $dbForPlatform, callable $logError, EventStatsResources $queue): void
|
||||
{
|
||||
$this->logError = $logError;
|
||||
$this->dbForPlatform = $dbForPlatform;
|
||||
|
||||
Console::title("Stats resources V1");
|
||||
|
||||
Console::success('Stats resources: started');
|
||||
|
||||
$interval = (int) System::getEnv('_APP_STATS_RESOURCES_INTERVAL', '3600');
|
||||
Console::loop(function () use ($queue) {
|
||||
Authorization::disable();
|
||||
Authorization::setDefaultStatus(false);
|
||||
|
||||
$last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours'));
|
||||
/**
|
||||
* For each project that were accessed in last 24 hours
|
||||
*/
|
||||
$this->foreachDocument($this->dbForPlatform, 'projects', [
|
||||
Query::greaterThanEqual('accessedAt', DateTime::format($last24Hours))
|
||||
], function ($project) use ($queue) {
|
||||
$queue
|
||||
->setProject($project)
|
||||
->trigger();
|
||||
Console::success('project: ' . $project->getId() . '(' . $project->getInternalId() . ')' . ' queued');
|
||||
});
|
||||
}, $interval);
|
||||
|
||||
Console::log("Stats resources: exited");
|
||||
}
|
||||
}
|
||||
@@ -6,15 +6,33 @@ use Appwrite\Auth\Auth;
|
||||
use Exception;
|
||||
use Throwable;
|
||||
use Utopia\Audit\Audit;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Exception\Authorization;
|
||||
use Utopia\Database\Exception\Structure;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Queue\Message;
|
||||
use Utopia\System\System;
|
||||
|
||||
class Audits extends Action
|
||||
{
|
||||
private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development
|
||||
private const BATCH_SIZE_PRODUCTION = 5_000;
|
||||
private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds
|
||||
|
||||
private int $lastTriggeredTime = 0;
|
||||
|
||||
private array $logs = [];
|
||||
|
||||
private function getBatchSize(): int
|
||||
{
|
||||
return System::getEnv('_APP_ENV', 'development') === 'development'
|
||||
? self::BATCH_SIZE_DEVELOPMENT
|
||||
: self::BATCH_SIZE_PRODUCTION;
|
||||
}
|
||||
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'audits';
|
||||
@@ -30,6 +48,8 @@ class Audits extends Action
|
||||
->inject('message')
|
||||
->inject('dbForProject')
|
||||
->callback(fn ($message, $dbForProject) => $this->action($message, $dbForProject));
|
||||
|
||||
$this->lastTriggeredTime = time();
|
||||
}
|
||||
|
||||
|
||||
@@ -44,13 +64,14 @@ class Audits extends Action
|
||||
*/
|
||||
public function action(Message $message, Database $dbForProject): void
|
||||
{
|
||||
|
||||
$payload = $message->getPayload() ?? [];
|
||||
|
||||
if (empty($payload)) {
|
||||
throw new Exception('Missing payload');
|
||||
}
|
||||
|
||||
Console::info('Aggregating audit logs');
|
||||
|
||||
$event = $payload['event'] ?? '';
|
||||
$auditPayload = $payload['payload'] ?? '';
|
||||
$mode = $payload['mode'] ?? '';
|
||||
@@ -63,23 +84,48 @@ class Audits extends Action
|
||||
$userEmail = $user->getAttribute('email', '');
|
||||
$userType = $user->getAttribute('type', Auth::ACTIVITY_TYPE_USER);
|
||||
|
||||
$audit = new Audit($dbForProject);
|
||||
$audit->log(
|
||||
userId: $user->getInternalId(),
|
||||
// Pass first, most verbose event pattern
|
||||
event: $event,
|
||||
resource: $resource,
|
||||
userAgent: $userAgent,
|
||||
ip: $ip,
|
||||
location: '',
|
||||
data: [
|
||||
// Create event data
|
||||
$eventData = [
|
||||
'userId' => $user->getInternalId(),
|
||||
'event' => $event,
|
||||
'resource' => $resource,
|
||||
'userAgent' => $userAgent,
|
||||
'ip' => $ip,
|
||||
'location' => '',
|
||||
'data' => [
|
||||
'userId' => $user->getId(),
|
||||
'userName' => $userName,
|
||||
'userEmail' => $userEmail,
|
||||
'userType' => $userType,
|
||||
'mode' => $mode,
|
||||
'data' => $auditPayload,
|
||||
]
|
||||
);
|
||||
],
|
||||
'timestamp' => DateTime::formatTz(DateTime::now())
|
||||
];
|
||||
|
||||
$this->logs[] = $eventData;
|
||||
|
||||
// Check if we should process the batch by checking both for the batch size and the elapsed time
|
||||
$batchSize = $this->getBatchSize();
|
||||
$shouldProcessBatch = count($this->logs) >= $batchSize;
|
||||
if (!$shouldProcessBatch && count($this->logs) > 0) {
|
||||
$shouldProcessBatch = (time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL;
|
||||
}
|
||||
|
||||
if ($shouldProcessBatch) {
|
||||
Console::log('Processing batch with ' . count($this->logs) . ' events');
|
||||
$audit = new Audit($dbForProject);
|
||||
|
||||
try {
|
||||
$audit->logBatch($this->logs);
|
||||
Console::success('Audit logs processed successfully');
|
||||
} catch (Throwable $e) {
|
||||
Console::error('Error processing audit logs: ' . $e->getMessage());
|
||||
} finally {
|
||||
// Clear the pending events after successful batch processing
|
||||
$this->logs = [];
|
||||
$this->lastTriggeredTime = time();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,7 @@ namespace Appwrite\Platform\Workers;
|
||||
use Ahc\Jwt\JWT;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Func;
|
||||
use Appwrite\Event\Usage;
|
||||
use Appwrite\Event\StatsUsage;
|
||||
use Appwrite\Messaging\Adapter\Realtime;
|
||||
use Appwrite\Utopia\Response\Model\Deployment;
|
||||
use Appwrite\Vcs\Comment;
|
||||
@@ -50,12 +50,14 @@ class Builds extends Action
|
||||
->inject('dbForPlatform')
|
||||
->inject('queueForEvents')
|
||||
->inject('queueForFunctions')
|
||||
->inject('queueForUsage')
|
||||
->inject('queueForStatsUsage')
|
||||
->inject('cache')
|
||||
->inject('dbForProject')
|
||||
->inject('deviceForFunctions')
|
||||
->inject('isResourceBlocked')
|
||||
->inject('log')
|
||||
->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log));
|
||||
->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) =>
|
||||
$this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -64,7 +66,7 @@ class Builds extends Action
|
||||
* @param Database $dbForPlatform
|
||||
* @param Event $queueForEvents
|
||||
* @param Func $queueForFunctions
|
||||
* @param Usage $queueForUsage
|
||||
* @param StatsUsage $queueForStatsUsage
|
||||
* @param Cache $cache
|
||||
* @param Database $dbForProject
|
||||
* @param Device $deviceForFunctions
|
||||
@@ -72,7 +74,7 @@ class Builds extends Action
|
||||
* @return void
|
||||
* @throws \Utopia\Database\Exception
|
||||
*/
|
||||
public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void
|
||||
public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void
|
||||
{
|
||||
$payload = $message->getPayload() ?? [];
|
||||
|
||||
@@ -93,7 +95,7 @@ class Builds extends Action
|
||||
case BUILD_TYPE_RETRY:
|
||||
Console::info('Creating build for deployment: ' . $deployment->getId());
|
||||
$github = new GitHub($cache);
|
||||
$this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log);
|
||||
$this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log);
|
||||
break;
|
||||
|
||||
default:
|
||||
@@ -105,7 +107,7 @@ class Builds extends Action
|
||||
* @param Device $deviceForFunctions
|
||||
* @param Func $queueForFunctions
|
||||
* @param Event $queueForEvents
|
||||
* @param Usage $queueForUsage
|
||||
* @param StatsUsage $queueForStatsUsage
|
||||
* @param Database $dbForPlatform
|
||||
* @param Database $dbForProject
|
||||
* @param GitHub $github
|
||||
@@ -118,7 +120,7 @@ class Builds extends Action
|
||||
* @throws \Utopia\Database\Exception
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void
|
||||
protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void
|
||||
{
|
||||
$executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
|
||||
|
||||
@@ -127,7 +129,11 @@ class Builds extends Action
|
||||
|
||||
$function = $dbForProject->getDocument('functions', $functionId);
|
||||
if ($function->isEmpty()) {
|
||||
throw new \Exception('Function not found', 404);
|
||||
throw new \Exception('Function not found');
|
||||
}
|
||||
|
||||
if ($isResourceBlocked($project, RESOURCE_TYPE_FUNCTIONS, $functionId)) {
|
||||
throw new \Exception('Function blocked');
|
||||
}
|
||||
|
||||
$deploymentId = $deployment->getId();
|
||||
@@ -135,15 +141,15 @@ class Builds extends Action
|
||||
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
if ($deployment->isEmpty()) {
|
||||
throw new \Exception('Deployment not found', 404);
|
||||
throw new \Exception('Deployment not found');
|
||||
}
|
||||
|
||||
if (empty($deployment->getAttribute('entrypoint', ''))) {
|
||||
throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500);
|
||||
throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".');
|
||||
}
|
||||
|
||||
$version = $function->getAttribute('version', 'v2');
|
||||
$spec = Config::getParam('runtime-specifications')[$function->getAttribute('specifications', APP_FUNCTION_SPECIFICATION_DEFAULT)];
|
||||
$spec = Config::getParam('runtime-specifications')[$function->getAttribute('specification', APP_FUNCTION_SPECIFICATION_DEFAULT)];
|
||||
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);
|
||||
$key = $function->getAttribute('runtime');
|
||||
$runtime = $runtimes[$key] ?? null;
|
||||
@@ -571,7 +577,7 @@ class Builds extends Action
|
||||
$build = $dbForProject->getDocument('builds', $build->getId());
|
||||
|
||||
if ($build->isEmpty()) {
|
||||
throw new \Exception('Build not found', 404);
|
||||
throw new \Exception('Build not found');
|
||||
}
|
||||
|
||||
if ($build->getAttribute('status') === 'canceled') {
|
||||
@@ -706,20 +712,20 @@ class Builds extends Action
|
||||
|
||||
/** Trigger usage queue */
|
||||
if ($build->getAttribute('status') === 'ready') {
|
||||
$queueForUsage
|
||||
$queueForStatsUsage
|
||||
->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000)
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000);
|
||||
} elseif ($build->getAttribute('status') === 'failed') {
|
||||
$queueForUsage
|
||||
$queueForStatsUsage
|
||||
->addMetric(METRIC_BUILDS_FAILED, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000)
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED), 1) // per function
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED), (int)$build->getAttribute('duration', 0) * 1000);
|
||||
}
|
||||
|
||||
$queueForUsage
|
||||
$queueForStatsUsage
|
||||
->addMetric(METRIC_BUILDS, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
|
||||
->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
|
||||
|
||||
@@ -370,6 +370,8 @@ class Certificates extends Action
|
||||
/** Trigger Webhook */
|
||||
$ruleModel = new Rule();
|
||||
$queueForEvents
|
||||
->setQueue(Event::WEBHOOK_QUEUE_NAME)
|
||||
->setClass(Event::WEBHOOK_CLASS_NAME)
|
||||
->setProject($project)
|
||||
->setEvent('rules.[ruleId].update')
|
||||
->setParam('ruleId', $rule->getId())
|
||||
|
||||
@@ -577,39 +577,25 @@ class Databases extends Action
|
||||
*/
|
||||
protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
|
||||
{
|
||||
$count = 0;
|
||||
$chunk = 0;
|
||||
$limit = 50;
|
||||
$sum = $limit;
|
||||
$start = \microtime(true);
|
||||
|
||||
$executionStart = \microtime(true);
|
||||
try {
|
||||
$documents = $database->deleteDocuments($collection, $queries);
|
||||
} catch (\Throwable $th) {
|
||||
Console::error('Failed to delete documents for collection ' . $collection . ': ' . $th->getMessage());
|
||||
return;
|
||||
}
|
||||
|
||||
while ($sum === $limit) {
|
||||
$chunk++;
|
||||
|
||||
$results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
|
||||
|
||||
$sum = count($results);
|
||||
|
||||
Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents');
|
||||
|
||||
foreach ($results as $document) {
|
||||
if ($database->deleteDocument($document->getCollection(), $document->getId())) {
|
||||
Console::success('Deleted document "' . $document->getId() . '" successfully');
|
||||
|
||||
if (\is_callable($callback)) {
|
||||
$callback($document);
|
||||
}
|
||||
} else {
|
||||
Console::warning('Failed to delete document: ' . $document->getId());
|
||||
}
|
||||
$count++;
|
||||
if (\is_callable($callback)) {
|
||||
foreach ($documents as $document) {
|
||||
$callback($document);
|
||||
}
|
||||
}
|
||||
|
||||
$executionEnd = \microtime(true);
|
||||
$end = \microtime(true);
|
||||
$count = \count($documents);
|
||||
|
||||
Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
|
||||
Console::info("Deleted {$count} documents by group in " . ($end - $start) . " seconds");
|
||||
}
|
||||
|
||||
protected function trigger(
|
||||
|
||||
@@ -7,6 +7,7 @@ use Appwrite\Certificates\Adapter as CertificatesAdapter;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Executor\Executor;
|
||||
use Throwable;
|
||||
use Utopia\Abuse\Adapters\TimeLimit\Database as AbuseDatabase;
|
||||
use Utopia\Audit\Audit;
|
||||
use Utopia\Cache\Adapter\Filesystem;
|
||||
use Utopia\Cache\Cache;
|
||||
@@ -505,7 +506,8 @@ class Deletes extends Action
|
||||
|
||||
$projectCollectionIds = [
|
||||
...\array_keys(Config::getParam('collections', [])['projects']),
|
||||
Audit::COLLECTION
|
||||
Audit::COLLECTION,
|
||||
AbuseDatabase::COLLECTION,
|
||||
];
|
||||
|
||||
$limit = \count($projectCollectionIds) + 25;
|
||||
@@ -927,65 +929,39 @@ class Deletes extends Action
|
||||
$this->deleteRuntimes($getProjectDB, $document, $project);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Document $document to be deleted
|
||||
* @param Database $database to delete it from
|
||||
* @param callable|null $callback to perform after document is deleted
|
||||
* @return void
|
||||
*/
|
||||
private function deleteById(Document $document, Database $database, callable $callback = null): void
|
||||
{
|
||||
if ($database->deleteDocument($document->getCollection(), $document->getId())) {
|
||||
Console::success('Deleted document "' . $document->getId() . '" successfully');
|
||||
|
||||
if (is_callable($callback)) {
|
||||
$callback($document);
|
||||
}
|
||||
} else {
|
||||
Console::error('Failed to delete document: ' . $document->getId());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $collection collectionID
|
||||
* @param array $queries
|
||||
* @param Database $database
|
||||
* @param callable|null $callback
|
||||
* @param ?callable $callback
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
|
||||
{
|
||||
$count = 0;
|
||||
$chunk = 0;
|
||||
$limit = 50;
|
||||
$sum = $limit;
|
||||
protected function deleteByGroup(
|
||||
string $collection,
|
||||
array $queries,
|
||||
Database $database,
|
||||
?callable $callback = null
|
||||
): void {
|
||||
$start = \microtime(true);
|
||||
|
||||
$executionStart = \microtime(true);
|
||||
try {
|
||||
$documents = $database->deleteDocuments($collection, $queries);
|
||||
} catch (\Throwable $th) {
|
||||
Console::error('Failed to delete documents for collection ' . $collection . ': ' . $th->getMessage());
|
||||
return;
|
||||
}
|
||||
|
||||
while ($sum === $limit) {
|
||||
$chunk++;
|
||||
|
||||
try {
|
||||
$results = $database->find($collection, [Query::limit($limit), ...$queries]);
|
||||
} catch (DatabaseException $e) {
|
||||
Console::error('Failed to find documents for collection ' . $collection . ': ' . $e->getMessage());
|
||||
return;
|
||||
}
|
||||
|
||||
$sum = count($results);
|
||||
|
||||
Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents');
|
||||
|
||||
foreach ($results as $document) {
|
||||
$this->deleteById($document, $database, $callback);
|
||||
$count++;
|
||||
if (\is_callable($callback)) {
|
||||
foreach ($documents as $document) {
|
||||
$callback($document);
|
||||
}
|
||||
}
|
||||
|
||||
$executionEnd = \microtime(true);
|
||||
$end = \microtime(true);
|
||||
$count = \count($documents);
|
||||
|
||||
Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
|
||||
Console::info("Deleted {$count} documents by group in " . ($end - $start) . " seconds");
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -999,25 +975,23 @@ class Deletes extends Action
|
||||
protected function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
|
||||
{
|
||||
$count = 0;
|
||||
$chunk = 0;
|
||||
$limit = 50;
|
||||
$results = [];
|
||||
$limit = 1000;
|
||||
$sum = $limit;
|
||||
$cursor = null;
|
||||
|
||||
$executionStart = \microtime(true);
|
||||
$start = \microtime(true);
|
||||
|
||||
while ($sum === $limit) {
|
||||
$chunk++;
|
||||
|
||||
$mergedQueries = \array_merge([Query::limit($limit)], $queries);
|
||||
if ($cursor instanceof Document) {
|
||||
$mergedQueries[] = Query::cursorAfter($cursor);
|
||||
$queries = \array_merge([Query::limit($limit)], $queries);
|
||||
|
||||
if ($cursor !== null) {
|
||||
$queries[] = Query::cursorAfter($cursor);
|
||||
}
|
||||
|
||||
$results = $database->find($collection, $mergedQueries);
|
||||
$results = $database->find($collection, $queries);
|
||||
|
||||
$sum = count($results);
|
||||
$sum = \count($results);
|
||||
|
||||
if ($sum > 0) {
|
||||
$cursor = $results[$sum - 1];
|
||||
@@ -1032,9 +1006,9 @@ class Deletes extends Action
|
||||
}
|
||||
}
|
||||
|
||||
$executionEnd = \microtime(true);
|
||||
$end = \microtime(true);
|
||||
|
||||
Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
|
||||
Console::info("Listed {$count} documents by group in " . ($end - $start) . " seconds");
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -5,7 +5,7 @@ namespace Appwrite\Platform\Workers;
|
||||
use Ahc\Jwt\JWT;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Func;
|
||||
use Appwrite\Event\Usage;
|
||||
use Appwrite\Event\StatsUsage;
|
||||
use Appwrite\Messaging\Adapter\Realtime;
|
||||
use Appwrite\Utopia\Response\Model\Execution;
|
||||
use Exception;
|
||||
@@ -46,13 +46,13 @@ class Functions extends Action
|
||||
->inject('dbForProject')
|
||||
->inject('queueForFunctions')
|
||||
->inject('queueForEvents')
|
||||
->inject('queueForUsage')
|
||||
->inject('queueForStatsUsage')
|
||||
->inject('log')
|
||||
->inject('isResourceBlocked')
|
||||
->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForUsage, $log, $isResourceBlocked));
|
||||
->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked));
|
||||
}
|
||||
|
||||
public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log, callable $isResourceBlocked): void
|
||||
public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void
|
||||
{
|
||||
$payload = $message->getPayload() ?? [];
|
||||
|
||||
@@ -137,7 +137,7 @@ class Functions extends Action
|
||||
log: $log,
|
||||
dbForProject: $dbForProject,
|
||||
queueForFunctions: $queueForFunctions,
|
||||
queueForUsage: $queueForUsage,
|
||||
queueForStatsUsage: $queueForStatsUsage,
|
||||
queueForEvents: $queueForEvents,
|
||||
project: $project,
|
||||
function: $function,
|
||||
@@ -177,7 +177,7 @@ class Functions extends Action
|
||||
log: $log,
|
||||
dbForProject: $dbForProject,
|
||||
queueForFunctions: $queueForFunctions,
|
||||
queueForUsage: $queueForUsage,
|
||||
queueForStatsUsage: $queueForStatsUsage,
|
||||
queueForEvents: $queueForEvents,
|
||||
project: $project,
|
||||
function: $function,
|
||||
@@ -199,7 +199,7 @@ class Functions extends Action
|
||||
log: $log,
|
||||
dbForProject: $dbForProject,
|
||||
queueForFunctions: $queueForFunctions,
|
||||
queueForUsage: $queueForUsage,
|
||||
queueForStatsUsage: $queueForStatsUsage,
|
||||
queueForEvents: $queueForEvents,
|
||||
project: $project,
|
||||
function: $function,
|
||||
@@ -284,7 +284,7 @@ class Functions extends Action
|
||||
* @param Log $log
|
||||
* @param Database $dbForProject
|
||||
* @param Func $queueForFunctions
|
||||
* @param Usage $queueForUsage
|
||||
* @param StatsUsage $queueForStatsUsage
|
||||
* @param Event $queueForEvents
|
||||
* @param Document $project
|
||||
* @param Document $function
|
||||
@@ -308,7 +308,7 @@ class Functions extends Action
|
||||
Log $log,
|
||||
Database $dbForProject,
|
||||
Func $queueForFunctions,
|
||||
Usage $queueForUsage,
|
||||
StatsUsage $queueForStatsUsage,
|
||||
Event $queueForEvents,
|
||||
Document $project,
|
||||
Document $function,
|
||||
@@ -552,7 +552,7 @@ class Functions extends Action
|
||||
$errorCode = $th->getCode();
|
||||
} finally {
|
||||
/** Trigger usage queue */
|
||||
$queueForUsage
|
||||
$queueForStatsUsage
|
||||
->setProject($project)
|
||||
->addMetric(METRIC_EXECUTIONS, 1)
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Workers;
|
||||
|
||||
use Appwrite\Event\Usage;
|
||||
use Appwrite\Event\StatsUsage;
|
||||
use Appwrite\Messaging\Status as MessageStatus;
|
||||
use Swoole\Runtime;
|
||||
use Utopia\CLI\Console;
|
||||
@@ -70,8 +70,8 @@ class Messaging extends Action
|
||||
->inject('log')
|
||||
->inject('dbForProject')
|
||||
->inject('deviceForFiles')
|
||||
->inject('queueForUsage')
|
||||
->callback(fn (Message $message, Document $project, Log $log, Database $dbForProject, Device $deviceForFiles, Usage $queueForUsage) => $this->action($message, $project, $log, $dbForProject, $deviceForFiles, $queueForUsage));
|
||||
->inject('queueForStatsUsage')
|
||||
->callback(fn (Message $message, Document $project, Log $log, Database $dbForProject, Device $deviceForFiles, StatsUsage $queueForStatsUsage) => $this->action($message, $project, $log, $dbForProject, $deviceForFiles, $queueForStatsUsage));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -80,7 +80,7 @@ class Messaging extends Action
|
||||
* @param Log $log
|
||||
* @param Database $dbForProject
|
||||
* @param Device $deviceForFiles
|
||||
* @param Usage $queueForUsage
|
||||
* @param StatsUsage $queueForStatsUsage
|
||||
* @return void
|
||||
* @throws \Exception
|
||||
*/
|
||||
@@ -90,7 +90,7 @@ class Messaging extends Action
|
||||
Log $log,
|
||||
Database $dbForProject,
|
||||
Device $deviceForFiles,
|
||||
Usage $queueForUsage
|
||||
StatsUsage $queueForStatsUsage
|
||||
): void {
|
||||
Runtime::setHookFlags(SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_TCP);
|
||||
$payload = $message->getPayload() ?? [];
|
||||
@@ -111,7 +111,7 @@ class Messaging extends Action
|
||||
case MESSAGE_SEND_TYPE_EXTERNAL:
|
||||
$message = $dbForProject->getDocument('messages', $payload['messageId']);
|
||||
|
||||
$this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $queueForUsage);
|
||||
$this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $queueForStatsUsage);
|
||||
break;
|
||||
default:
|
||||
throw new \Exception('Unknown message type: ' . $type);
|
||||
@@ -123,7 +123,7 @@ class Messaging extends Action
|
||||
Document $message,
|
||||
Device $deviceForFiles,
|
||||
Document $project,
|
||||
Usage $queueForUsage
|
||||
StatsUsage $queueForStatsUsage
|
||||
): void {
|
||||
$topicIds = $message->getAttribute('topics', []);
|
||||
$targetIds = $message->getAttribute('targets', []);
|
||||
@@ -229,8 +229,8 @@ class Messaging extends Action
|
||||
/**
|
||||
* @var array<array> $results
|
||||
*/
|
||||
$results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
|
||||
return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
|
||||
$results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) {
|
||||
return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) {
|
||||
if (\array_key_exists($providerId, $providers)) {
|
||||
$provider = $providers[$providerId];
|
||||
} else {
|
||||
@@ -257,8 +257,8 @@ class Messaging extends Action
|
||||
$adapter->getMaxMessagesPerRequest()
|
||||
);
|
||||
|
||||
return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
|
||||
return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
|
||||
return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) {
|
||||
return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) {
|
||||
$deliveredTotal = 0;
|
||||
$deliveryErrors = [];
|
||||
$messageData = clone $message;
|
||||
@@ -298,7 +298,7 @@ class Messaging extends Action
|
||||
$deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage();
|
||||
} finally {
|
||||
$errorTotal = \count($deliveryErrors);
|
||||
$queueForUsage
|
||||
$queueForStatsUsage
|
||||
->setProject($project)
|
||||
->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal))
|
||||
->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal)
|
||||
|
||||
@@ -188,28 +188,21 @@ class Migrations extends Action
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws \Utopia\Database\Exception
|
||||
* @throws Authorization
|
||||
* @throws Conflict
|
||||
* @throws Restricted
|
||||
* @throws Structure
|
||||
*/
|
||||
protected function removeAPIKey(Document $apiKey): void
|
||||
{
|
||||
$this->dbForPlatform->deleteDocument('keys', $apiKey->getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws Authorization
|
||||
* @throws Structure
|
||||
* @throws \Utopia\Database\Exception
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function generateAPIKey(Document $project): string
|
||||
{
|
||||
$jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 86400, 0);
|
||||
|
||||
$apiKey = $jwt->encode([
|
||||
'projectId' => $project->getId(),
|
||||
'disabledMetrics' => [
|
||||
METRIC_DATABASES_OPERATIONS_READS,
|
||||
METRIC_DATABASES_OPERATIONS_WRITES,
|
||||
METRIC_NETWORK_REQUESTS,
|
||||
METRIC_NETWORK_INBOUND,
|
||||
METRIC_NETWORK_OUTBOUND,
|
||||
],
|
||||
'scopes' => [
|
||||
'users.read',
|
||||
'users.write',
|
||||
@@ -222,12 +215,9 @@ class Migrations extends Action
|
||||
'functions.read',
|
||||
'functions.write',
|
||||
'databases.read',
|
||||
'databases.write',
|
||||
'collections.read',
|
||||
'collections.write',
|
||||
'documents.read',
|
||||
'documents.write'
|
||||
]
|
||||
],
|
||||
]);
|
||||
|
||||
return API_KEY_DYNAMIC . '_' . $apiKey;
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Workers;
|
||||
|
||||
use Appwrite\Platform\Action;
|
||||
use Exception;
|
||||
use Throwable;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Queue\Message;
|
||||
|
||||
class StatsResources extends Action
|
||||
{
|
||||
/**
|
||||
* Date format for different periods
|
||||
*/
|
||||
protected array $periods = [
|
||||
'1h' => 'Y-m-d H:00',
|
||||
'1d' => 'Y-m-d 00:00',
|
||||
'inf' => '0000-00-00 00:00'
|
||||
];
|
||||
|
||||
/**
|
||||
* @var array $documents
|
||||
*
|
||||
* Array of documents to batch write
|
||||
*
|
||||
*/
|
||||
protected array $documents = [];
|
||||
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'stats-resources';
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @throws Exception
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->desc('Stats resources worker')
|
||||
->inject('message')
|
||||
->inject('project')
|
||||
->inject('getProjectDB')
|
||||
->inject('getLogsDB')
|
||||
->inject('dbForPlatform')
|
||||
->inject('logError')
|
||||
->callback([$this, 'action']);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Message $message
|
||||
* @param Document $project
|
||||
* @param callable $getProjectDB
|
||||
* @return void
|
||||
* @throws \Utopia\Database\Exception
|
||||
* @throws Exception
|
||||
*/
|
||||
public function action(Message $message, Document $project, callable $getProjectDB, callable $getLogsDB, Database $dbForPlatform, callable $logError): void
|
||||
{
|
||||
$this->logError = $logError;
|
||||
|
||||
$payload = $message->getPayload() ?? [];
|
||||
if (empty($payload)) {
|
||||
throw new Exception('Missing payload');
|
||||
}
|
||||
|
||||
if (empty($project->getAttribute('database'))) {
|
||||
var_dump($payload);
|
||||
return;
|
||||
}
|
||||
|
||||
// Reset documents for each job
|
||||
$this->documents = [];
|
||||
|
||||
$this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project);
|
||||
}
|
||||
|
||||
|
||||
protected function countForProject(Database $dbForPlatform, callable $getLogsDB, callable $getProjectDB, Document $project): void
|
||||
{
|
||||
Console::info('Begining count for: ' . $project->getId());
|
||||
|
||||
$dbForLogs = null;
|
||||
$dbForProject = null;
|
||||
try {
|
||||
/** @var \Utopia\Database\Database $dbForLogs */
|
||||
$dbForLogs = call_user_func($getLogsDB, $project);
|
||||
/** @var \Utopia\Database\Database $dbForProject */
|
||||
$dbForProject = call_user_func($getProjectDB, $project);
|
||||
} catch (Throwable $th) {
|
||||
Console::error('Unable to get database');
|
||||
Console::error($th->getMessage());
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
$region = $project->getAttribute('region');
|
||||
|
||||
$platforms = $dbForPlatform->count('platforms', [
|
||||
Query::equal('projectInternalId', [$project->getInternalId()])
|
||||
]);
|
||||
$webhooks = $dbForPlatform->count('webhooks', [
|
||||
Query::equal('projectInternalId', [$project->getInternalId()])
|
||||
]);
|
||||
$keys = $dbForPlatform->count('keys', [
|
||||
Query::equal('projectInternalId', [$project->getInternalId()])
|
||||
]);
|
||||
$databases = $dbForProject->count('databases');
|
||||
$buckets = $dbForProject->count('buckets');
|
||||
$users = $dbForProject->count('users');
|
||||
|
||||
$last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00');
|
||||
$usersMAU = $dbForProject->count('users', [
|
||||
Query::greaterThanEqual('accessedAt', $last30Days)
|
||||
]);
|
||||
$last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours'))->format('Y-m-d h:m:00');
|
||||
$usersDAU = $dbForProject->count('users', [
|
||||
Query::greaterThanEqual('accessedAt', $last24Hours)
|
||||
]);
|
||||
$last7Days = (new \DateTime())->sub(\DateInterval::createFromDateString('7 days'))->format('Y-m-d 00:00:00');
|
||||
$usersWAU = $dbForProject->count('users', [
|
||||
Query::greaterThanEqual('accessedAt', $last7Days)
|
||||
]);
|
||||
$teams = $dbForProject->count('teams');
|
||||
$functions = $dbForProject->count('functions');
|
||||
$messages = $dbForProject->count('messages');
|
||||
$providers = $dbForProject->count('providers');
|
||||
$topics = $dbForProject->count('topics');
|
||||
|
||||
$metrics = [
|
||||
METRIC_DATABASES => $databases,
|
||||
METRIC_BUCKETS => $buckets,
|
||||
METRIC_USERS => $users,
|
||||
METRIC_FUNCTIONS => $functions,
|
||||
METRIC_TEAMS => $teams,
|
||||
METRIC_MESSAGES => $messages,
|
||||
METRIC_MAU => $usersMAU,
|
||||
METRIC_DAU => $usersDAU,
|
||||
METRIC_WAU => $usersWAU,
|
||||
METRIC_WEBHOOKS => $webhooks,
|
||||
METRIC_PLATFORMS => $platforms,
|
||||
METRIC_PROVIDERS => $providers,
|
||||
METRIC_TOPICS => $topics,
|
||||
METRIC_KEYS => $keys,
|
||||
];
|
||||
|
||||
foreach ($metrics as $metric => $value) {
|
||||
$this->createStatsDocuments($region, $metric, $value);
|
||||
}
|
||||
|
||||
try {
|
||||
$this->countForBuckets($dbForProject, $dbForLogs, $region);
|
||||
} catch (Throwable $th) {
|
||||
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]);
|
||||
}
|
||||
|
||||
try {
|
||||
$this->countImageTransformations($dbForProject, $dbForLogs, $region);
|
||||
} catch (Throwable $th) {
|
||||
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]);
|
||||
}
|
||||
|
||||
try {
|
||||
$this->countForDatabase($dbForProject, $dbForLogs, $region);
|
||||
} catch (Throwable $th) {
|
||||
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]);
|
||||
}
|
||||
|
||||
try {
|
||||
$this->countForFunctions($dbForProject, $dbForLogs, $region);
|
||||
} catch (Throwable $th) {
|
||||
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_functions_{$project->getId()}"]);
|
||||
}
|
||||
|
||||
$this->writeDocuments($dbForLogs, $project);
|
||||
} catch (Throwable $th) {
|
||||
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]);
|
||||
}
|
||||
|
||||
Console::info('End of count for: ' . $project->getId());
|
||||
}
|
||||
|
||||
protected function countForBuckets(Database $dbForProject, Database $dbForLogs, string $region)
|
||||
{
|
||||
$totalFiles = 0;
|
||||
$totalStorage = 0;
|
||||
$this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $dbForLogs, $region, &$totalFiles, &$totalStorage) {
|
||||
$files = $dbForProject->count('bucket_' . $bucket->getInternalId());
|
||||
|
||||
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES);
|
||||
$this->createStatsDocuments($region, $metric, $files);
|
||||
|
||||
$storage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeActual');
|
||||
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE);
|
||||
$this->createStatsDocuments($region, $metric, $storage);
|
||||
|
||||
$totalStorage += $storage;
|
||||
$totalFiles += $files;
|
||||
});
|
||||
|
||||
$this->createStatsDocuments($region, METRIC_FILES, $totalFiles);
|
||||
$this->createStatsDocuments($region, METRIC_FILES_STORAGE, $totalStorage);
|
||||
}
|
||||
|
||||
/**
|
||||
* Need separate function to count per period data
|
||||
*/
|
||||
protected function countImageTransformations(Database $dbForProject, Database $dbForLogs, string $region)
|
||||
{
|
||||
$totalImageTransformations = 0;
|
||||
$last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00');
|
||||
$this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $last30Days, $region, &$totalImageTransformations) {
|
||||
$imageTransformations = $dbForProject->count('bucket_' . $bucket->getInternalId(), [
|
||||
Query::greaterThanEqual('transformedAt', $last30Days),
|
||||
]);
|
||||
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED);
|
||||
$this->createStatsDocuments($region, $metric, $imageTransformations);
|
||||
$totalImageTransformations += $imageTransformations;
|
||||
});
|
||||
|
||||
$this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalImageTransformations);
|
||||
}
|
||||
|
||||
protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region)
|
||||
{
|
||||
$totalCollections = 0;
|
||||
$totalDocuments = 0;
|
||||
|
||||
$this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $dbForLogs, $region, &$totalCollections, &$totalDocuments) {
|
||||
$collections = $dbForProject->count('database_' . $database->getInternalId());
|
||||
|
||||
$metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS);
|
||||
$this->createStatsDocuments($region, $metric, $collections);
|
||||
|
||||
$documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region);
|
||||
|
||||
$totalDocuments += $documents;
|
||||
$totalCollections += $collections;
|
||||
});
|
||||
|
||||
$this->createStatsDocuments($region, METRIC_COLLECTIONS, $totalCollections);
|
||||
$this->createStatsDocuments($region, METRIC_DOCUMENTS, $totalDocuments);
|
||||
}
|
||||
protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int
|
||||
{
|
||||
$databaseDocuments = 0;
|
||||
$this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $dbForLogs, $database, $region, &$totalCollections, &$databaseDocuments) {
|
||||
$documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
|
||||
|
||||
$metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS);
|
||||
$this->createStatsDocuments($region, $metric, $documents);
|
||||
|
||||
$databaseDocuments += $documents;
|
||||
});
|
||||
|
||||
$metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS);
|
||||
$this->createStatsDocuments($region, $metric, $databaseDocuments);
|
||||
|
||||
return $databaseDocuments;
|
||||
}
|
||||
|
||||
protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region)
|
||||
{
|
||||
$deploymentsStorage = $dbForProject->sum('deployments', 'size');
|
||||
$buildsStorage = $dbForProject->sum('builds', 'size');
|
||||
$this->createStatsDocuments($region, METRIC_DEPLOYMENTS_STORAGE, $deploymentsStorage);
|
||||
$this->createStatsDocuments($region, METRIC_BUILDS_STORAGE, $buildsStorage);
|
||||
|
||||
$deployments = $dbForProject->count('deployments');
|
||||
$builds = $dbForProject->count('builds');
|
||||
$this->createStatsDocuments($region, METRIC_DEPLOYMENTS, $deployments);
|
||||
$this->createStatsDocuments($region, METRIC_BUILDS, $builds);
|
||||
|
||||
|
||||
$this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForLogs, $region) {
|
||||
$functionDeploymentsStorage = $dbForProject->sum('deployments', 'size', [
|
||||
Query::equal('resourceInternalId', [$function->getInternalId()]),
|
||||
Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]),
|
||||
]);
|
||||
$this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $functionDeploymentsStorage);
|
||||
|
||||
$functionDeployments = $dbForProject->count('deployments', [
|
||||
Query::equal('resourceInternalId', [$function->getInternalId()]),
|
||||
Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]),
|
||||
]);
|
||||
$this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $functionDeployments);
|
||||
|
||||
/**
|
||||
* As deployments and builds have 1-1 relationship,
|
||||
* the count for one should match the other
|
||||
*/
|
||||
$this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS), $functionDeployments);
|
||||
|
||||
$functionBuildsStorage = 0;
|
||||
|
||||
$this->foreachDocument($dbForProject, 'deployments', [
|
||||
Query::equal('resourceInternalId', [$function->getInternalId()]),
|
||||
Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]),
|
||||
], function (Document $deployment) use ($dbForProject, &$functionBuildsStorage): void {
|
||||
$build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''));
|
||||
$functionBuildsStorage += $build->getAttribute('size', 0);
|
||||
});
|
||||
|
||||
$this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionBuildsStorage);
|
||||
});
|
||||
}
|
||||
|
||||
protected function createStatsDocuments(string $region, string $metric, int $value)
|
||||
{
|
||||
foreach ($this->periods as $period => $format) {
|
||||
$time = 'inf' === $period ? null : \date($format, \time());
|
||||
$id = \md5("{$time}_{$period}_{$metric}");
|
||||
|
||||
$this->documents[] = new Document([
|
||||
'$id' => $id,
|
||||
'metric' => $metric,
|
||||
'period' => $period,
|
||||
'region' => $region,
|
||||
'value' => $value,
|
||||
'time' => $time,
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
protected function writeDocuments(Database $dbForLogs, Document $project): void
|
||||
{
|
||||
$dbForLogs->createOrUpdateDocuments(
|
||||
'stats',
|
||||
$this->documents
|
||||
);
|
||||
$this->documents = [];
|
||||
Console::success('Stats written to logs db for project: ' . $project->getId() . '(' . $project->getInternalId() . ')');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Workers;
|
||||
|
||||
use Appwrite\Event\StatsUsageDump;
|
||||
use Exception;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Queue\Message;
|
||||
use Utopia\System\System;
|
||||
|
||||
class StatsUsage extends Action
|
||||
{
|
||||
private array $stats = [];
|
||||
private int $lastTriggeredTime = 0;
|
||||
private int $keys = 0;
|
||||
private const INFINITY_PERIOD = '_inf_';
|
||||
private const KEYS_THRESHOLD = 10000;
|
||||
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'stats-usage';
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws Exception
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
|
||||
$this
|
||||
->desc('Stats usage worker')
|
||||
->inject('message')
|
||||
->inject('getProjectDB')
|
||||
->inject('queueForStatsUsageDump')
|
||||
->callback([$this, 'action']);
|
||||
|
||||
$this->lastTriggeredTime = time();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Message $message
|
||||
* @param callable $getProjectDB
|
||||
* @param StatsUsageDump $queueForStatsUsageDump
|
||||
* @return void
|
||||
* @throws \Utopia\Database\Exception
|
||||
* @throws Exception
|
||||
*/
|
||||
public function action(Message $message, callable $getProjectDB, StatsUsageDump $queueForStatsUsageDump): void
|
||||
{
|
||||
$payload = $message->getPayload() ?? [];
|
||||
if (empty($payload)) {
|
||||
throw new Exception('Missing payload');
|
||||
}
|
||||
//Todo Figure out way to preserve keys when the container is being recreated @shimonewman
|
||||
|
||||
$aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
|
||||
$project = new Document($payload['project'] ?? []);
|
||||
$projectId = $project->getInternalId();
|
||||
foreach ($payload['reduce'] ?? [] as $document) {
|
||||
if (empty($document)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->reduce(
|
||||
project: $project,
|
||||
document: new Document($document),
|
||||
metrics: $payload['metrics'],
|
||||
getProjectDB: $getProjectDB
|
||||
);
|
||||
}
|
||||
|
||||
$this->stats[$projectId]['project'] = $project;
|
||||
$this->stats[$projectId]['receivedAt'] = DateTime::now();
|
||||
foreach ($payload['metrics'] ?? [] as $metric) {
|
||||
$this->keys++;
|
||||
if (!isset($this->stats[$projectId]['keys'][$metric['key']])) {
|
||||
$this->stats[$projectId]['keys'][$metric['key']] = $metric['value'];
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->stats[$projectId]['keys'][$metric['key']] += $metric['value'];
|
||||
}
|
||||
|
||||
// If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
|
||||
if (
|
||||
$this->keys >= self::KEYS_THRESHOLD ||
|
||||
(time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0)
|
||||
) {
|
||||
Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
|
||||
|
||||
$queueForStatsUsageDump
|
||||
->setStats($this->stats)
|
||||
->trigger();
|
||||
|
||||
$this->stats = [];
|
||||
$this->keys = 0;
|
||||
$this->lastTriggeredTime = time();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
|
||||
* When we remove a parent document we need to deduct his children aggregation from the project scope.
|
||||
* @param Document $project
|
||||
* @param Document $document
|
||||
* @param array $metrics
|
||||
* @param callable $getProjectDB
|
||||
* @return void
|
||||
*/
|
||||
private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void
|
||||
{
|
||||
$dbForProject = $getProjectDB($project);
|
||||
|
||||
try {
|
||||
switch (true) {
|
||||
case $document->getCollection() === 'users': // users
|
||||
$sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
|
||||
if (!empty($sessions)) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_SESSIONS,
|
||||
'value' => ($sessions * -1),
|
||||
];
|
||||
}
|
||||
break;
|
||||
case $document->getCollection() === 'databases': // databases
|
||||
$collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
|
||||
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
|
||||
if (!empty($collections['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_COLLECTIONS,
|
||||
'value' => ($collections['value'] * -1),
|
||||
];
|
||||
}
|
||||
|
||||
if (!empty($documents['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_DOCUMENTS,
|
||||
'value' => ($documents['value'] * -1),
|
||||
];
|
||||
}
|
||||
break;
|
||||
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
|
||||
$parts = explode('_', $document->getCollection());
|
||||
$databaseInternalId = $parts[1] ?? 0;
|
||||
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
|
||||
|
||||
if (!empty($documents['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_DOCUMENTS,
|
||||
'value' => ($documents['value'] * -1),
|
||||
];
|
||||
$metrics[] = [
|
||||
'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
|
||||
'value' => ($documents['value'] * -1),
|
||||
];
|
||||
}
|
||||
break;
|
||||
|
||||
case $document->getCollection() === 'buckets':
|
||||
$files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
|
||||
$storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
|
||||
|
||||
if (!empty($files['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_FILES,
|
||||
'value' => ($files['value'] * -1),
|
||||
];
|
||||
}
|
||||
|
||||
if (!empty($storage['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_FILES_STORAGE,
|
||||
'value' => ($storage['value'] * -1),
|
||||
];
|
||||
}
|
||||
break;
|
||||
|
||||
case $document->getCollection() === 'functions':
|
||||
$deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
|
||||
$deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
|
||||
$builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
|
||||
$buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
|
||||
$buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
|
||||
$executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
|
||||
$executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
|
||||
|
||||
if (!empty($deployments['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_DEPLOYMENTS,
|
||||
'value' => ($deployments['value'] * -1),
|
||||
];
|
||||
}
|
||||
|
||||
if (!empty($deploymentsStorage['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_DEPLOYMENTS_STORAGE,
|
||||
'value' => ($deploymentsStorage['value'] * -1),
|
||||
];
|
||||
}
|
||||
|
||||
if (!empty($builds['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_BUILDS,
|
||||
'value' => ($builds['value'] * -1),
|
||||
];
|
||||
}
|
||||
|
||||
if (!empty($buildsStorage['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_BUILDS_STORAGE,
|
||||
'value' => ($buildsStorage['value'] * -1),
|
||||
];
|
||||
}
|
||||
|
||||
if (!empty($buildsCompute['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_BUILDS_COMPUTE,
|
||||
'value' => ($buildsCompute['value'] * -1),
|
||||
];
|
||||
}
|
||||
|
||||
if (!empty($executions['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_EXECUTIONS,
|
||||
'value' => ($executions['value'] * -1),
|
||||
];
|
||||
}
|
||||
|
||||
if (!empty($executionsCompute['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_EXECUTIONS_COMPUTE,
|
||||
'value' => ($executionsCompute['value'] * -1),
|
||||
];
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Workers;
|
||||
|
||||
use Appwrite\Extend\Exception;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Queue\Message;
|
||||
use Utopia\Registry\Registry;
|
||||
use Utopia\System\System;
|
||||
|
||||
class StatsUsageDump extends Action
|
||||
{
|
||||
public const METRIC_COLLECTION_LEVEL_STORAGE = 4;
|
||||
public const METRIC_DATABASE_LEVEL_STORAGE = 3;
|
||||
public const METRIC_PROJECT_LEVEL_STORAGE = 2;
|
||||
protected array $stats = [];
|
||||
|
||||
protected Registry $register;
|
||||
|
||||
/**
|
||||
* Metrics to skip writing to logsDB
|
||||
* As these metrics are calculated separately
|
||||
* by logs DB
|
||||
* @var array
|
||||
*/
|
||||
protected array $skipBaseMetrics = [
|
||||
METRIC_DATABASES => true,
|
||||
METRIC_BUCKETS => true,
|
||||
METRIC_USERS => true,
|
||||
METRIC_FUNCTIONS => true,
|
||||
METRIC_TEAMS => true,
|
||||
METRIC_MESSAGES => true,
|
||||
METRIC_MAU => true,
|
||||
METRIC_WEBHOOKS => true,
|
||||
METRIC_PLATFORMS => true,
|
||||
METRIC_PROVIDERS => true,
|
||||
METRIC_TOPICS => true,
|
||||
METRIC_KEYS => true,
|
||||
METRIC_FILES => true,
|
||||
METRIC_FILES_STORAGE => true,
|
||||
METRIC_DEPLOYMENTS_STORAGE => true,
|
||||
METRIC_BUILDS_STORAGE => true,
|
||||
METRIC_DEPLOYMENTS => true,
|
||||
METRIC_BUILDS => true,
|
||||
METRIC_COLLECTIONS => true,
|
||||
METRIC_DOCUMENTS => true,
|
||||
];
|
||||
|
||||
/**
|
||||
* Skip metrics associated with parent IDs
|
||||
* these need to be checked individually with `str_ends_with`
|
||||
*/
|
||||
protected array $skipParentIdMetrics = [
|
||||
'.files',
|
||||
'.files.storage',
|
||||
'.collections',
|
||||
'.documents',
|
||||
'.deployments',
|
||||
'.deployments.storage',
|
||||
'.builds',
|
||||
'.builds.storage',
|
||||
];
|
||||
|
||||
/**
|
||||
* @var callable
|
||||
*/
|
||||
protected mixed $getLogsDB;
|
||||
|
||||
protected array $periods = [
|
||||
'1h' => 'Y-m-d H:00',
|
||||
'1d' => 'Y-m-d 00:00',
|
||||
'inf' => '0000-00-00 00:00'
|
||||
];
|
||||
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'stats-usage-dump';
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws \Exception
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->inject('message')
|
||||
->inject('getProjectDB')
|
||||
->inject('getLogsDB')
|
||||
->inject('register')
|
||||
->callback([$this, 'action']);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Message $message
|
||||
* @param callable $getProjectDB
|
||||
* @param callable $getLogsDB
|
||||
* @param Registry $register
|
||||
* @return void
|
||||
* @throws Exception
|
||||
* @throws \Throwable
|
||||
* @throws \Utopia\Database\Exception
|
||||
*/
|
||||
public function action(Message $message, callable $getProjectDB, callable $getLogsDB, Registry $register): void
|
||||
{
|
||||
$this->getLogsDB = $getLogsDB;
|
||||
$this->register = $register;
|
||||
$payload = $message->getPayload() ?? [];
|
||||
if (empty($payload)) {
|
||||
throw new Exception('Missing payload');
|
||||
}
|
||||
|
||||
foreach ($payload['stats'] ?? [] as $stats) {
|
||||
$project = new Document($stats['project'] ?? []);
|
||||
|
||||
$numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
|
||||
$receivedAt = $stats['receivedAt'] ?? 'NONE';
|
||||
if ($numberOfKeys === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys);
|
||||
|
||||
try {
|
||||
/** @var \Utopia\Database\Database $dbForProject */
|
||||
$dbForProject = $getProjectDB($project);
|
||||
foreach ($stats['keys'] ?? [] as $key => $value) {
|
||||
if ($value == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (str_contains($key, METRIC_DATABASES_STORAGE)) {
|
||||
try {
|
||||
$this->handleDatabaseStorage($key, $dbForProject, $project);
|
||||
} catch (\Exception $e) {
|
||||
console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($this->periods as $period => $format) {
|
||||
$time = 'inf' === $period ? null : date($format, time());
|
||||
$id = \md5("{$time}_{$period}_{$key}");
|
||||
|
||||
$document = new Document([
|
||||
'$id' => $id,
|
||||
'period' => $period,
|
||||
'time' => $time,
|
||||
'metric' => $key,
|
||||
'value' => $value,
|
||||
'region' => System::getEnv('_APP_REGION', 'default'),
|
||||
]);
|
||||
|
||||
$documentClone = new Document($document->getArrayCopy());
|
||||
|
||||
$dbForProject->createOrUpdateDocumentsWithIncrease(
|
||||
'stats',
|
||||
'value',
|
||||
[$document]
|
||||
);
|
||||
|
||||
$this->writeToLogsDB($project, $documentClone);
|
||||
}
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project): void
|
||||
{
|
||||
$data = explode('.', $key);
|
||||
$start = microtime(true);
|
||||
|
||||
$updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) {
|
||||
$id = \md5("{$time}_{$period}_{$key}");
|
||||
|
||||
$document = new Document([
|
||||
'$id' => $id,
|
||||
'period' => $period,
|
||||
'time' => $time,
|
||||
'metric' => $key,
|
||||
'value' => $value,
|
||||
'region' => System::getEnv('_APP_REGION', 'default'),
|
||||
]);
|
||||
$documentClone = new Document($document->getArrayCopy());
|
||||
$dbForProject->createOrUpdateDocumentsWithIncrease(
|
||||
'stats',
|
||||
'value',
|
||||
[$document]
|
||||
);
|
||||
$this->writeToLogsDB($project, $documentClone);
|
||||
};
|
||||
|
||||
foreach ($this->periods as $period => $format) {
|
||||
$time = 'inf' === $period ? null : date($format, time());
|
||||
$id = \md5("{$time}_{$period}_{$key}");
|
||||
|
||||
$value = 0;
|
||||
$previousValue = 0;
|
||||
try {
|
||||
$previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0);
|
||||
} catch (\Exception $e) {
|
||||
// No previous value
|
||||
}
|
||||
|
||||
switch (count($data)) {
|
||||
// Collection Level
|
||||
case self::METRIC_COLLECTION_LEVEL_STORAGE:
|
||||
Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']');
|
||||
$databaseInternalId = $data[0];
|
||||
$collectionInternalId = $data[1];
|
||||
|
||||
try {
|
||||
$value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId);
|
||||
} catch (\Exception $e) {
|
||||
// Collection not found
|
||||
if ($e->getMessage() !== 'Collection not found') {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
// Compare with previous value
|
||||
$diff = $value - $previousValue;
|
||||
|
||||
if ($diff === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Update Collection
|
||||
$updateMetric($dbForProject, $project, $diff, $key, $period, $time);
|
||||
|
||||
// Update Database
|
||||
$databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE);
|
||||
$updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time);
|
||||
|
||||
// Update Project
|
||||
$projectKey = METRIC_DATABASES_STORAGE;
|
||||
$updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time);
|
||||
break;
|
||||
// Database Level
|
||||
case self::METRIC_DATABASE_LEVEL_STORAGE:
|
||||
Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']');
|
||||
$databaseInternalId = $data[0];
|
||||
|
||||
$collections = [];
|
||||
try {
|
||||
$collections = $dbForProject->find('database_' . $databaseInternalId);
|
||||
} catch (\Exception $e) {
|
||||
// Database not found
|
||||
if ($e->getMessage() !== 'Collection not found') {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($collections as $collection) {
|
||||
try {
|
||||
$value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId());
|
||||
} catch (\Exception $e) {
|
||||
// Collection not found
|
||||
if ($e->getMessage() !== 'Collection not found') {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$diff = $value - $previousValue;
|
||||
|
||||
if ($diff === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Update Database
|
||||
$databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE);
|
||||
$updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time);
|
||||
|
||||
// Update Project
|
||||
$projectKey = METRIC_DATABASES_STORAGE;
|
||||
$updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time);
|
||||
break;
|
||||
// Project Level
|
||||
case self::METRIC_PROJECT_LEVEL_STORAGE:
|
||||
Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']');
|
||||
// Get all project databases
|
||||
$databases = $dbForProject->find('database');
|
||||
|
||||
// Recalculate all databases
|
||||
foreach ($databases as $database) {
|
||||
$collections = $dbForProject->find('database_' . $database->getInternalId());
|
||||
|
||||
foreach ($collections as $collection) {
|
||||
try {
|
||||
$value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
|
||||
} catch (\Exception $e) {
|
||||
// Collection not found
|
||||
if ($e->getMessage() !== 'Collection not found') {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$diff = $value - $previousValue;
|
||||
|
||||
// Update Project
|
||||
$projectKey = METRIC_DATABASES_STORAGE;
|
||||
$updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
$end = microtime(true);
|
||||
|
||||
console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds');
|
||||
}
|
||||
|
||||
protected function writeToLogsDB(Document $project, Document $document): void
|
||||
{
|
||||
if (!System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', false)) {
|
||||
Console::log('Dual Writing is disabled. Skipping...');
|
||||
return;
|
||||
}
|
||||
|
||||
if (array_key_exists($document->getAttribute('metric'), $this->skipBaseMetrics)) {
|
||||
return;
|
||||
}
|
||||
foreach ($this->skipParentIdMetrics as $skipMetric) {
|
||||
if (str_ends_with($document->getAttribute('metric'), $skipMetric)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/** @var \Utopia\Database\Database $dbForLogs*/
|
||||
$dbForLogs = call_user_func($this->getLogsDB, $project);
|
||||
|
||||
try {
|
||||
$dbForLogs->createOrUpdateDocumentsWithIncrease(
|
||||
'stats',
|
||||
'value',
|
||||
[$document]
|
||||
);
|
||||
Console::success('Usage logs pushed to Logs DB');
|
||||
} catch (\Throwable $th) {
|
||||
Console::error($th->getMessage());
|
||||
}
|
||||
|
||||
$this->register->get('pools')->get('logs')->reclaim();
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
namespace Appwrite\Platform\Workers;
|
||||
|
||||
use Appwrite\Event\Mail;
|
||||
use Appwrite\Event\Usage;
|
||||
use Appwrite\Event\StatsUsage;
|
||||
use Appwrite\Template\Template;
|
||||
use Exception;
|
||||
use Utopia\Database\Database;
|
||||
@@ -35,9 +35,9 @@ class Webhooks extends Action
|
||||
->inject('project')
|
||||
->inject('dbForPlatform')
|
||||
->inject('queueForMails')
|
||||
->inject('queueForUsage')
|
||||
->inject('queueForStatsUsage')
|
||||
->inject('log')
|
||||
->callback(fn (Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForMails, $queueForUsage, $log));
|
||||
->callback(fn (Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForMails, $queueForStatsUsage, $log));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -49,7 +49,7 @@ class Webhooks extends Action
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log): void
|
||||
public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, Log $log): void
|
||||
{
|
||||
$this->errors = [];
|
||||
$payload = $message->getPayload() ?? [];
|
||||
@@ -66,7 +66,7 @@ class Webhooks extends Action
|
||||
|
||||
foreach ($project->getAttribute('webhooks', []) as $webhook) {
|
||||
if (array_intersect($webhook->getAttribute('events', []), $events)) {
|
||||
$this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForUsage);
|
||||
$this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForStatsUsage);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,7 +85,7 @@ class Webhooks extends Action
|
||||
* @param Mail $queueForMails
|
||||
* @return void
|
||||
*/
|
||||
private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage): void
|
||||
private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage): void
|
||||
{
|
||||
if ($webhook->getAttribute('enabled') !== true) {
|
||||
return;
|
||||
@@ -168,7 +168,7 @@ class Webhooks extends Action
|
||||
$dbForPlatform->purgeCachedDocument('projects', $project->getId());
|
||||
|
||||
$this->errors[] = $logs;
|
||||
$queueForUsage
|
||||
$queueForStatsUsage
|
||||
->addMetric(METRIC_WEBHOOKS_FAILED, 1)
|
||||
->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_FAILED), 1)
|
||||
;
|
||||
@@ -178,13 +178,13 @@ class Webhooks extends Action
|
||||
$webhook->setAttribute('attempts', 0); // Reset attempts on success
|
||||
$dbForPlatform->updateDocument('webhooks', $webhook->getId(), $webhook);
|
||||
$dbForPlatform->purgeCachedDocument('projects', $project->getId());
|
||||
$queueForUsage
|
||||
$queueForStatsUsage
|
||||
->addMetric(METRIC_WEBHOOKS_SENT, 1)
|
||||
->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_SENT), 1)
|
||||
;
|
||||
}
|
||||
|
||||
$queueForUsage
|
||||
$queueForStatsUsage
|
||||
->setProject($project)
|
||||
->trigger();
|
||||
}
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
|
||||
namespace Appwrite\Utopia;
|
||||
|
||||
use Appwrite\Auth\Auth;
|
||||
use Appwrite\Utopia\Request\Filter;
|
||||
use Swoole\Http\Request as SwooleRequest;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Route;
|
||||
use Utopia\Swoole\Request as UtopiaRequest;
|
||||
|
||||
@@ -180,4 +182,27 @@ class Request extends UtopiaRequest
|
||||
$headers = $this->getHeaders();
|
||||
return $headers[$key] ?? $default;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get User Agent
|
||||
*
|
||||
* Method for getting User Agent. Preferring forwarded agent for privileged users; otherwise returns default.
|
||||
*
|
||||
* @param string $default
|
||||
* @return string
|
||||
*/
|
||||
public function getUserAgent(string $default = ''): string
|
||||
{
|
||||
$forwardedUserAgent = $this->getHeader('x-forwarded-user-agent');
|
||||
if (!empty($forwardedUserAgent)) {
|
||||
$roles = Authorization::getRoles();
|
||||
$isAppUser = Auth::isAppUser($roles);
|
||||
|
||||
if ($isAppUser) {
|
||||
return $forwardedUserAgent;
|
||||
}
|
||||
}
|
||||
|
||||
return UtopiaRequest::getUserAgent($default);
|
||||
}
|
||||
}
|
||||
@@ -323,9 +323,9 @@ class Project extends Model
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Collection
|
||||
* Filter document structure
|
||||
*
|
||||
* @return string
|
||||
* @return Document
|
||||
*/
|
||||
public function filter(Document $document): Document
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user