resolve merge conflicts

This commit is contained in:
prateek banga committed 2023-08-14 21:13:47 +05:30
commit 4b63a4abbb
216 files changed
+10808 -9855

No files matched your search

-1
View File
@@ -9,7 +9,6 @@ use Appwrite\Auth\Hash\Phpass;
use Appwrite\Auth\Hash\Scrypt;
use Appwrite\Auth\Hash\Scryptmodified;
use Appwrite\Auth\Hash\Sha;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\Role;
+29 -3
View File
@@ -158,6 +158,18 @@ class Github extends OAuth2
return $user['name'] ?? '';
}
/**
* @param string $accessToken
*
* @return string
*/
public function getUserSlug(string $accessToken): string
{
$user = $this->getUser($accessToken);
return $user['login'] ?? '';
}
/**
* @param string $accessToken
*
@@ -171,13 +183,27 @@ class Github extends OAuth2
$emails = $this->request('GET', 'https://api.github.com/user/emails', ['Authorization: token ' . \urlencode($accessToken)]);
$emails = \json_decode($emails, true);
$verifiedEmail = null;
$primaryEmail = null;
foreach ($emails as $email) {
if (isset($email['verified']) && $email['verified'] === true) {
$this->user['email'] = $email['email'];
$this->user['verified'] = $email['verified'];
break;
$verifiedEmail = $email;
if (isset($email['primary']) && $email['primary'] === true) {
$primaryEmail = $email;
}
}
}
if (!empty($primaryEmail)) {
$this->user['email'] = $primaryEmail['email'];
$this->user['verified'] = $primaryEmail['verified'];
} elseif (!empty($verifiedEmail)) {
$this->user['email'] = $verifiedEmail['email'];
$this->user['verified'] = $verifiedEmail['verified'];
}
}
return $this->user;
@@ -0,0 +1,102 @@
<?php
namespace Appwrite\Auth\Validator;
/**
* Validates user password string against their personal data
*/
class PersonalData extends Password
{
public function __construct(
protected ?string $userId = null,
protected ?string $email = null,
protected ?string $name = null,
protected ?string $phone = null,
protected bool $strict = false
) {
}
/**
* Get Description.
*
* Returns validator description
*
* @return string
*/
public function getDescription(): string
{
return 'Password must not include any personal data like your name, email, phone number, etc.';
}
/**
* Is valid.
*
* @param mixed $value
*
* @return bool
*/
public function isValid($password): bool
{
if (!parent::isValid($password)) {
return false;
}
if (!$this->strict) {
$password = strtolower($password);
$this->userId = strtolower($this->userId);
$this->email = strtolower($this->email);
$this->name = strtolower($this->name);
$this->phone = strtolower($this->phone);
}
if ($this->userId && strpos($password, $this->userId) !== false) {
return false;
}
if ($this->email && strpos($password, $this->email) !== false) {
return false;
}
if ($this->email && strpos($password, explode('@', $this->email)[0] ?? '') !== false) {
return false;
}
if ($this->name && strpos($password, $this->name) !== false) {
return false;
}
if ($this->phone && strpos($password, str_replace('+', '', $this->phone)) !== false) {
return false;
}
if ($this->phone && strpos($password, $this->phone) !== false) {
return false;
}
return true;
}
/**
* Is array
*
* Function will return true if object is array.
*
* @return bool
*/
public function isArray(): bool
{
return false;
}
/**
* Get Type
*
* Returns validator type.
*
* @return string
*/
public function getType(): string
{
return self::TYPE_STRING;
}
}
-135
View File
@@ -1,135 +0,0 @@
<?php
namespace Appwrite\DSN;
class DSN
{
/**
* @var string
*/
protected string $scheme;
/**
* @var ?string
*/
protected ?string $user;
/**
* @var ?string
*/
protected ?string $password;
/**
* @var string
*/
protected string $host;
/**
* @var ?string
*/
protected ?string $port;
/**
* @var ?string
*/
protected ?string $database;
/**
* @var ?string
*/
protected ?string $query;
/**
* Construct
*
* Construct a new DSN object
*
* @param string $dsn
*/
public function __construct(string $dsn)
{
$parts = parse_url($dsn);
if (!$parts) {
throw new \InvalidArgumentException("Unable to parse DSN: $dsn");
}
$this->scheme = $parts['scheme'] ?? null;
$this->user = isset($parts['user']) ? \urldecode($parts['user']) : null;
$this->password = isset($parts['pass']) ? \urldecode($parts['pass']) : null;
$this->host = $parts['host'] ?? null;
$this->port = $parts['port'] ?? null;
$this->database = $parts['path'] ?? null;
$this->query = $parts['query'] ?? null;
}
/**
* Return the scheme.
*
* @return string
*/
public function getScheme(): string
{
return $this->scheme;
}
/**
* Return the user.
*
* @return ?string
*/
public function getUser(): ?string
{
return $this->user;
}
/**
* Return the password.
*
* @return ?string
*/
public function getPassword(): ?string
{
return $this->password;
}
/**
* Return the host
*
* @return string
*/
public function getHost(): string
{
return $this->host;
}
/**
* Return the port
*
* @return ?string
*/
public function getPort(): ?string
{
return $this->port;
}
/**
* Return the database
*
* @return ?string
*/
public function getDatabase(): ?string
{
return ltrim($this->database, '/');
}
/**
* Return the raw query string
*
* @return ?string
*/
public function getQuery(): ?string
{
return $this->query;
}
}
+31 -7
View File
@@ -23,6 +23,9 @@ class Event
public const FUNCTIONS_QUEUE_NAME = 'v1-functions';
public const FUNCTIONS_CLASS_NAME = 'FunctionsV1';
public const USAGE_QUEUE_NAME = 'v1-usage';
public const USAGE_CLASS_NAME = 'UsageV1';
public const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
public const WEBHOOK_CLASS_NAME = 'WebhooksV1';
@@ -43,6 +46,7 @@ class Event
protected array $context = [];
protected ?Document $project = null;
protected ?Document $user = null;
protected bool $paused = false;
/**
* @param string $queue
@@ -116,9 +120,9 @@ class Event
/**
* Get project for this event.
*
* @return Document
* @return ?Document
*/
public function getProject(): Document
public function getProject(): ?Document
{
return $this->project;
}
@@ -137,11 +141,11 @@ class Event
}
/**
* Get project for this event.
* Get user responsible for triggering this event.
*
* @return Document
* @return ?Document
*/
public function getUser(): Document
public function getUser(): ?Document
{
return $this->user;
}
@@ -260,6 +264,10 @@ class Event
*/
public function trigger(): string|bool
{
if ($this->paused) {
return false;
}
return Resque::enqueue($this->queue, $this->class, [
'project' => $this->project,
'user' => $this->user,
@@ -337,8 +345,6 @@ class Event
default => false
};
return [
'type' => $type,
'resource' => $resource,
@@ -467,4 +473,22 @@ class Event
*/
return \array_values($events);
}
/**
* Get the value of paused
*/
public function isPaused(): bool
{
return $this->paused;
}
/**
* Set the value of paused
*/
public function setPaused(bool $paused): self
{
$this->paused = $paused;
return $this;
}
}
+26 -21
View File
@@ -2,10 +2,9 @@
namespace Appwrite\Event;
use DateTime;
use Resque;
use ResqueScheduler;
use Utopia\Database\Document;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
class Func extends Event
{
@@ -15,7 +14,7 @@ class Func extends Event
protected ?Document $function = null;
protected ?Document $execution = null;
public function __construct()
public function __construct(protected Connection $connection)
{
parent::__construct(Event::FUNCTIONS_QUEUE_NAME, Event::FUNCTIONS_CLASS_NAME);
}
@@ -143,7 +142,15 @@ class Func extends Event
*/
public function trigger(): string|bool
{
return Resque::enqueue($this->queue, $this->class, [
if ($this->paused) {
return false;
}
$client = new Client($this->queue, $this->connection);
$events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null;
return $client->enqueue([
'project' => $this->project,
'user' => $this->user,
'function' => $this->function,
@@ -151,28 +158,26 @@ class Func extends Event
'type' => $this->type,
'jwt' => $this->jwt,
'payload' => $this->payload,
'data' => $this->data
'events' => $events,
'data' => $this->data,
]);
}
/**
* Schedules the function event and schedules it in the functions worker queue.
* Generate a function event from a base event
*
* @param Event $event
*
* @return self
*
* @param \DateTime|int $at
* @return void
* @throws \Resque_Exception
* @throws \ResqueScheduler_InvalidTimestampException
*/
public function schedule(DateTime|int $at): void
public function from(Event $event): self
{
ResqueScheduler::enqueueAt($at, $this->queue, $this->class, [
'project' => $this->project,
'user' => $this->user,
'function' => $this->function,
'execution' => $this->execution,
'type' => $this->type,
'payload' => $this->payload,
'data' => $this->data
]);
$this->project = $event->getProject();
$this->user = $event->getUser();
$this->payload = $event->getPayload();
$this->event = $event->getEvent();
$this->params = $event->getParams();
return $this;
}
}
+64
View File
@@ -0,0 +1,64 @@
<?php
namespace Appwrite\Event;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
use Utopia\Database\Document;
class Usage extends Event
{
protected array $metrics = [];
protected array $reduce = [];
public function __construct(protected Connection $connection)
{
parent::__construct(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME);
}
/**
* Add reduce.
*
* @param Document $document
* @return self
*/
public function addReduce(Document $document): self
{
$this->reduce[] = $document;
return $this;
}
/**
* Add metric.
*
* @param string $key
* @param int $value
* @return self
*/
public function addMetric(string $key, int $value): self
{
$this->metrics[] = [
'key' => $key,
'value' => $value,
];
return $this;
}
/**
* Sends metrics to the usage worker.
*
* @return string|bool
*/
public function trigger(): string|bool
{
$client = new Client($this->queue, $this->connection);
return $client->enqueue([
'project' => $this->getProject(),
'reduce' => $this->reduce,
'metrics' => $this->metrics,
]);
}
}
+12 -1
View File
@@ -7,6 +7,11 @@ use Utopia\Validator;
class Event extends Validator
{
/**
* @var string
*/
protected string $message = 'Event is not valid.';
/**
* Get Description.
*
@@ -16,7 +21,7 @@ class Event extends Validator
*/
public function getDescription(): string
{
return 'Event is not valid.';
return $this->message;
}
/**
@@ -40,6 +45,12 @@ class Event extends Validator
* Identify all sections of the pattern.
*/
$type = $parts[0] ?? false;
if ($type == 'functions') {
$this->message = 'Triggering a function on a function event is not allowed.';
return false;
}
$resource = $parts[1] ?? false;
$hasSubResource = $count > 3 && ($events[$type]['$resource'] ?? false) && ($events[$type][$parts[2]]['$resource'] ?? false);
$hasSubSubResource = $count > 5 && $hasSubResource && ($events[$type][$parts[2]][$parts[4]]['$resource'] ?? false);
+5
View File
@@ -51,6 +51,7 @@ class Exception extends \Exception
public const GENERAL_CURSOR_NOT_FOUND = 'general_cursor_not_found';
public const GENERAL_SERVER_ERROR = 'general_server_error';
public const GENERAL_PROTOCOL_UNSUPPORTED = 'general_protocol_unsupported';
public const GENERAL_CODES_DISABLED = 'general_codes_disabled';
public const GENERAL_USAGE_DISABLED = 'general_usage_disabled';
/** Users */
@@ -62,11 +63,13 @@ class Exception extends \Exception
public const USER_PASSWORD_RESET_REQUIRED = 'user_password_reset_required';
public const USER_EMAIL_NOT_WHITELISTED = 'user_email_not_whitelisted';
public const USER_IP_NOT_WHITELISTED = 'user_ip_not_whitelisted';
public const USER_INVALID_CODE = 'user_invalid_code';
public const USER_INVALID_CREDENTIALS = 'user_invalid_credentials';
public const USER_ANONYMOUS_CONSOLE_PROHIBITED = 'user_anonymous_console_prohibited';
public const USER_SESSION_ALREADY_EXISTS = 'user_session_already_exists';
public const USER_NOT_FOUND = 'user_not_found';
public const USER_PASSWORD_RECENTLY_USED = 'password_recently_used';
public const USER_PASSWORD_PERSONAL_DATA = 'password_personal_data';
public const USER_EMAIL_ALREADY_EXISTS = 'user_email_already_exists';
public const USER_PASSWORD_MISMATCH = 'user_password_mismatch';
public const USER_SESSION_NOT_FOUND = 'user_session_not_found';
@@ -192,6 +195,7 @@ class Exception extends \Exception
/** Domain */
public const DOMAIN_NOT_FOUND = 'domain_not_found';
public const DOMAIN_ALREADY_EXISTS = 'domain_already_exists';
public const DOMAIN_FORBIDDEN = 'domain_forbidden';
public const DOMAIN_VERIFICATION_FAILED = 'domain_verification_failed';
public const DOMAIN_TARGET_INVALID = 'domain_target_invalid';
@@ -200,6 +204,7 @@ class Exception extends \Exception
public const GRAPHQL_TOO_MANY_QUERIES = 'graphql_too_many_queries';
protected $type = '';
protected $errors = [];
public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
{
-110
View File
@@ -1,110 +0,0 @@
<?php
namespace Appwrite\Extend;
use PDO as PDONative;
class PDO extends PDONative
{
/**
* @var PDONative
*/
protected $pdo;
/**
* @var mixed
*/
protected $dsn;
/**
* @var mixed
*/
protected $username;
/**
* @var mixed
*/
protected $passwd;
/**
* @var mixed
*/
protected $options;
/**
* Create A Proxy PDO Object
*/
public function __construct($dsn, $username = null, $passwd = null, $options = null)
{
$this->dsn = $dsn;
$this->username = $username;
$this->passwd = $passwd;
$this->options = $options;
$this->pdo = new PDONative($dsn, $username, $passwd, $options);
}
public function setAttribute($attribute, $value)
{
return $this->pdo->setAttribute($attribute, $value);
}
public function prepare($statement, $driver_options = null)
{
return new PDOStatement($this, $this->pdo->prepare($statement, []));
}
public function quote($string, $parameter_type = PDONative::PARAM_STR)
{
return $this->pdo->quote($string, $parameter_type);
}
public function beginTransaction()
{
try {
$result = $this->pdo->beginTransaction();
} catch (\Throwable $th) {
$this->pdo = $this->reconnect();
$result = $this->pdo->beginTransaction();
}
return $result;
}
public function rollBack()
{
try {
$result = $this->pdo->rollBack();
} catch (\Throwable $th) {
$this->pdo = $this->reconnect();
return false;
}
return $result;
}
public function commit()
{
try {
$result = $this->pdo->commit();
} catch (\Throwable $th) {
$this->pdo = $this->reconnect();
$result = $this->pdo->commit();
}
return $result;
}
public function reconnect(): PDONative
{
$this->pdo = new PDONative($this->dsn, $this->username, $this->passwd, $this->options);
echo '[PDO] MySQL connection restarted' . PHP_EOL;
// Connection settings
$this->pdo->setAttribute(PDONative::ATTR_DEFAULT_FETCH_MODE, PDONative::FETCH_ASSOC); // Return arrays
$this->pdo->setAttribute(PDONative::ATTR_ERRMODE, PDONative::ERRMODE_EXCEPTION); // Handle all errors with exceptions
return $this->pdo;
}
}
-115
View File
@@ -1,115 +0,0 @@
<?php
namespace Appwrite\Extend;
use PDO as PDONative;
use PDOStatement as PDOStatementNative;
class PDOStatement extends PDOStatementNative
{
/**
* @var PDO
*/
protected $pdo;
/**
* Params
*/
protected $params = [];
/**
* Values
*/
protected $values = [];
/**
* Columns
*/
protected $columns = [];
/**
* @var PDOStatementNative
*/
protected $PDOStatement;
public function __construct(PDO &$pdo, PDOStatementNative $PDOStatement)
{
$this->pdo = &$pdo;
$this->PDOStatement = $PDOStatement;
}
public function bindValue($parameter, $value, $data_type = PDONative::PARAM_STR)
{
$this->values[$parameter] = ['value' => $value, 'data_type' => $data_type];
$result = $this->PDOStatement->bindValue($parameter, $value, $data_type);
return $result;
}
public function bindParam($parameter, &$variable, $data_type = PDONative::PARAM_STR, $length = null, $driver_options = null)
{
$this->params[$parameter] = ['value' => &$variable, 'data_type' => $data_type, 'length' => $length, 'driver_options' => $driver_options];
$result = $this->PDOStatement->bindParam($parameter, $variable, $data_type, $length, $driver_options);
return $result;
}
public function bindColumn($column, &$param, $type = null, $maxlen = null, $driverdata = null)
{
$this->columns[$column] = ['param' => &$param, 'type' => $type, 'maxlen' => $maxlen, 'driverdata' => $driverdata];
$result = $this->PDOStatement->bindColumn($column, $param, $type, $maxlen, $driverdata);
return $result;
}
public function execute($input_parameters = null)
{
try {
$result = $this->PDOStatement->execute($input_parameters);
} catch (\Throwable $th) {
$this->pdo = $this->pdo->reconnect();
$this->PDOStatement = $this->pdo->prepare($this->PDOStatement->queryString, []);
foreach ($this->values as $key => $set) {
$this->PDOStatement->bindValue($key, $set['value'], $set['data_type']);
}
foreach ($this->params as $key => $set) {
$this->PDOStatement->bindParam($key, $set['variable'], $set['data_type'], $set['length'], $set['driver_options']);
}
foreach ($this->columns as $key => $set) {
$this->PDOStatement->bindColumn($key, $set['param'], $set['type'], $set['maxlen'], $set['driverdata']);
}
$result = $this->PDOStatement->execute($input_parameters);
}
return $result;
}
public function fetch($fetch_style = PDONative::FETCH_ASSOC, $cursor_orientation = PDONative::FETCH_ORI_NEXT, $cursor_offset = 0)
{
$result = $this->PDOStatement->fetch($fetch_style, $cursor_orientation, $cursor_offset);
return $result;
}
/**
* Fetch All
*
* @param int $fetch_style
* @param mixed $fetch_args
*
* @return array|false
*/
public function fetchAll(int $fetch_style = PDO::FETCH_BOTH, mixed ...$fetch_args)
{
$result = $this->PDOStatement->fetchAll();
return $result;
}
}
+2
View File
@@ -252,6 +252,8 @@ class Mapper
case 'Appwrite\Utopia\Database\Validator\Queries\Base':
case 'Appwrite\Utopia\Database\Validator\Queries\Buckets':
case 'Appwrite\Utopia\Database\Validator\Queries\Collections':
case 'Appwrite\Utopia\Database\Validator\Queries\Attributes':
case 'Appwrite\Utopia\Database\Validator\Queries\Indexes':
case 'Appwrite\Utopia\Database\Validator\Queries\Databases':
case 'Appwrite\Utopia\Database\Validator\Queries\Deployments':
case 'Utopia\Database\Validator\Queries\Documents':
-2
View File
@@ -106,8 +106,6 @@ abstract class Migration
{
$this->project = $project;
$this->projectDB = $projectDB;
$this->projectDB->setNamespace('_' . $this->project->getId());
$this->consoleDB = $consoleDB;
return $this;
+153
View File
@@ -70,6 +70,159 @@ class V17 extends Migration
$this->projectDB->setNamespace("_{$this->project->getInternalId()}");
switch ($id) {
case 'files':
try {
/**
* Update 'mimeType' attribute size (127->255)
*/
$this->projectDB->updateAttribute($id, 'mimeType', Database::VAR_STRING, 255, true, false);
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'mimeType' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'bucketInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}");
}
break;
case 'builds':
try {
/**
* Delete 'endTime' attribute (use startTime+duration if needed)
*/
$this->projectDB->deleteAttribute($id, 'endTime');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'endTime' from {$id}: {$th->getMessage()}");
}
try {
/**
* Rename 'outputPath' to 'path'
*/
$this->projectDB->renameAttribute($id, 'outputPath', 'path');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'path' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'deploymentInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}");
}
break;
case 'stats':
try {
/**
* Delete 'type' attribute
*/
$this->projectDB->deleteAttribute($id, 'type');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'type' from {$id}: {$th->getMessage()}");
}
break;
case 'schedules':
try {
/**
* Create 'resourceInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}");
}
break;
case 'functions':
try {
/**
* Create 'deploymentInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'scheduleInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'scheduleInternalId');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'scheduleInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Delete 'scheduleUpdatedAt' attribute
*/
$this->projectDB->deleteAttribute($id, 'scheduleUpdatedAt');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'scheduleUpdatedAt' from {$id}: {$th->getMessage()}");
}
break;
case 'deployments':
try {
/**
* Create 'resourceInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'resourceInternalId');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'buildInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'buildInternalId');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'buildInternalId' from {$id}: {$th->getMessage()}");
}
break;
case 'executions':
try {
/**
* Create 'functionInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'functionInternalId');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'functionInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'deploymentInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}");
}
break;
default:
break;
}
+14
View File
@@ -0,0 +1,14 @@
<?php
namespace Appwrite\Platform;
use Appwrite\Platform\Services\Tasks;
use Utopia\Platform\Platform;
class Appwrite extends Platform
{
public function __construct()
{
$this->addService('tasks', new Tasks());
}
}
+52
View File
@@ -0,0 +1,52 @@
<?php
namespace Appwrite\Platform\Services;
use Utopia\Platform\Service;
use Appwrite\Platform\Tasks\Doctor;
use Appwrite\Platform\Tasks\Install;
use Appwrite\Platform\Tasks\Maintenance;
use Appwrite\Platform\Tasks\Migrate;
use Appwrite\Platform\Tasks\Schedule;
use Appwrite\Platform\Tasks\PatchCreateMissingSchedules;
use Appwrite\Platform\Tasks\SDKs;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\SSL;
use Appwrite\Platform\Tasks\Hamster;
use Appwrite\Platform\Tasks\PatchDeleteScheduleUpdatedAtAttribute;
use Appwrite\Platform\Tasks\ClearCardCache;
use Appwrite\Platform\Tasks\Usage;
use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
use Appwrite\Platform\Tasks\VolumeSync;
use Appwrite\Platform\Tasks\CalcUsersStats;
use Appwrite\Platform\Tasks\CalcTierStats;
use Appwrite\Platform\Tasks\PatchDeleteProjectCollections;
class Tasks extends Service
{
public function __construct()
{
$this->type = self::TYPE_CLI;
$this
->addAction(Version::getName(), new Version())
->addAction(Vars::getName(), new Vars())
->addAction(SSL::getName(), new SSL())
->addAction(Hamster::getName(), new Hamster())
->addAction(Doctor::getName(), new Doctor())
->addAction(Install::getName(), new Install())
->addAction(Maintenance::getName(), new Maintenance())
->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules())
->addAction(ClearCardCache::getName(), new ClearCardCache())
->addAction(PatchDeleteScheduleUpdatedAtAttribute::getName(), new PatchDeleteScheduleUpdatedAtAttribute())
->addAction(Schedule::getName(), new Schedule())
->addAction(Migrate::getName(), new Migrate())
->addAction(SDKs::getName(), new SDKs())
->addAction(VolumeSync::getName(), new VolumeSync())
->addAction(Specs::getName(), new Specs())
->addAction(CalcUsersStats::getName(), new CalcUsersStats())
->addAction(CalcTierStats::getName(), new CalcTierStats())
->addAction(PatchDeleteProjectCollections::getName(), new PatchDeleteProjectCollections())
;
}
}
@@ -0,0 +1,351 @@
<?php
namespace Appwrite\Platform\Tasks;
use Exception;
use League\Csv\CannotInsertRecord;
use Utopia\App;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use League\Csv\Writer;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
class CalcTierStats extends Action
{
/*
* Csv cols headers
*/
private array $columns = [
'Project ID',
'Organization ID',
'Organization Members',
'Teams',
'Users',
'Requests',
'Bandwidth',
'Domains',
'Api keys',
'Webhooks',
'Platforms',
'Buckets',
'Files',
'Storage (bytes)',
'Max File Size (bytes)',
'Databases',
'Functions',
'Deployments',
'Executions',
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
private array $usageStats = [
'project.$all.network.requests' => 'Requests',
'project.$all.network.bandwidth' => 'Bandwidth',
];
public static function getName(): string
{
return 'calc-tier-stats';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('register')
->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
$this->action($pools, $cache, $dbForConsole, $register);
});
}
/**
* @throws \Utopia\Exception
* @throws CannotInsertRecord
*/
public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
{
//docker compose exec -t appwrite calc-tier-stats
Console::title('Cloud free tier stats calculation V1');
Console::success(APP_NAME . ' cloud free tier stats calculation has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/tier_stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 30;
$sum = 30;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
/** Get Project ID */
$stats['Project ID'] = $project->getId();
$stats['Organization ID'] = $project->getAttribute('teamId', null);
/** Get Total Members */
$teamInternalId = $project->getAttribute('teamInternalId', null);
if ($teamInternalId) {
$stats['Organization Members'] = $dbForConsole->count('memberships', [
Query::equal('teamInternalId', [$teamInternalId])
]);
} else {
$stats['Organization Members'] = 0;
}
/** Get Total internal Teams */
try {
$stats['Teams'] = $dbForProject->count('teams', []);
} catch (\Throwable) {
$stats['Teams'] = 0;
}
/** Get Total users */
try {
$stats['Users'] = $dbForProject->count('users', []);
} catch (\Throwable) {
$stats['Users'] = 0;
}
/** Get Usage stats */
$range = '90d';
$periods = [
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$tmp = [];
$metrics = $this->usageStats;
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
foreach ($metrics as $metric => $name) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$tmp[$metric] = [];
foreach ($requestDocs as $requestDoc) {
if (empty($requestDoc)) {
continue;
}
$tmp[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$tmp[$metric] = array_reverse($tmp[$metric]);
$tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
}
});
foreach ($tmp as $key => $value) {
$stats[$metrics[$key]] = $value;
}
try {
/** Get Domains */
$stats['Domains'] = $dbForConsole->count('domains', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Domains'] = 0;
}
try {
/** Get Api keys */
$stats['Api keys'] = $dbForConsole->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Api keys'] = 0;
}
try {
/** Get Webhooks */
$stats['Webhooks'] = $dbForConsole->count('webhooks', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Webhooks'] = 0;
}
try {
/** Get Platforms */
$stats['Platforms'] = $dbForConsole->count('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Platforms'] = 0;
}
/** Get Files & Buckets */
$filesCount = 0;
$filesSum = 0;
$maxFileSize = 0;
$counter = 0;
try {
$buckets = $dbForProject->find('buckets', []);
foreach ($buckets as $bucket) {
$file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
if (empty($file)) {
continue;
}
$filesSum += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', [], 0);
$filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
$maxFileSize = $file->getAttribute('sizeOriginal');
}
$counter++;
}
} catch (\Throwable) {
;
}
$stats['Buckets'] = $counter;
$stats['Files'] = $filesCount;
$stats['Storage (bytes)'] = $filesSum;
$stats['Max File Size (bytes)'] = $maxFileSize;
try {
/** Get Total Functions */
$stats['Databases'] = $dbForProject->count('databases', []);
} catch (\Throwable) {
$stats['Databases'] = 0;
}
/** Get Total Functions */
try {
$stats['Functions'] = $dbForProject->count('functions', []);
} catch (\Throwable) {
$stats['Functions'] = 0;
}
/** Get Total Deployments */
try {
$stats['Deployments'] = $dbForProject->count('deployments', []);
} catch (\Throwable) {
$stats['Deployments'] = 0;
}
/** Get Total Executions */
try {
$stats['Executions'] = $dbForProject->count('executions', []);
} catch (\Throwable) {
$stats['Executions'] = 0;
}
$csv->insertOne(array_values($stats));
} catch (\Throwable $th) {
Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$pools
->get('console')
->reclaim();
/** @var PHPMailer $mail */
$mail = $register->get('smtp');
$mail->clearAddresses();
$mail->clearAllRecipients();
$mail->clearReplyTos();
$mail->clearAttachments();
$mail->clearBCCs();
$mail->clearCCs();
try {
/** Addresses */
$mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
$recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
foreach ($recipients as $recipient) {
$mail->addAddress($recipient);
}
/** Attachments */
$mail->addAttachment($this->path);
/** Content */
$mail->Subject = "Cloud Report for {$this->date}";
$mail->Body = "Please find the daily cloud report atttached";
$mail->send();
Console::success('Email has been sent!');
} catch (Exception $e) {
Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
}
}
}
@@ -0,0 +1,176 @@
<?php
namespace Appwrite\Platform\Tasks;
use Exception;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use League\Csv\Writer;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
class CalcUsersStats extends Action
{
private array $columns = [
'Project ID',
'Project Name',
'Team ID',
'Team name',
'Users'
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
public static function getName(): string
{
return 'calc-users-stats';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('register')
->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
$this->action($pools, $cache, $dbForConsole, $register);
});
}
public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
{
//docker compose exec -t appwrite calc-users-stats
Console::title('Cloud Users calculation V1');
Console::success(APP_NAME . ' cloud Users calculation has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/users_stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 30;
$sum = 30;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
/** Get Project ID */
$stats['Project ID'] = $project->getId();
/** Get Project Name */
$stats['Project Name'] = $project->getAttribute('name');
/** Get Team Name and Id */
$teamId = $project->getAttribute('teamId', null);
$teamName = null;
if ($teamId) {
$team = $dbForConsole->getDocument('teams', $teamId);
$teamName = $team->getAttribute('name');
}
$stats['Team ID'] = $teamId;
$stats['Team name'] = $teamName;
/** Get Total Users */
$stats['users'] = $dbForProject->count('users', []);
$csv->insertOne(array_values($stats));
} catch (\Throwable $th) {
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$pools
->get('console')
->reclaim();
/** @var PHPMailer $mail */
$mail = $register->get('smtp');
$mail->clearAddresses();
$mail->clearAllRecipients();
$mail->clearReplyTos();
$mail->clearAttachments();
$mail->clearBCCs();
$mail->clearCCs();
try {
/** Addresses */
$mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
$recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
foreach ($recipients as $recipient) {
$mail->addAddress($recipient);
}
/** Attachments */
$mail->addAttachment($this->path);
/** Content */
$mail->Subject = "Cloud Report for {$this->date}";
$mail->Body = "Please find the daily cloud report atttached";
$mail->send();
Console::success('Email has been sent!');
} catch (Exception $e) {
Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
}
}
}
@@ -0,0 +1,62 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
class ClearCardCache extends Action
{
public static function getName(): string
{
return 'clear-card-cache';
}
public function __construct()
{
$this
->desc('Deletes card cache for specific user')
->param('userId', '', new UID(), 'User UID.', false)
->inject('dbForConsole')
->callback(fn (string $userId, Database $dbForConsole) => $this->action($userId, $dbForConsole));
}
public function action(string $userId, Database $dbForConsole): void
{
Authorization::disable();
Authorization::setDefaultStatus(false);
Console::title('ClearCardCache V1');
Console::success(APP_NAME . ' ClearCardCache v1 has started');
$resources = ['card/' . $userId, 'card-back/' . $userId, 'card-og/' . $userId];
$caches = Authorization::skip(fn () => $dbForConsole->find('cache', [
Query::equal('resource', $resources),
Query::limit(100)
]));
$count = \count($caches);
Console::info("Going to delete {$count} cache records in 10 seconds...");
\sleep(10);
foreach ($caches as $cache) {
$key = $cache->getId();
$cacheFolder = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-console')
);
$cacheFolder->purge($key);
Authorization::skip(fn () => $dbForConsole->deleteDocument('cache', $cache->getId()));
}
Console::success(APP_NAME . ' ClearCardCache v1 has finished');
}
}
+264
View File
@@ -0,0 +1,264 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\App;
use Utopia\CLI\Console;
use Appwrite\ClamAV\Network;
use Utopia\Logger\Logger;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
use Utopia\Config\Config;
use Utopia\Domains\Domain;
use Utopia\Platform\Action;
use Utopia\Registry\Registry;
class Doctor extends Action
{
public static function getName(): string
{
return 'doctor';
}
public function __construct()
{
$this
->desc('Validate server health')
->inject('register')
->callback(fn (Registry $register) => $this->action($register));
}
public function action(Registry $register): void
{
Console::log(" __ ____ ____ _ _ ____ __ ____ ____ __ __
/ _\ ( _ \( _ \/ )( \( _ \( )(_ _)( __) ( )/ \
/ \ ) __/ ) __/\ /\ / ) / )( )( ) _) _ )(( O )
\_/\_/(__) (__) (_/\_)(__\_)(__) (__) (____)(_)(__)\__/ ");
Console::log("\n" . '👩‍⚕️ Running ' . APP_NAME . ' Doctor for version ' . App::getEnv('_APP_VERSION', 'UNKNOWN') . ' ...' . "\n");
Console::log('[Settings]');
$domain = new Domain(App::getEnv('_APP_DOMAIN'));
if (!$domain->isKnown() || $domain->isTest()) {
Console::log('🔴 Hostname has no public suffix (' . $domain->get() . ')');
} else {
Console::log('🟢 Hostname has a public suffix (' . $domain->get() . ')');
}
$domain = new Domain(App::getEnv('_APP_DOMAIN_TARGET'));
if (!$domain->isKnown() || $domain->isTest()) {
Console::log('🔴 CNAME target has no public suffix (' . $domain->get() . ')');
} else {
Console::log('🟢 CNAME target has a public suffix (' . $domain->get() . ')');
}
if (App::getEnv('_APP_OPENSSL_KEY_V1') === 'your-secret-key' || empty(App::getEnv('_APP_OPENSSL_KEY_V1'))) {
Console::log('🔴 Not using a unique secret key for encryption');
} else {
Console::log('🟢 Using a unique secret key for encryption');
}
if (App::getEnv('_APP_ENV', 'development') !== 'production') {
Console::log('🔴 App environment is set for development');
} else {
Console::log('🟢 App environment is set for production');
}
if ('enabled' !== App::getEnv('_APP_OPTIONS_ABUSE', 'disabled')) {
Console::log('🔴 Abuse protection is disabled');
} else {
Console::log('🟢 Abuse protection is enabled');
}
$authWhitelistRoot = App::getEnv('_APP_CONSOLE_WHITELIST_ROOT', null);
$authWhitelistEmails = App::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null);
$authWhitelistIPs = App::getEnv('_APP_CONSOLE_WHITELIST_IPS', null);
if (
empty($authWhitelistRoot)
&& empty($authWhitelistEmails)
&& empty($authWhitelistIPs)
) {
Console::log('🔴 Console access limits are disabled');
} else {
Console::log('🟢 Console access limits are enabled');
}
if ('enabled' !== App::getEnv('_APP_OPTIONS_FORCE_HTTPS', 'disabled')) {
Console::log('🔴 HTTPS force option is disabled');
} else {
Console::log('🟢 HTTPS force option is enabled');
}
$providerName = App::getEnv('_APP_LOGGING_PROVIDER', '');
$providerConfig = App::getEnv('_APP_LOGGING_CONFIG', '');
if (empty($providerName) || empty($providerConfig) || !Logger::hasProvider($providerName)) {
Console::log('🔴 Logging adapter is disabled');
} else {
Console::log('🟢 Logging adapter is enabled (' . $providerName . ')');
}
\sleep(0.2);
try {
Console::log("\n" . '[Connectivity]');
} catch (\Throwable $th) {
//throw $th;
}
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$configs = [
'Console.DB' => Config::getParam('pools-console'),
'Projects.DB' => Config::getParam('pools-database'),
];
foreach ($configs as $key => $config) {
foreach ($config as $database) {
try {
$adapter = $pools->get($database)->pop()->getResource();
if ($adapter->ping()) {
Console::success('🟢 ' . str_pad("{$key}({$database})", 50, '.') . 'connected');
} else {
Console::error('🔴 ' . str_pad("{$key}({$database})", 47, '.') . 'disconnected');
}
} catch (\Throwable $th) {
Console::error('🔴 ' . str_pad("{$key}.({$database})", 47, '.') . 'disconnected');
}
}
}
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$configs = [
'Cache' => Config::getParam('pools-cache'),
'Queue' => Config::getParam('pools-queue'),
'PubSub' => Config::getParam('pools-pubsub'),
];
foreach ($configs as $key => $config) {
foreach ($config as $pool) {
try {
$adapter = $pools->get($pool)->pop()->getResource();
if ($adapter->ping()) {
Console::success('🟢 ' . str_pad("{$key}({$pool})", 50, '.') . 'connected');
} else {
Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected');
}
} catch (\Throwable $th) {
Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected');
}
}
}
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled') { // Check if scans are enabled
try {
$antivirus = new Network(
App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
(int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)
);
if ((@$antivirus->ping())) {
Console::success('🟢 ' . str_pad("Antivirus", 50, '.') . 'connected');
} else {
Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected');
}
} catch (\Throwable $th) {
Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected');
}
}
try {
$mail = $register->get('smtp'); /* @var $mail \PHPMailer\PHPMailer\PHPMailer */
$mail->addAddress('demo@example.com', 'Example.com');
$mail->Subject = 'Test SMTP Connection';
$mail->Body = 'Hello World';
$mail->AltBody = 'Hello World';
$mail->send();
Console::success('🟢 ' . str_pad("SMTP", 50, '.') . 'connected');
} catch (\Throwable $th) {
Console::error('🔴 ' . str_pad("SMTP", 47, '.') . 'disconnected');
}
\sleep(0.2);
Console::log('');
Console::log('[Volumes]');
foreach (
[
'Uploads' => APP_STORAGE_UPLOADS,
'Cache' => APP_STORAGE_CACHE,
'Config' => APP_STORAGE_CONFIG,
'Certs' => APP_STORAGE_CERTIFICATES
] as $key => $volume
) {
$device = new Local($volume);
if (\is_readable($device->getRoot())) {
Console::success('🟢 ' . $key . ' Volume is readable');
} else {
Console::error('🔴 ' . $key . ' Volume is unreadable');
}
if (\is_writable($device->getRoot())) {
Console::success('🟢 ' . $key . ' Volume is writeable');
} else {
Console::error('🔴 ' . $key . ' Volume is unwriteable');
}
}
\sleep(0.2);
Console::log('');
Console::log('[Disk Space]');
foreach (
[
'Uploads' => APP_STORAGE_UPLOADS,
'Cache' => APP_STORAGE_CACHE,
'Config' => APP_STORAGE_CONFIG,
'Certs' => APP_STORAGE_CERTIFICATES
] as $key => $volume
) {
$device = new Local($volume);
$percentage = (($device->getPartitionTotalSpace() - $device->getPartitionFreeSpace())
/ $device->getPartitionTotalSpace()) * 100;
$message = $key . ' Volume has ' . Storage::human($device->getPartitionFreeSpace()) . ' free space (' . \round($percentage, 2) . '% used)';
if ($percentage < 80) {
Console::success('🟢 ' . $message);
} else {
Console::error('🔴 ' . $message);
}
}
try {
if (App::isProduction()) {
Console::log('');
$version = \json_decode(@\file_get_contents(App::getEnv('_APP_HOME', 'http://localhost') . '/v1/health/version'), true);
if ($version && isset($version['version'])) {
if (\version_compare($version['version'], App::getEnv('_APP_VERSION', 'UNKNOWN')) === 0) {
Console::info('You are running the latest version of ' . APP_NAME . '! 🥳');
} else {
Console::info('A new version (' . $version['version'] . ') is available! 🥳' . "\n");
}
} else {
Console::error('Failed to check for a newer version' . "\n");
}
}
} catch (\Throwable $th) {
Console::error('Failed to check for a newer version' . "\n");
}
}
}
+444
View File
@@ -0,0 +1,444 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Network\Validator\Origin;
use Exception;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Analytics\Adapter\Mixpanel;
use Utopia\Analytics\Event;
use Utopia\Config\Config;
use Utopia\Database\Document;
use Utopia\Pools\Group;
class Hamster extends Action
{
private array $metrics = [
'usage_files' => 'files.$all.count.total',
'usage_buckets' => 'buckets.$all.count.total',
'usage_databases' => 'databases.$all.count.total',
'usage_documents' => 'documents.$all.count.total',
'usage_collections' => 'collections.$all.count.total',
'usage_storage' => 'project.$all.storage.size',
'usage_requests' => 'project.$all.network.requests',
'usage_bandwidth' => 'project.$all.network.bandwidth',
'usage_users' => 'users.$all.count.total',
'usage_sessions' => 'sessions.email.requests.create',
'usage_executions' => 'executions.$all.compute.total',
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
protected Mixpanel $mixpanel;
public static function getName(): string
{
return 'hamster';
}
public function __construct()
{
$this->mixpanel = new Mixpanel(App::getEnv('_APP_MIXPANEL_TOKEN', ''));
$this
->desc('Get stats for projects')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->callback(function (Group $pools, Cache $cache, Database $dbForConsole) {
$this->action($pools, $cache, $dbForConsole);
});
}
private function getStatsPerProject(Group $pools, Cache $cache, Database $dbForConsole)
{
$this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools, $cache) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
Console::info("Skipping project console");
return;
}
Console::log("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
$statsPerProject = [];
$statsPerProject['time'] = microtime(true);
/** Get Project ID */
$statsPerProject['project_id'] = $project->getId();
/** Get project created time */
$statsPerProject['project_created'] = $project->getAttribute('$createdAt');
/** Get Project Name */
$statsPerProject['project_name'] = $project->getAttribute('name');
/** Get Total Functions */
$statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);
foreach (\array_keys(Config::getParam('runtimes')) as $runtime) {
$statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [
Query::equal('runtime', [$runtime]),
], APP_LIMIT_COUNT);
}
/** Get Total Deployments */
$statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);
/** Get Total Teams */
$statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT);
/** Get Total Members */
$teamInternalId = $project->getAttribute('teamInternalId', null);
if ($teamInternalId) {
$statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [
Query::equal('teamInternalId', [$teamInternalId])
], APP_LIMIT_COUNT);
} else {
$statsPerProject['custom_organization_members'] = 0;
}
/** Get Email and Name of the project owner */
if ($teamInternalId) {
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$teamInternalId]),
]);
if (!$membership || $membership->isEmpty()) {
throw new Exception('Membership not found. Skipping project : ' . $project->getId());
}
$userInternalId = $membership->getAttribute('userInternalId', null);
if ($userInternalId) {
$user = $dbForConsole->findOne('users', [
Query::equal('_id', [$userInternalId]),
]);
$statsPerProject['email'] = $user->getAttribute('email', null);
$statsPerProject['name'] = $user->getAttribute('name', null);
}
}
/** Get Domains */
$statsPerProject['custom_domains'] = $dbForConsole->count('domains', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
/** Get Platforms */
$platforms = $dbForConsole->find('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
$statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) {
return $platform['type'] === 'web';
}));
$statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) {
return $platform['type'] === 'android';
}));
$statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) {
return str_contains($platform['type'], 'apple');
}));
$statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) {
return str_contains($platform['type'], 'flutter');
}));
$flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX];
foreach ($flutterPlatforms as $flutterPlatform) {
$statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) {
return $platform['type'] === $flutterPlatform;
}));
}
$statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
/** Get Usage $statsPerProject */
$periods = [
'infinity' => [
'period' => '1d',
'limit' => 90,
],
'24h' => [
'period' => '1h',
'limit' => 24,
],
];
Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) {
foreach ($this->metrics as $key => $metric) {
foreach ($periods as $periodKey => $periodValue) {
$limit = $periodValue['limit'];
$period = $periodValue['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$statsPerProject[$key . '_' . $periodKey] = [];
foreach ($requestDocs as $requestDoc) {
$statsPerProject[$key . '_' . $periodKey][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]);
// Calculate aggregate of each metric
$statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value'));
}
}
});
if (isset($statsPerProject['email'])) {
/** Send data to mixpanel */
$res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
'name' => $statsPerProject['name'],
'email' => $statsPerProject['email']
]);
if (!$res) {
Console::error('Failed to create user profile for project: ' . $project->getId());
}
$event = new Event();
$event
->setName('Project Daily Usage')
->setProps($statsPerProject);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
Console::error('Failed to create event for project: ' . $project->getId());
}
}
} catch (Exception $e) {
Console::error('Failed to send stats for project: ' . $project->getId());
Console::error($e->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
});
}
public function action(Group $pools, Cache $cache, Database $dbForConsole): void
{
Console::title('Cloud Hamster V1');
Console::success(APP_NAME . ' cloud hamster process has started');
$sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default)
$jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes)
$now = new \DateTime();
$now->setTimezone(new \DateTimeZone(date_default_timezone_get()));
$next = new \DateTime($now->format("Y-m-d $jobInitTime"));
$next->setTimezone(new \DateTimeZone(date_default_timezone_get()));
$delay = $next->getTimestamp() - $now->getTimestamp();
/**
* If time passed for the target day.
*/
if ($delay <= 0) {
$next->add(\DateInterval::createFromDateString('1 days'));
$delay = $next->getTimestamp() - $now->getTimestamp();
}
Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']');
Console::loop(function () use ($pools, $cache, $dbForConsole, $sleep) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Getting Cloud Usage Stats every {$sleep} seconds");
$loopStart = microtime(true);
/* Initialise new Utopia app */
$app = new App('UTC');
Console::info('Getting stats for all projects');
$this->getStatsPerProject($pools, $cache, $dbForConsole);
Console::success('Completed getting stats for all projects');
Console::info('Getting stats for all organizations');
$this->getStatsPerOrganization($dbForConsole);
Console::success('Completed getting stats for all organizations');
Console::info('Getting stats for all users');
$this->getStatsPerUser($dbForConsole);
Console::success('Completed getting stats for all users');
$pools
->get('console')
->reclaim();
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Cloud Stats took {$loopTook} seconds");
}, $sleep, $delay);
}
protected function calculateByGroup(string $collection, Database $dbForConsole, callable $callback)
{
$count = 0;
$chunk = 0;
$limit = 50;
$results = [];
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$chunk++;
$results = $dbForConsole->find($collection, \array_merge([
Query::limit($limit),
Query::offset($count)
]));
$sum = count($results);
Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents');
foreach ($results as $document) {
call_user_func($callback, $dbForConsole, $document);
$count++;
}
}
$executionEnd = \microtime(true);
Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
}
protected function getStatsPerOrganization(Database $dbForConsole)
{
$this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $document) {
try {
$statsPerOrganization = [];
/** Organization name */
$statsPerOrganization['name'] = $document->getAttribute('name');
/** Get Email and of the organization owner */
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$document->getInternalId()]),
]);
if (!$membership || $membership->isEmpty()) {
throw new Exception('Membership not found. Skipping organization : ' . $document->getId());
}
$userInternalId = $membership->getAttribute('userInternalId', null);
if ($userInternalId) {
$user = $dbForConsole->findOne('users', [
Query::equal('_id', [$userInternalId]),
]);
$statsPerOrganization['email'] = $user->getAttribute('email', null);
}
/** Organization Creation Date */
$statsPerOrganization['created'] = $document->getAttribute('$createdAt');
/** Number of team members */
$statsPerOrganization['members'] = $document->getAttribute('total');
/** Number of projects in this organization */
$statsPerOrganization['projects'] = $dbForConsole->count('projects', [
Query::equal('teamId', [$document->getId()]),
Query::limit(APP_LIMIT_COUNT)
]);
if (!isset($statsPerOrganization['email'])) {
throw new Exception('Email not found. Skipping organization : ' . $document->getId());
}
$event = new Event();
$event
->setName('Organization Daily Usage')
->setProps($statsPerOrganization);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
throw new Exception('Failed to create event for organization : ' . $document->getId());
}
} catch (Exception $e) {
Console::error($e->getMessage());
}
});
}
protected function getStatsPerUser(Database $dbForConsole)
{
$this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $document) {
try {
$statsPerUser = [];
/** Organization name */
$statsPerUser['name'] = $document->getAttribute('name');
/** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */
$statsPerUser['email'] = $document->getAttribute('email');
/** Organization Creation Date */
$statsPerUser['created'] = $document->getAttribute('$createdAt');
/** Number of teams this user is a part of */
$statsPerUser['memberships'] = $dbForConsole->count('memberships', [
Query::equal('userInternalId', [$document->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
if (!isset($statsPerUser['email'])) {
throw new Exception('User has no email: ' . $document->getId());
}
/** Send data to mixpanel */
$event = new Event();
$event
->setName('User Daily Usage')
->setProps($statsPerUser);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
throw new Exception('Failed to create user profile for user: ' . $document->getId());
}
} catch (Exception $e) {
Console::error($e->getMessage());
}
});
}
}
+256
View File
@@ -0,0 +1,256 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Auth\Auth;
use Appwrite\Docker\Compose;
use Appwrite\Docker\Env;
use Appwrite\Utopia\View;
use Utopia\Analytics\GoogleAnalytics;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Validator\Text;
use Utopia\Platform\Action;
class Install extends Action
{
public static function getName(): string
{
return 'install';
}
public function __construct()
{
$this
->desc('Install Appwrite')
->param('httpPort', '', new Text(4), 'Server HTTP port', true)
->param('httpsPort', '', new Text(4), 'Server HTTPS port', true)
->param('organization', 'appwrite', new Text(0), 'Docker Registry organization', true)
->param('image', 'appwrite', new Text(0), 'Main appwrite docker image', true)
->param('interactive', 'Y', new Text(1), 'Run an interactive session', true)
->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive));
}
public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive): void
{
/**
* 1. Start - DONE
* 2. Check for older setup and get older version - DONE
* 2.1 If older version is equal or bigger(?) than current version, **stop setup**
* 2.2. Get ENV vars - DONE
* 2.2.1 Fetch from older docker-compose.yml file
* 2.2.2 Fetch from older .env file (manually parse)
* 2.3 Use old ENV vars as default values
* 2.4 Ask for all required vars not given as CLI args and if in interactive mode
* Otherwise, just use default vars. - DONE
* 3. Ask user to backup important volumes, env vars, and SQL tables
* In th future we can try and automate this for smaller/medium size setups
* 4. Drop new docker-compose.yml setup (located inside the container, no network dependencies with appwrite.io) - DONE
* 5. Run docker compose up -d - DONE
* 6. Run data migration
*/
$config = Config::getParam('variables');
$path = '/usr/src/code/appwrite';
$defaultHTTPPort = '80';
$defaultHTTPSPort = '443';
$vars = [];
/**
* We are using a random value every execution for identification.
* This allows us to collect information without invading the privacy of our users.
*/
$analytics = new GoogleAnalytics('UA-26264668-9', uniqid('server.', true));
foreach ($config as $category) {
foreach ($category['variables'] ?? [] as $var) {
$vars[] = $var;
}
}
Console::success('Starting Appwrite installation...');
// Create directory with write permissions
if (null !== $path && !\file_exists(\dirname($path))) {
if (!@\mkdir(\dirname($path), 0755, true)) {
Console::error('Can\'t create directory ' . \dirname($path));
Console::exit(1);
}
}
$data = @file_get_contents($path . '/docker-compose.yml');
if ($data !== false) {
$time = \time();
Console::info('Compose file found, creating backup: docker-compose.yml.' . $time . '.backup');
file_put_contents($path . '/docker-compose.yml.' . $time . '.backup', $data);
$compose = new Compose($data);
$appwrite = $compose->getService('appwrite');
$oldVersion = ($appwrite) ? $appwrite->getImageVersion() : null;
try {
$ports = $compose->getService('traefik')->getPorts();
} catch (\Throwable $th) {
$ports = [
$defaultHTTPPort => $defaultHTTPPort,
$defaultHTTPSPort => $defaultHTTPSPort
];
Console::warning('Traefik not found. Falling back to default ports.');
}
if ($oldVersion) {
foreach ($compose->getServices() as $service) { // Fetch all env vars from previous compose file
if (!$service) {
continue;
}
$env = $service->getEnvironment()->list();
foreach ($env as $key => $value) {
if (is_null($value)) {
continue;
}
foreach ($vars as $i => $var) {
if ($var['name'] === $key) {
$vars[$i]['default'] = $value;
}
}
}
}
$data = @file_get_contents($path . '/.env');
if ($data !== false) { // Fetch all env vars from previous .env file
Console::info('Env file found, creating backup: .env.' . $time . '.backup');
file_put_contents($path . '/.env.' . $time . '.backup', $data);
$env = new Env($data);
foreach ($env->list() as $key => $value) {
if (is_null($value)) {
continue;
}
foreach ($vars as $i => $var) {
if ($var['name'] === $key) {
$vars[$i]['default'] = $value;
}
}
}
}
foreach ($ports as $key => $value) {
if ($value === $defaultHTTPPort) {
$defaultHTTPPort = $key;
}
if ($value === $defaultHTTPSPort) {
$defaultHTTPSPort = $key;
}
}
}
}
if (empty($httpPort)) {
$httpPort = Console::confirm('Choose your server HTTP port: (default: ' . $defaultHTTPPort . ')');
$httpPort = ($httpPort) ? $httpPort : $defaultHTTPPort;
}
if (empty($httpsPort)) {
$httpsPort = Console::confirm('Choose your server HTTPS port: (default: ' . $defaultHTTPSPort . ')');
$httpsPort = ($httpsPort) ? $httpsPort : $defaultHTTPSPort;
}
$input = [];
foreach ($vars as $var) {
if (!empty($var['filter']) && ($interactive !== 'Y' || !Console::isInteractive())) {
if ($data && $var['default'] !== null) {
$input[$var['name']] = $var['default'];
continue;
}
if ($var['filter'] === 'token') {
$input[$var['name']] = Auth::tokenGenerator();
continue;
}
if ($var['filter'] === 'password') {
$input[$var['name']] = Auth::passwordGenerator();
continue;
}
}
if (!$var['required'] || !Console::isInteractive() || $interactive !== 'Y') {
$input[$var['name']] = $var['default'];
continue;
}
$input[$var['name']] = Console::confirm($var['question'] . ' (default: \'' . $var['default'] . '\')');
if (empty($input[$var['name']])) {
$input[$var['name']] = $var['default'];
}
if ($var['filter'] === 'domainTarget') {
if ($input[$var['name']] !== 'localhost') {
Console::warning("\nIf you haven't already done so, set the following record for {$input[$var['name']]} on your DNS provider:\n");
$mask = "%-15.15s %-10.10s %-30.30s\n";
printf($mask, "Type", "Name", "Value");
printf($mask, "A or AAAA", "@", "<YOUR PUBLIC IP>");
Console::warning("\nUse 'AAAA' if you're using an IPv6 address and 'A' if you're using an IPv4 address.\n");
}
}
}
$templateForCompose = new View(__DIR__ . '/../views/install/compose.phtml');
$templateForEnv = new View(__DIR__ . '/../views/install/env.phtml');
$templateForCompose
->setParam('httpPort', $httpPort)
->setParam('httpsPort', $httpsPort)
->setParam('version', APP_VERSION_STABLE)
->setParam('organization', $organization)
->setParam('image', $image)
;
$templateForEnv
->setParam('vars', $input)
;
if (!file_put_contents($path . '/docker-compose.yml', $templateForCompose->render(false))) {
$message = 'Failed to save Docker Compose file';
$analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
Console::error($message);
Console::exit(1);
}
if (!file_put_contents($path . '/.env', $templateForEnv->render(false))) {
$message = 'Failed to save environment variables file';
$analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
Console::error($message);
Console::exit(1);
}
$env = '';
$stdout = '';
$stderr = '';
foreach ($input as $key => $value) {
if ($value) {
$env .= $key . '=' . \escapeshellarg($value) . ' ';
}
}
Console::log("Running \"docker compose up -d --remove-orphans --renew-anon-volumes\"");
$exit = Console::execute("${env} docker compose --project-directory {$path} up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr);
if ($exit !== 0) {
$message = 'Failed to install Appwrite dockers';
$analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
Console::error($message);
Console::error($stderr);
Console::exit($exit);
} else {
$message = 'Appwrite installed successfully';
$analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
Console::success($message);
}
}
}
+153
View File
@@ -0,0 +1,153 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Auth\Auth;
use Appwrite\Event\Certificate;
use Appwrite\Event\Delete;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\DateTime;
use Utopia\Database\Query;
use Utopia\Platform\Action;
class Maintenance extends Action
{
public static function getName(): string
{
return 'maintenance';
}
public function __construct()
{
$this
->desc('Schedules maintenance tasks and publishes them to resque')
->inject('dbForConsole')
->callback(fn (Database $dbForConsole) => $this->action($dbForConsole));
}
public function action(Database $dbForConsole): void
{
Console::title('Maintenance V1');
Console::success(APP_NAME . ' maintenance process v1 has started');
function notifyDeleteExecutionLogs(int $interval)
{
(new Delete())
->setType(DELETE_TYPE_EXECUTIONS)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
}
function notifyDeleteAbuseLogs(int $interval)
{
(new Delete())
->setType(DELETE_TYPE_ABUSE)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
}
function notifyDeleteAuditLogs(int $interval)
{
(new Delete())
->setType(DELETE_TYPE_AUDIT)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
}
function notifyDeleteUsageStats(int $usageStatsRetentionHourly)
{
(new Delete())
->setType(DELETE_TYPE_USAGE)
->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly))
->trigger();
}
function notifyDeleteConnections()
{
(new Delete())
->setType(DELETE_TYPE_REALTIME)
->setDatetime(DateTime::addSeconds(new \DateTime(), -60))
->trigger();
}
function notifyDeleteExpiredSessions()
{
(new Delete())
->setType(DELETE_TYPE_SESSIONS)
->trigger();
}
function renewCertificates($dbForConsole)
{
$time = DateTime::now();
$certificates = $dbForConsole->find('certificates', [
Query::lessThan('attempts', 5), // Maximum 5 attempts
Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew)
Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains)
]);
if (\count($certificates) > 0) {
Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs.");
$event = new Certificate();
foreach ($certificates as $certificate) {
$event
->setDomain(new Document([
'domain' => $certificate->getAttribute('domain')
]))
->trigger();
}
} else {
Console::info("[{$time}] No certificates for renewal.");
}
}
function notifyDeleteCache($interval)
{
(new Delete())
->setType(DELETE_TYPE_CACHE_BY_TIMESTAMP)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
}
function notifyDeleteSchedules($interval)
{
(new Delete())
->setType(DELETE_TYPE_SCHEDULES)
->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
->trigger();
}
// # of days in seconds (1 day = 86400s)
$interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
$executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
$auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600');
$abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400');
$usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
$schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole) {
$time = DateTime::now();
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
notifyDeleteExecutionLogs($executionLogsRetention);
notifyDeleteAbuseLogs($abuseLogsRetention);
notifyDeleteAuditLogs($auditLogRetention);
notifyDeleteUsageStats($usageStatsRetentionHourly);
notifyDeleteConnections();
notifyDeleteExpiredSessions();
renewCertificates($dbForConsole);
notifyDeleteCache($cacheRetention);
notifyDeleteSchedules($schedulesDeletionRetention);
}, $interval);
}
}
+121
View File
@@ -0,0 +1,121 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Appwrite\Migration\Migration;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\Registry\Registry;
use Utopia\Validator\Text;
class Migrate extends Action
{
public static function getName(): string
{
return 'migrate';
}
public function __construct()
{
$this
->desc('Migrate Appwrite to new version')
/** @TODO APP_VERSION_STABLE needs to be defined */
->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true)
->inject('register')
->callback(fn ($version, $register) => $this->action($version, $register));
}
private function clearProjectsCache(Redis $redis, Document $project)
{
try {
$redis->del($redis->keys("cache-_{$project->getInternalId()}:*"));
} catch (\Throwable $th) {
Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage());
}
}
public function action(string $version, Registry $register)
{
Authorization::disable();
if (!array_key_exists($version, Migration::$versions)) {
Console::error("Version {$version} not found.");
Console::exit(1);
return;
}
$app = new App('UTC');
Console::success('Starting Data Migration to version ' . $version);
$dbPool = $register->get('dbPool', true);
$redis = $register->get('cache', true);
$cache = new Cache(new RedisCache($redis));
$dbForConsole = $dbPool->getDB('console', $cache);
$dbForConsole->setNamespace('_project_console');
$console = $app->getResource('console');
$limit = 30;
$sum = 30;
$offset = 0;
/**
* @var \Utopia\Database\Document[] $projects
*/
$projects = [$console];
$count = 0;
try {
$totalProjects = $dbForConsole->count('projects') + 1;
} catch (\Throwable $th) {
$dbForConsole->setNamespace('_console');
$totalProjects = $dbForConsole->count('projects') + 1;
}
$class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
$migration = new $class();
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console' && $project->getInternalId() !== 'console') {
continue;
}
$this->clearProjectsCache($redis, $project);
try {
// TODO: Iterate through all project DBs
$projectDB = $dbPool->getDB($project->getId(), $cache);
$migration
->setProject($project, $projectDB, $dbForConsole)
->execute();
} catch (\Throwable $th) {
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
throw $th;
}
$this->clearProjectsCache($redis, $project);
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', limit: $limit, offset: $offset);
$offset = $offset + $limit;
$count = $count + $sum;
Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...');
}
Swoole\Event::wait(); // Wait for Coroutines to finish
Console::success('Data Migration Completed');
}
}
@@ -0,0 +1,97 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\Authorization;
class PatchCreateMissingSchedules extends Action
{
public static function getName(): string
{
return 'patch-create-missing-schedules';
}
public function __construct()
{
$this
->desc('Ensure every function has a schedule')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB));
}
/**
* Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule.
*/
public function action(Database $dbForConsole, callable $getProjectDB): void
{
Authorization::disable();
Authorization::setDefaultStatus(false);
Console::title('PatchCreateMissingSchedules V1');
Console::success(APP_NAME . ' PatchCreateMissingSchedules v1 has started');
$limit = 100;
$projectCursor = null;
while (true) {
$projectsQueries = [Query::limit($limit)];
if ($projectCursor !== null) {
$projectsQueries[] = Query::cursorAfter($projectCursor);
}
$projects = $dbForConsole->find('projects', $projectsQueries);
if (count($projects) === 0) {
break;
}
foreach ($projects as $project) {
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")");
$dbForProject = $getProjectDB($project);
$functionCursor = null;
while (true) {
$functionsQueries = [Query::limit($limit)];
if ($functionCursor !== null) {
$functionsQueries[] = Query::cursorAfter($functionCursor);
}
$functions = $dbForProject->find('functions', $functionsQueries);
if (count($functions) === 0) {
break;
}
foreach ($functions as $function) {
$scheduleId = $function->getAttribute('scheduleId');
$schedule = $dbForConsole->getDocument('schedules', $scheduleId);
if ($schedule->isEmpty()) {
$functionId = $function->getId();
$schedule = $dbForConsole->createDocument('schedules', new Document([
'$id' => ID::custom($scheduleId),
'region' => $project->getAttribute('region', 'default'),
'resourceType' => 'function',
'resourceId' => $functionId,
'resourceUpdatedAt' => DateTime::now(),
'projectId' => $project->getId(),
'schedule' => $function->getAttribute('schedule'),
'active' => !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')),
]));
Console::success('Recreated schedule for function ' . $functionId);
}
}
$functionCursor = $functions[array_key_last($functions)];
}
}
$projectCursor = $projects[array_key_last($projects)];
}
}
}
@@ -0,0 +1,129 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Pools\Group;
use Utopia\Validator\Numeric;
class PatchDeleteProjectCollections extends Action
{
private array $names = [
'webhooks',
'platforms',
'schedules',
'projects',
'domains',
'certificates',
'keys',
'realtime',
];
public static function getName(): string
{
return 'patch-delete-project-collections';
}
public function __construct()
{
$this
->desc('Delete unnecessary project collections')
->param('offset', 0, new Numeric(), 'Resume deletion from param pos', true)
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->callback(function (int $offset, Group $pools, Cache $cache, Database $dbForConsole) {
$this->action($offset, $pools, $cache, $dbForConsole);
});
}
public function action(int $offset, Group $pools, Cache $cache, Database $dbForConsole): void
{
//docker compose exec -t appwrite patch-delete-project-collections
Console::title('Delete project collections V1');
Console::success(APP_NAME . ' delete project collections has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 50;
$sum = 50;
$offset = $offset;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Deleting collections for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$dbForProject->setNamespace('_' . $project->getInternalId());
foreach ($this->names as $name) {
if (empty($name)) {
continue;
}
if ($dbForProject->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) {
if ($dbForProject->deleteCollection($name)) {
Console::log('Deleted ' . $name);
} else {
Console::error('Failed to delete ' . $name);
}
}
}
} catch (\Throwable $th) {
Console::error('Failed on project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
if (!empty($projects)) {
Console::log('Querying..... offset=' . $offset . ' , limit=' . $limit . ', count=' . $count);
}
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$pools
->get('console')
->reclaim();
}
}
@@ -0,0 +1,74 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Database\Validator\Authorization;
use Utopia\Pools\Group;
class PatchDeleteScheduleUpdatedAtAttribute extends Action
{
public static function getName(): string
{
return 'patch-delete-schedule-updated-at-attribute';
}
public function __construct()
{
$this
->desc('Ensure function collections do not have scheduleUpdatedAt attribute')
->inject('pools')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB));
}
/**
* Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule.
*/
public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void
{
Authorization::disable();
Authorization::setDefaultStatus(false);
Console::title('PatchDeleteScheduleUpdatedAtAttribute V1');
Console::success(APP_NAME . ' PatchDeleteScheduleUpdatedAtAttribute v1 has started');
$limit = 100;
$projectCursor = null;
while (true) {
$projectsQueries = [Query::limit($limit)];
if ($projectCursor !== null) {
$projectsQueries[] = Query::cursorAfter($projectCursor);
}
$projects = $dbForConsole->find('projects', $projectsQueries);
if (count($projects) === 0) {
break;
}
foreach ($projects as $project) {
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")");
$dbForProject = $getProjectDB($project);
try {
/**
* Delete 'scheduleUpdatedAt' attribute
*/
$dbForProject->deleteAttribute('functions', 'scheduleUpdatedAt');
$dbForProject->deleteCachedCollection('functions');
Console::success("'scheduleUpdatedAt' deleted.");
} catch (\Throwable $th) {
Console::warning("'scheduleUpdatedAt' errored: {$th->getMessage()}");
}
$pools->reclaim();
}
$projectCursor = $projects[array_key_last($projects)];
}
}
}
+294
View File
@@ -0,0 +1,294 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Platform\Action;
use Appwrite\SDK\Language\Android;
use Appwrite\SDK\Language\CLI;
use Appwrite\SDK\Language\Dart;
use Appwrite\SDK\Language\Deno;
use Appwrite\SDK\Language\DotNet;
use Appwrite\SDK\Language\Flutter;
use Appwrite\SDK\Language\Go;
use Appwrite\SDK\Language\GraphQL;
use Appwrite\SDK\Language\Kotlin;
use Appwrite\SDK\Language\Node;
use Appwrite\SDK\Language\PHP;
use Appwrite\SDK\Language\Python;
use Appwrite\SDK\Language\REST;
use Appwrite\SDK\Language\Ruby;
use Appwrite\SDK\Language\Swift;
use Exception;
use Throwable;
use Appwrite\SDK\Language\Apple;
use Appwrite\SDK\Language\Web;
use Appwrite\SDK\SDK;
use Appwrite\Spec\Swagger2;
use Utopia\CLI\Console;
use Utopia\Config\Config;
class SDKs extends Action
{
public static function getName(): string
{
return 'sdks';
}
public function __construct()
{
$this
->desc('Generate Appwrite SDKs')
->callback(fn () => $this->action());
}
public function action(): void
{
$platforms = Config::getParam('platforms');
$selectedPlatform = Console::confirm('Choose Platform ("' . APP_PLATFORM_CLIENT . '", "' . APP_PLATFORM_SERVER . '", "' . APP_PLATFORM_CONSOLE . '" or "*" for all):');
$selectedSDK = \strtolower(Console::confirm('Choose SDK ("*" for all):'));
$version = Console::confirm('Choose an Appwrite version');
$git = (Console::confirm('Should we use git push? (yes/no)') == 'yes');
$production = ($git) ? (Console::confirm('Type "Appwrite" to push code to production git repos') == 'Appwrite') : false;
$message = ($git) ? Console::confirm('Please enter your commit message:') : '';
if (!in_array($version, ['0.6.x', '0.7.x', '0.8.x', '0.9.x', '0.10.x', '0.11.x', '0.12.x', '0.13.x', '0.14.x', '0.15.x', '1.0.x', '1.1.x', '1.2.x', '1.3.x', 'latest'])) {
throw new Exception('Unknown version given');
}
foreach ($platforms as $key => $platform) {
if ($selectedPlatform !== $key && $selectedPlatform !== '*') {
continue;
}
foreach ($platform['sdks'] as $language) {
if ($selectedSDK !== $language['key'] && $selectedSDK !== '*') {
continue;
}
if (!$language['enabled']) {
Console::warning($language['name'] . ' for ' . $platform['name'] . ' is disabled');
continue;
}
Console::info('Fetching API Spec for ' . $language['name'] . ' for ' . $platform['name'] . ' (version: ' . $version . ')');
$spec = file_get_contents(__DIR__ . '/../../../../app/config/specs/swagger2-' . $version . '-' . $language['family'] . '.json');
$cover = 'https://appwrite.io/images/github.png';
$result = \realpath(__DIR__ . '/../../../../app') . '/sdks/' . $key . '-' . $language['key'];
$resultExamples = \realpath(__DIR__ . '/../../../..') . '/docs/examples/' . $version . '/' . $key . '-' . $language['key'];
$target = \realpath(__DIR__ . '/../../../../app') . '/sdks/git/' . $language['key'] . '/';
$readme = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/README.md');
$readme = ($readme) ? \file_get_contents($readme) : '';
$gettingStarted = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md');
$gettingStarted = ($gettingStarted) ? \file_get_contents($gettingStarted) : '';
$examples = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/EXAMPLES.md');
$examples = ($examples) ? \file_get_contents($examples) : '';
$changelog = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/CHANGELOG.md');
$changelog = ($changelog) ? \file_get_contents($changelog) : '# Change Log';
$warning = '**This SDK is compatible with Appwrite server version ' . $version . '. For older versions, please check [previous releases](' . $language['url'] . '/releases).**';
$license = 'BSD-3-Clause';
$licenseContent = 'Copyright (c) ' . date('Y') . ' Appwrite (https://appwrite.io) and individual contributors.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.';
switch ($language['key']) {
case 'web':
$config = new Web();
if ($platform['key'] === APP_PLATFORM_CONSOLE) {
$config->setNPMPackage('@appwrite.io/console');
$config->setBowerPackage('@appwrite.io/console');
} else {
$config->setNPMPackage('appwrite');
$config->setBowerPackage('appwrite');
}
break;
case 'cli':
$config = new CLI();
$config->setNPMPackage('appwrite-cli');
$config->setExecutableName('appwrite');
$config->setLogo(json_encode("
_ _ _ ___ __ _____
/_\ _ __ _ ____ ___ __(_) |_ ___ / __\ / / \_ \
//_\\\| '_ \| '_ \ \ /\ / / '__| | __/ _ \ / / / / / /\/
/ _ \ |_) | |_) \ V V /| | | | || __/ / /___/ /___/\/ /_
\_/ \_/ .__/| .__/ \_/\_/ |_| |_|\__\___| \____/\____/\____/
|_| |_|
"));
$config->setLogoUnescaped("
_ _ _ ___ __ _____
/_\ _ __ _ ____ ___ __(_) |_ ___ / __\ / / \_ \
//_\\\| '_ \| '_ \ \ /\ / / '__| | __/ _ \ / / / / / /\/
/ _ \ |_) | |_) \ V V /| | | | || __/ / /___/ /___/\/ /_
\_/ \_/ .__/| .__/ \_/\_/ |_| |_|\__\___| \____/\____/\____/
|_| |_| ");
break;
case 'php':
$config = new PHP();
$config->setComposerVendor('appwrite');
$config->setComposerPackage('appwrite');
break;
case 'nodejs':
$config = new Node();
$config->setNPMPackage('node-appwrite');
$config->setBowerPackage('appwrite');
$warning = $warning . "\n\n > This is the Node.js SDK for integrating with Appwrite from your Node.js server-side code.
If you're looking to integrate from the browser, you should check [appwrite/sdk-for-web](https://github.com/appwrite/sdk-for-web)";
break;
case 'deno':
$config = new Deno();
break;
case 'python':
$config = new Python();
$config->setPipPackage('appwrite');
$license = 'BSD License'; // license edited due to classifiers in pypi
break;
case 'ruby':
$config = new Ruby();
$config->setGemPackage('appwrite');
break;
case 'flutter':
$config = new Flutter();
$config->setPackageName('appwrite');
break;
case 'flutter-dev':
$config = new Flutter();
$config->setPackageName('appwrite_dev');
break;
case 'dart':
$config = new Dart();
$config->setPackageName('dart_appwrite');
$warning = $warning . "\n\n > This is the Dart SDK for integrating with Appwrite from your Dart server-side code. If you're looking for the Flutter SDK you should check [appwrite/sdk-for-flutter](https://github.com/appwrite/sdk-for-flutter)";
break;
case 'go':
$config = new Go();
break;
case 'swift':
$config = new Swift();
$warning = $warning . "\n\n > This is the Swift SDK for integrating with Appwrite from your Swift server-side code. If you're looking for the Apple SDK you should check [appwrite/sdk-for-apple](https://github.com/appwrite/sdk-for-apple)";
break;
case 'apple':
$config = new Apple();
break;
case 'dotnet':
$cover = '';
$config = new DotNet();
break;
case 'android':
$config = new Android();
break;
case 'kotlin':
$config = new Kotlin();
$warning = $warning . "\n\n > This is the Kotlin SDK for integrating with Appwrite from your Kotlin server-side code. If you're looking for the Android SDK you should check [appwrite/sdk-for-android](https://github.com/appwrite/sdk-for-android)";
break;
case 'graphql':
$config = new GraphQL();
break;
case 'rest':
$config = new REST();
break;
default:
throw new Exception('Language "' . $language['key'] . '" not supported');
}
Console::info("Generating {$language['name']} SDK...");
$sdk = new SDK($config, new Swagger2($spec));
$sdk
->setName($language['name'])
->setNamespace('io appwrite')
->setDescription("Appwrite is an open-source backend as a service server that abstract and simplify complex and repetitive development tasks behind a very simple to use REST API. Appwrite aims to help you develop your apps faster and in a more secure way. Use the {$language['name']} SDK to integrate your app with the Appwrite server to easily start interacting with all of Appwrite backend APIs and tools. For full API documentation and tutorials go to [https://appwrite.io/docs](https://appwrite.io/docs)")
->setShortDescription('Appwrite is an open-source self-hosted backend server that abstract and simplify complex and repetitive development tasks behind a very simple REST API')
->setLicense($license)
->setLicenseContent($licenseContent)
->setVersion($language['version'])
->setPlatform($key)
->setGitURL($language['url'])
->setGitRepo($language['gitUrl'])
->setGitRepoName($language['gitRepoName'])
->setGitUserName($language['gitUserName'])
->setLogo($cover)
->setURL('https://appwrite.io')
->setShareText('Appwrite is a backend as a service for building web or mobile apps')
->setShareURL('http://appwrite.io')
->setShareTags('JS,javascript,reactjs,angular,ios,android,serverless')
->setShareVia('appwrite')
->setWarning($warning)
->setReadme($readme)
->setGettingStarted($gettingStarted)
->setChangelog($changelog)
->setExamples($examples)
->setTwitter(APP_SOCIAL_TWITTER_HANDLE)
->setDiscord(APP_SOCIAL_DISCORD_CHANNEL, APP_SOCIAL_DISCORD)
->setDefaultHeaders([
'X-Appwrite-Response-Format' => '1.0.0',
]);
try {
$sdk->generate($result);
} catch (Exception $exception) {
Console::error($exception->getMessage());
} catch (Throwable $exception) {
Console::error($exception->getMessage());
}
$gitUrl = $language['gitUrl'];
$gitBranch = $language['gitBranch'];
if (!$production) {
$gitUrl = 'git@github.com:aw-tests/' . $language['gitRepoName'] . '.git';
}
if ($git && !empty($gitUrl)) {
\exec('rm -rf ' . $target . ' && \
mkdir -p ' . $target . ' && \
cd ' . $target . ' && \
git init --initial-branch=' . $gitBranch . ' && \
git remote add origin ' . $gitUrl . ' && \
git fetch origin ' . $gitBranch . ' && \
git pull origin ' . $gitBranch . ' && \
rm -rf ' . $target . '/* && \
cp -r ' . $result . '/* ' . $target . '/ && \
git add . && \
git commit -m "' . $message . '" && \
git push -u origin ' . $gitBranch . '
');
Console::success("Pushed {$language['name']} SDK to {$gitUrl}");
\exec('rm -rf ' . $target);
Console::success("Remove temp directory '{$target}' for {$language['name']} SDK");
}
$docDirectories = $language['docDirectories'] ?? [''];
if ($version === 'latest') {
continue;
}
foreach ($docDirectories as $languageTitle => $path) {
$languagePath = strtolower($languageTitle !== 0 ? '/' . $languageTitle : '');
\exec(
'mkdir -p ' . $resultExamples . $languagePath . ' && \
cp -r ' . $result . '/docs/examples' . $languagePath . ' ' . $resultExamples
);
Console::success("Copied code examples for {$language['name']} SDK to: {$resultExamples}");
}
}
}
Console::exit();
}
}
+38
View File
@@ -0,0 +1,38 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Platform\Action;
use Appwrite\Event\Certificate;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Document;
use Utopia\Validator\Hostname;
class SSL extends Action
{
public static function getName(): string
{
return 'ssl';
}
public function __construct()
{
$this
->desc('Validate server certificates')
->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true)
->callback(fn ($domain) => $this->action($domain));
}
public function action(string $domain): void
{
Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain);
(new Certificate())
->setDomain(new Document([
'domain' => $domain
]))
->setSkipRenewCheck(true)
->trigger();
}
}
+237
View File
@@ -0,0 +1,237 @@
<?php
namespace Appwrite\Platform\Tasks;
use Cron\CronExpression;
use Swoole\Timer;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Pools\Group;
use Appwrite\Event\Func;
use function Swoole\Coroutine\run;
class Schedule extends Action
{
public const FUNCTION_UPDATE_TIMER = 10; //seconds
public const FUNCTION_ENQUEUE_TIMER = 60; //seconds
public static function getName(): string
{
return 'schedule';
}
public function __construct()
{
$this
->desc('Execute functions scheduled in Appwrite')
->inject('pools')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB));
}
/**
* 1. Load all documents from 'schedules' collection to create local copy
* 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute
* 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutime sleeps until exact time before sending request to worker.
*/
public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void
{
Console::title('Scheduler V1');
Console::success(APP_NAME . ' Scheduler v1 has started');
/**
* Extract only nessessary attributes to lower memory used.
*
* @var Document $schedule
* @return array
*/
$getSchedule = function (Document $schedule) use ($dbForConsole, $getProjectDB): array {
$project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId'));
$function = $getProjectDB($project)->getDocument('functions', $schedule->getAttribute('resourceId'));
return [
'resourceId' => $schedule->getAttribute('resourceId'),
'schedule' => $schedule->getAttribute('schedule'),
'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'),
'project' => $project, // TODO: @Meldiron Send only ID to worker to reduce memory usage here
'function' => $function, // TODO: @Meldiron Send only ID to worker to reduce memory usage here
];
};
$schedules = []; // Local copy of 'schedules' collection
$lastSyncUpdate = DateTime::now();
$limit = 10000;
$sum = $limit;
$total = 0;
$loadStart = \microtime(true);
$latestDocument = null;
while ($sum === $limit) {
$paginationQueries = [Query::limit($limit)];
if ($latestDocument !== null) {
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
$results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', ['function']),
Query::equal('active', [true]),
]));
$sum = count($results);
$total = $total + $sum;
foreach ($results as $document) {
$schedules[$document['resourceId']] = $getSchedule($document);
}
$latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null;
}
$pools->reclaim();
Console::success("{$total} functions were loaded in " . (microtime(true) - $loadStart) . " seconds");
Console::success("Starting timers at " . DateTime::now());
run(
function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule, $pools) {
/**
* The timer synchronize $schedules copy with database collection.
*/
Timer::tick(self::FUNCTION_UPDATE_TIMER * 1000, function () use ($dbForConsole, &$schedules, &$lastSyncUpdate, $getSchedule, $pools) {
$time = DateTime::now();
$timerStart = \microtime(true);
$limit = 1000;
$sum = $limit;
$total = 0;
$latestDocument = null;
Console::log("Sync tick: Running at $time");
while ($sum === $limit) {
$paginationQueries = [Query::limit($limit)];
if ($latestDocument !== null) {
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
$results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', ['function']),
Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate),
]));
$sum = count($results);
$total = $total + $sum;
foreach ($results as $document) {
$localDocument = $schedules[$document['resourceId']] ?? null;
$org = $localDocument !== null ? strtotime($localDocument['resourceUpdatedAt']) : null;
$new = strtotime($document['resourceUpdatedAt']);
if ($document['active'] === false) {
Console::info("Removing: {$document['resourceId']}");
unset($schedules[$document['resourceId']]);
} elseif ($new !== $org) {
Console::info("Updating: {$document['resourceId']}");
$schedules[$document['resourceId']] = $getSchedule($document);
}
}
$latestDocument = !empty(array_key_last($results)) ? $results[array_key_last($results)] : null;
}
$lastSyncUpdate = $time;
$timerEnd = \microtime(true);
$pools->reclaim();
Console::log("Sync tick: {$total} schedules were updated in " . ($timerEnd - $timerStart) . " seconds");
});
/**
* The timer to prepare soon-to-execute schedules.
*/
$lastEnqueueUpdate = null;
$enqueueFunctions = function () use (&$schedules, $lastEnqueueUpdate, $pools) {
$timerStart = \microtime(true);
$time = DateTime::now();
$enqueueDiff = $lastEnqueueUpdate === null ? 0 : $timerStart - $lastEnqueueUpdate;
$timeFrame = DateTime::addSeconds(new \DateTime(), self::FUNCTION_ENQUEUE_TIMER - $enqueueDiff);
Console::log("Enqueue tick: started at: $time (with diff $enqueueDiff)");
$total = 0;
$delayedExecutions = []; // Group executions with same delay to share one coroutine
foreach ($schedules as $key => $schedule) {
$cron = new CronExpression($schedule['schedule']);
$nextDate = $cron->getNextRunDate();
$next = DateTime::format($nextDate);
$currentTick = $next < $timeFrame;
if (!$currentTick) {
continue;
}
$total++;
$promiseStart = \time(); // in seconds
$executionStart = $nextDate->getTimestamp(); // in seconds
$delay = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued
if (!isset($delayedExecutions[$delay])) {
$delayedExecutions[$delay] = [];
}
$delayedExecutions[$delay][] = $key;
}
foreach ($delayedExecutions as $delay => $scheduleKeys) {
\go(function () use ($delay, $schedules, $scheduleKeys, $pools) {
\sleep($delay); // in seconds
$queue = $pools->get('queue')->pop();
$connection = $queue->getResource();
foreach ($scheduleKeys as $scheduleKey) {
// Ensure schedule was not deleted
if (!isset($schedules[$scheduleKey])) {
return;
}
$schedule = $schedules[$scheduleKey];
$functions = new Func($connection);
$functions
->setType('schedule')
->setFunction($schedule['function'])
->setProject($schedule['project'])
->trigger();
}
$queue->reclaim();
});
}
$timerEnd = \microtime(true);
$lastEnqueueUpdate = $timerStart;
Console::log("Enqueue tick: {$total} executions were enqueued in " . ($timerEnd - $timerStart) . " seconds");
};
Timer::tick(self::FUNCTION_ENQUEUE_TIMER * 1000, fn() => $enqueueFunctions());
$enqueueFunctions();
}
);
}
}
+279
View File
@@ -0,0 +1,279 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Platform\Action;
use Utopia\Validator\Text;
use Appwrite\Specification\Format\OpenAPI3;
use Appwrite\Specification\Format\Swagger2;
use Appwrite\Specification\Specification;
use Appwrite\Utopia\Response;
use Exception;
use Swoole\Http\Response as HttpResponse;
use Utopia\App;
use Utopia\Cache\Adapter\None;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Adapter\MySQL;
use Utopia\Database\Database;
use Utopia\Registry\Registry;
use Utopia\Request;
use Utopia\Validator\WhiteList;
class Specs extends Action
{
public static function getName(): string
{
return 'specs';
}
public function __construct()
{
$this
->desc('Generate Appwrite API specifications')
->param('version', 'latest', new Text(16), 'Spec version', true)
->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true)
->inject('register')
->callback(fn (string $version, string $mode, Registry $register) => $this->action($version, $mode, $register));
}
public function action(string $version, string $mode, Registry $register): void
{
//$db = $register->get('db');
//$redis = $register->get('cache');
$appRoutes = App::getRoutes();
$response = new Response(new HttpResponse());
$mocks = ($mode === 'mocks');
// Mock dependencies
App::setResource('request', fn () => new Request());
App::setResource('response', fn () => $response);
App::setResource('dbForConsole', fn () => new Database(new MySQL(''), new Cache(new None())));
App::setResource('dbForProject', fn () => new Database(new MySQL(''), new Cache(new None())));
$platforms = [
'client' => APP_PLATFORM_CLIENT,
'server' => APP_PLATFORM_SERVER,
'console' => APP_PLATFORM_CONSOLE,
];
$authCounts = [
'client' => 1,
'server' => 2,
'console' => 1,
];
$keys = [
APP_PLATFORM_CLIENT => [
'Project' => [
'type' => 'apiKey',
'name' => 'X-Appwrite-Project',
'description' => 'Your project ID',
'in' => 'header',
],
'JWT' => [
'type' => 'apiKey',
'name' => 'X-Appwrite-JWT',
'description' => 'Your secret JSON Web Token',
'in' => 'header',
],
'Locale' => [
'type' => 'apiKey',
'name' => 'X-Appwrite-Locale',
'description' => '',
'in' => 'header',
],
],
APP_PLATFORM_SERVER => [
'Project' => [
'type' => 'apiKey',
'name' => 'X-Appwrite-Project',
'description' => 'Your project ID',
'in' => 'header',
],
'Key' => [
'type' => 'apiKey',
'name' => 'X-Appwrite-Key',
'description' => 'Your secret API key',
'in' => 'header',
],
'JWT' => [
'type' => 'apiKey',
'name' => 'X-Appwrite-JWT',
'description' => 'Your secret JSON Web Token',
'in' => 'header',
],
'Locale' => [
'type' => 'apiKey',
'name' => 'X-Appwrite-Locale',
'description' => '',
'in' => 'header',
],
],
APP_PLATFORM_CONSOLE => [
'Project' => [
'type' => 'apiKey',
'name' => 'X-Appwrite-Project',
'description' => 'Your project ID',
'in' => 'header',
],
'Key' => [
'type' => 'apiKey',
'name' => 'X-Appwrite-Key',
'description' => 'Your secret API key',
'in' => 'header',
],
'JWT' => [
'type' => 'apiKey',
'name' => 'X-Appwrite-JWT',
'description' => 'Your secret JSON Web Token',
'in' => 'header',
],
'Locale' => [
'type' => 'apiKey',
'name' => 'X-Appwrite-Locale',
'description' => '',
'in' => 'header',
],
'Mode' => [
'type' => 'apiKey',
'name' => 'X-Appwrite-Mode',
'description' => '',
'in' => 'header',
],
],
];
foreach ($platforms as $platform) {
$routes = [];
$models = [];
$services = [];
foreach ($appRoutes as $key => $method) {
foreach ($method as $route) {
/** @var \Utopia\Route $route */
$routeSecurity = $route->getLabel('sdk.auth', []);
$sdkPlaforms = [];
foreach ($routeSecurity as $value) {
switch ($value) {
case APP_AUTH_TYPE_SESSION:
$sdkPlaforms[] = APP_PLATFORM_CLIENT;
break;
case APP_AUTH_TYPE_KEY:
$sdkPlaforms[] = APP_PLATFORM_SERVER;
break;
case APP_AUTH_TYPE_JWT:
$sdkPlaforms[] = APP_PLATFORM_SERVER;
break;
case APP_AUTH_TYPE_ADMIN:
$sdkPlaforms[] = APP_PLATFORM_CONSOLE;
break;
}
}
if (empty($routeSecurity)) {
$sdkPlaforms[] = APP_PLATFORM_CLIENT;
}
if (!$route->getLabel('docs', true)) {
continue;
}
if ($route->getLabel('sdk.mock', false) && !$mocks) {
continue;
}
if (!$route->getLabel('sdk.mock', false) && $mocks) {
continue;
}
if (empty($route->getLabel('sdk.namespace', null))) {
continue;
}
if ($platform !== APP_PLATFORM_CONSOLE && !\in_array($platforms[$platform], $sdkPlaforms)) {
continue;
}
$routes[] = $route;
}
}
foreach (Config::getParam('services', []) as $service) {
if (
!isset($service['docs']) // Skip service if not part of the public API
|| !isset($service['sdk'])
|| !$service['docs']
|| !$service['sdk']
) {
continue;
}
$services[] = [
'name' => $service['key'] ?? '',
'description' => $service['subtitle'] ?? '',
'x-globalAttributes' => $service['globalAttributes'] ?? [],
];
}
$models = $response->getModels();
foreach ($models as $key => $value) {
if ($platform !== APP_PLATFORM_CONSOLE && !$value->isPublic()) {
unset($models[$key]);
}
}
$arguments = [new App('UTC'), $services, $routes, $models, $keys[$platform], $authCounts[$platform] ?? 0];
foreach (['swagger2', 'open-api3'] as $format) {
$formatInstance = match ($format) {
'swagger2' => new Swagger2(...$arguments),
'open-api3' => new OpenAPI3(...$arguments),
default => throw new Exception('Format not found: ' . $format)
};
$specs = new Specification($formatInstance);
$endpoint = App::getEnv('_APP_HOME', '[HOSTNAME]');
$email = App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM);
$formatInstance
->setParam('name', APP_NAME)
->setParam('description', 'Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https://appwrite.io/docs](https://appwrite.io/docs)')
->setParam('endpoint', 'https://HOSTNAME/v1')
->setParam('version', APP_VERSION_STABLE)
->setParam('terms', $endpoint . '/policy/terms')
->setParam('support.email', $email)
->setParam('support.url', $endpoint . '/support')
->setParam('contact.name', APP_NAME . ' Team')
->setParam('contact.email', $email)
->setParam('contact.url', $endpoint . '/support')
->setParam('license.name', 'BSD-3-Clause')
->setParam('license.url', 'https://raw.githubusercontent.com/appwrite/appwrite/master/LICENSE')
->setParam('docs.description', 'Full API docs, specs and tutorials')
->setParam('docs.url', $endpoint . '/docs');
if ($mocks) {
$path = __DIR__ . '/../config/specs/' . $format . '-mocks-' . $platform . '.json';
if (!file_put_contents($path, json_encode($specs->parse()))) {
throw new Exception('Failed to save mocks spec file: ' . $path);
}
Console::success('Saved mocks spec file: ' . realpath($path));
continue;
}
$path = __DIR__ . '/../../../../app/config/specs/' . $format . '-' . $version . '-' . $platform . '.json';
if (!file_put_contents($path, json_encode($specs->parse()))) {
throw new Exception('Failed to save spec file: ' . $path);
}
Console::success('Saved spec file: ' . realpath($path));
}
}
}
}
+39
View File
@@ -0,0 +1,39 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\App;
use Utopia\Config\Config;
use Utopia\CLI\Console;
use Utopia\Platform\Action;
class Vars extends Action
{
public static function getName(): string
{
return 'vars';
}
public function __construct()
{
$this
->desc('List all the server environment variables')
->callback(fn () => $this->action());
}
public function action(): void
{
$config = Config::getParam('variables', []);
$vars = [];
foreach ($config as $category) {
foreach ($category['variables'] ?? [] as $var) {
$vars[] = $var;
}
}
foreach ($vars as $key => $value) {
Console::log('- ' . $value['name'] . '=' . App::getEnv($value['name'], ''));
}
}
}
+24
View File
@@ -0,0 +1,24 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Platform\Action;
class Version extends Action
{
public static function getName(): string
{
return 'version';
}
public function __construct()
{
$this
->desc('Get the server version')
->callback(function () {
Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN'));
});
}
}
@@ -0,0 +1,59 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Platform\Action;
use Utopia\Validator\Integer;
use Utopia\Validator\Text;
class VolumeSync extends Action
{
public static function getName(): string
{
return 'volume-sync';
}
public function __construct()
{
$this
->desc('Runs rsync to sync certificates between the storage mount and traefik.')
->param('source', null, new Text(255), 'Source path to sync from.', false)
->param('destination', null, new Text(255), 'Destination path to sync to.', false)
->param('interval', null, new Integer(true), 'Interval to run rsync', false)
->callback(fn ($source, $destination, $interval) => $this->action($source, $destination, $interval));
}
public function action(string $source, string $destination, int $interval)
{
Console::title('RSync V1');
Console::success(APP_NAME . ' rsync process v1 has started');
if (!file_exists($source)) {
Console::error('Source directory does not exist. Exiting ... ');
Console::exit(0);
}
Console::loop(function () use ($interval, $source, $destination) {
$time = DateTime::now();
Console::info("[{$time}] Executing rsync every {$interval} seconds");
Console::info("Syncing between $source and $destination");
if (!file_exists($source)) {
Console::error('Source directory does not exist. Skipping ... ');
return;
}
$stdin = "";
$stdout = "";
$stderr = "";
Console::execute("rsync -av $source $destination", $stdin, $stdout, $stderr);
Console::success($stdout);
Console::error($stderr);
}, $interval);
}
}
+130 -117
View File
@@ -2,21 +2,24 @@
namespace Appwrite\Resque;
use Appwrite\Event\Usage;
use Exception;
use Utopia\App;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Cache\Adapter\Sharding;
use Utopia\CLI\Console;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\Pools\Group;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Backblaze;
use Utopia\Storage\Device\DOSpaces;
use Utopia\Storage\Device\Linode;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\S3;
use Utopia\Database\Validator\Authorization;
use Utopia\DSN\DSN;
use Utopia\Storage\Device\Wasabi;
use Utopia\Storage\Storage;
@@ -137,7 +140,12 @@ abstract class Worker
*/
public function tearDown(): void
{
global $register;
try {
$pools = $register->get('pools'); /** @var Group $pools */
$pools->reclaim();
$this->shutdown();
} catch (\Throwable $error) {
foreach (self::$errorCallbacks as $errorCallback) {
@@ -161,24 +169,43 @@ abstract class Worker
/**
* Get internal project database
* @param string $projectId
* @param Document $project
* @return Database
* @throws Exception
*/
protected function getProjectDB(string $projectId, ?Document $project = null): Database
protected static $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
protected function getProjectDB(Document $project): Database
{
if ($project === null) {
$consoleDB = $this->getConsoleDB();
global $register;
if ($projectId === 'console') {
return $consoleDB;
}
$pools = $register->get('pools'); /** @var Group $pools */
/** @var Document $project */
$project = Authorization::skip(fn() => $consoleDB->getDocument('projects', $projectId));
if ($project->isEmpty() || $project->getId() === 'console') {
return $this->getConsoleDB();
}
return $this->getDB(self::DATABASE_PROJECT, $projectId, $project->getInternalId(), $project);
$databaseName = $project->getAttribute('database');
if (isset(self::$databases[$databaseName])) {
$database = self::$databases[$databaseName];
$database->setNamespace('_' . $project->getInternalId());
return $database;
}
$dbAdapter = $pools
->get($project->getAttribute('database'))
->pop()
->getResource()
;
$database = new Database($dbAdapter, $this->getCache());
self::$databases[$databaseName] = $database;
$database->setNamespace('_' . $project->getInternalId());
return $database;
}
/**
@@ -188,81 +215,74 @@ abstract class Worker
*/
protected function getConsoleDB(): Database
{
return $this->getDB(self::DATABASE_CONSOLE);
global $register;
$pools = $register->get('pools'); /** @var Group $pools */
$databaseName = 'console';
if (isset(self::$databases[$databaseName])) {
$database = self::$databases[$databaseName];
$database->setNamespace('console');
return $database;
}
$dbAdapter = $pools
->get('console')
->pop()
->getResource()
;
$database = new Database($dbAdapter, $this->getCache());
self::$databases[$databaseName] = $database;
$database->setNamespace('console');
return $database;
}
/**
* Get Cache
* @return Cache
*/
protected function getCache(): Cache
{
global $register;
$pools = $register->get('pools'); /** @var Group $pools */
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource()
;
}
return new Cache(new Sharding($adapters));
}
/**
* Get database
* @param string $type One of (project, console)
* @param string $projectId of project or console DB
* @param string $projectInternalId
* @param Document|null $project
* @return Database
* Get usage queue
* @return Usage
* @throws Exception
*/
private function getDB(
string $type,
string $projectId = '',
string $projectInternalId = '',
?Document $project = null
): Database {
protected function getUsageQueue(): Usage
{
global $register;
$sleep = DATABASE_RECONNECT_SLEEP; // overwritten when necessary
$pools = $register->get('pools'); /** @var Group $pools */
$queue = $pools
->get('queue')
->pop()
->getResource();
if ($project !== null) {
$projectId = $project->getId();
$projectInternalId = $project->getInternalId();
}
switch ($type) {
case self::DATABASE_PROJECT:
if (!$projectId) {
throw new \Exception('ProjectID not provided - cannot get database');
}
$namespace = "_$projectInternalId";
break;
case self::DATABASE_CONSOLE:
$namespace = "_console";
$sleep = 5; // ConsoleDB needs extra sleep time to ensure tables are created
break;
default:
throw new \Exception('Unknown database type: ' . $type);
}
$attempts = 0;
while (true) {
try {
$attempts++;
$cache = new Cache(new RedisCache($register->get('cache')));
$database = new Database(new MariaDB($register->get('db')), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace($namespace); // Main DB
if (
$project === null
&& !empty($projectId)
&& !$database->getDocument('projects', $projectId)->isEmpty()
) {
throw new \Exception("Project does not exist: $projectId");
}
if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), Database::METADATA)) {
throw new \Exception('Console project not ready');
}
break; // leave loop if successful
} catch (\Exception $e) {
Console::warning("Database not ready. Retrying connection ($attempts)...");
if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
throw new \Exception('Failed to connect to database: ' . $e->getMessage());
}
sleep($sleep);
}
}
return $database;
return new Usage($queue);
}
/**
@@ -285,7 +305,6 @@ abstract class Worker
return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId);
}
/**
* Get Builds Storage Device
* @param string $projectId of the project
@@ -308,45 +327,39 @@ abstract class Worker
*/
public function getDevice(string $root): Device
{
switch (strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL))) {
$connection = App::getEnv('_APP_CONNECTIONS_STORAGE', '');
$acl = 'private';
$device = Storage::DEVICE_LOCAL;
$accessKey = '';
$accessSecret = '';
$bucket = '';
$region = '';
try {
$dsn = new DSN($connection);
$device = $dsn->getScheme();
$accessKey = $dsn->getUser();
$accessSecret = $dsn->getPassword();
$bucket = $dsn->getPath();
$region = $dsn->getParam('region');
} catch (\Exception $e) {
Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.');
}
switch ($device) {
case Storage::DEVICE_S3:
return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE::DEVICE_DO_SPACES:
return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_BACKBLAZE:
return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_LINODE:
return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_WASABI:
return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_LOCAL:
default:
return new Local($root);
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
case Storage::DEVICE_BACKBLAZE:
$backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
$backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', '');
$backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', '');
$backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', '');
$backblazeAcl = 'private';
return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl);
case Storage::DEVICE_LINODE:
$linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', '');
$linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', '');
$linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', '');
$linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', '');
$linodeAcl = 'private';
return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl);
case Storage::DEVICE_WASABI:
$wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', '');
$wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', '');
$wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', '');
$wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', '');
$wasabiAcl = 'private';
return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl);
}
}
}
@@ -343,6 +343,8 @@ class OpenAPI3 extends Format
case 'Utopia\Validator\ArrayList':
case 'Appwrite\Utopia\Database\Validator\Queries\Buckets':
case 'Appwrite\Utopia\Database\Validator\Queries\Collections':
case 'Appwrite\Utopia\Database\Validator\Queries\Indexes':
case 'Appwrite\Utopia\Database\Validator\Queries\Attributes':
case 'Appwrite\Utopia\Database\Validator\Queries\Databases':
case 'Appwrite\Utopia\Database\Validator\Queries\Deployments':
case 'Utopia\Database\Validator\Queries\Documents':
@@ -342,6 +342,8 @@ class Swagger2 extends Format
case 'Utopia\Validator\ArrayList':
case 'Appwrite\Utopia\Database\Validator\Queries\Buckets':
case 'Appwrite\Utopia\Database\Validator\Queries\Collections':
case 'Appwrite\Utopia\Database\Validator\Queries\Indexes':
case 'Appwrite\Utopia\Database\Validator\Queries\Attributes':
case 'Appwrite\Utopia\Database\Validator\Queries\Databases':
case 'Appwrite\Utopia\Database\Validator\Queries\Deployments':
case 'Utopia\Database\Validator\Queries\Documents':
-15
View File
@@ -1,15 +0,0 @@
<?php
namespace Appwrite\Usage;
abstract class Calculator
{
protected string $region;
public function __construct(string $region)
{
$this->region = $region;
}
abstract public function collect(): void;
}
@@ -1,539 +0,0 @@
<?php
namespace Appwrite\Usage\Calculators;
use Utopia\App;
use Appwrite\Usage\Calculator;
use Utopia\Database\Database;
use Utopia\Database\Document;
use InfluxDB\Database as InfluxDatabase;
use DateTime;
class TimeSeries extends Calculator
{
/**
* InfluxDB
*
* @var InfluxDatabase
*/
protected InfluxDatabase $influxDB;
/**
* Utopia Database
*
* @var Database
*/
protected Database $database;
/**
* Error Handler Callback
*
* @var callable
*/
protected $errorHandler;
/**
* Latest times for metric that was synced to the database
*
* @var array
*/
private array $latestTime = [];
/**
* Periods the metrics are collected for
* @var array
*/
protected array $periods = [
[
'key' => '1h',
'startTime' => '-24 hours'
],
[
'key' => '1d',
'startTime' => '-30 days'
]
];
/**
* All the metrics that we are collecting
*
* @var array
*/
protected array $metrics = [
'project.$all.network.requests' => [
'table' => 'appwrite_usage_project_{scope}_network_requests',
],
'project.$all.network.bandwidth' => [
'table' => 'appwrite_usage_project_{scope}_network_bandwidth',
],
'project.$all.network.inbound' => [
'table' => 'appwrite_usage_project_{scope}_network_inbound',
],
'project.$all.network.outbound' => [
'table' => 'appwrite_usage_project_{scope}_network_outbound',
],
/* Users service metrics */
'users.$all.requests.create' => [
'table' => 'appwrite_usage_users_{scope}_requests_create',
],
'users.$all.requests.read' => [
'table' => 'appwrite_usage_users_{scope}_requests_read',
],
'users.$all.requests.update' => [
'table' => 'appwrite_usage_users_{scope}_requests_update',
],
'users.$all.requests.delete' => [
'table' => 'appwrite_usage_users_{scope}_requests_delete',
],
'databases.$all.requests.create' => [
'table' => 'appwrite_usage_databases_{scope}_requests_create',
],
'databases.$all.requests.read' => [
'table' => 'appwrite_usage_databases_{scope}_requests_read',
],
'databases.$all.requests.update' => [
'table' => 'appwrite_usage_databases_{scope}_requests_update',
],
'databases.$all.requests.delete' => [
'table' => 'appwrite_usage_databases_{scope}_requests_delete',
],
'collections.$all.requests.create' => [
'table' => 'appwrite_usage_collections_{scope}_requests_create',
],
'collections.$all.requests.read' => [
'table' => 'appwrite_usage_collections_{scope}_requests_read',
],
'collections.$all.requests.update' => [
'table' => 'appwrite_usage_collections_{scope}_requests_update',
],
'collections.$all.requests.delete' => [
'table' => 'appwrite_usage_collections_{scope}_requests_delete',
],
'documents.$all.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
],
'documents.$all.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
],
'documents.$all.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
],
'documents.$all.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
],
'collections.databaseId.requests.create' => [
'table' => 'appwrite_usage_collections_{scope}_requests_create',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.read' => [
'table' => 'appwrite_usage_collections_{scope}_requests_read',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.update' => [
'table' => 'appwrite_usage_collections_{scope}_requests_update',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.delete' => [
'table' => 'appwrite_usage_collections_{scope}_requests_delete',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
'groupBy' => ['databaseId'],
],
'documents.databaseId/collectionId.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
'groupBy' => ['databaseId', 'collectionId'],
],
'buckets.$all.requests.create' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_create',
],
'buckets.$all.requests.read' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_read',
],
'buckets.$all.requests.update' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_update',
],
'buckets.$all.requests.delete' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_delete',
],
'files.$all.requests.create' => [
'table' => 'appwrite_usage_files_{scope}_requests_create',
],
'files.$all.requests.read' => [
'table' => 'appwrite_usage_files_{scope}_requests_read',
],
'files.$all.requests.update' => [
'table' => 'appwrite_usage_files_{scope}_requests_update',
],
'files.$all.requests.delete' => [
'table' => 'appwrite_usage_files_{scope}_requests_delete',
],
'files.bucketId.requests.create' => [
'table' => 'appwrite_usage_files_{scope}_requests_create',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.read' => [
'table' => 'appwrite_usage_files_{scope}_requests_read',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.update' => [
'table' => 'appwrite_usage_files_{scope}_requests_update',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.delete' => [
'table' => 'appwrite_usage_files_{scope}_requests_delete',
'groupBy' => ['bucketId'],
],
'sessions.$all.requests.create' => [
'table' => 'appwrite_usage_sessions__{scope}_requests_create',
],
'sessions.provider.requests.create' => [
'table' => 'appwrite_usage_sessions_{scope}_requests_create',
'groupBy' => ['provider'],
],
'sessions.$all.requests.delete' => [
'table' => 'appwrite_usage_sessions_{scope}_requests_delete',
],
'executions.$all.compute.total' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
],
'builds.$all.compute.total' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
],
'executions.$all.compute.failure' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'filters' => [
'functionStatus' => 'failed',
],
],
'builds.$all.compute.failure' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'filters' => [
'functionStatus' => 'failed',
],
],
'executions.$all.compute.success' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'filters' => [
'functionStatus' => 'success',
],
],
'builds.$all.compute.success' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'filters' => [
'functionStatus' => 'success',
],
],
'executions.functionId.compute.total' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
],
'builds.functionId.compute.total' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
],
'executions.functionId.compute.failure' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionStatus' => 'failed',
],
],
'builds.functionId.compute.failure' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionBuildStatus' => 'failed',
],
],
'executions.functionId.compute.success' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionStatus' => 'success',
],
],
'builds.functionId.compute.success' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionBuildStatus' => 'success',
],
],
// counters
'users.$all.count.total' => [
'table' => 'appwrite_usage_users_{scope}_count_total',
],
'buckets.$all.count.total' => [
'table' => 'appwrite_usage_buckets_{scope}_count_total',
],
'files.$all.count.total' => [
'table' => 'appwrite_usage_files_{scope}_count_total',
],
'files.bucketId.count.total' => [
'table' => 'appwrite_usage_files_{scope}_count_total',
'groupBy' => ['bucketId']
],
'databases.$all.count.total' => [
'table' => 'appwrite_usage_databases_{scope}_count_total',
],
'collections.$all.count.total' => [
'table' => 'appwrite_usage_collections_{scope}_count_total',
],
'documents.$all.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
],
'collections.databaseId.count.total' => [
'table' => 'appwrite_usage_collections_{scope}_count_total',
'groupBy' => ['databaseId']
],
'documents.databaseId.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
'groupBy' => ['databaseId']
],
'documents.databaseId/collectionId.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
'groupBy' => ['databaseId', 'collectionId']
],
'deployments.$all.storage.size' => [
'table' => 'appwrite_usage_deployments_{scope}_storage_size',
],
'project.$all.storage.size' => [
'table' => 'appwrite_usage_project_{scope}_storage_size',
],
'files.$all.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
],
'files.$bucketId.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
'groupBy' => ['bucketId']
],
'builds.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.functionId.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'builds.functionId.compute.time' => [
'table' => 'appwrite_usage_builds_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'project.$all.compute.time' => [ // Built time + execution time
'table' => 'appwrite_usage_project_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'deployments.$all.storage.size' => [
'table' => 'appwrite_usage_deployments_{scope}_storage_size'
],
'project.$all.storage.size' => [
'table' => 'appwrite_usage_project_{scope}_storage_size'
],
'files.$all.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size'
],
'files.bucketId.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
'groupBy' => ['bucketId']
]
];
public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $errorHandler = null)
{
parent::__construct($region);
$this->database = $database;
$this->influxDB = $influxDB;
$this->errorHandler = $errorHandler;
}
/**
* Create or Update Mertic
* Create or update each metric in the stats collection for the given project
*
* @param string $projectId
* @param int $time
* @param string $period
* @param string $metric
* @param int $value
* @param int $type
*
* @return void
*/
private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void
{
$id = \md5("{$time}_{$period}_{$metric}");
$this->database->setNamespace('_' . $projectId);
try {
$document = $this->database->getDocument('stats', $id);
if ($document->isEmpty()) {
$this->database->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $metric,
'value' => $value,
'type' => $type,
'region' => $this->region,
]));
} else {
$this->database->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $value)
);
}
} catch (\Exception $e) { // if projects are deleted this might fail
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
} else {
throw $e;
}
}
}
/**
* Sync From InfluxDB
* Sync stats from influxDB to stats collection in the Appwrite database
*
* @param string $metric
* @param array $options
* @param array $period
*
* @return void
*/
private function syncFromInfluxDB(string $metric, array $options, array $period): void
{
$start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339);
if (!empty($this->latestTime[$metric][$period['key']])) {
$start = $this->latestTime[$metric][$period['key']];
}
$end = (new DateTime())->format(DateTime::RFC3339);
$table = $options['table']; //Which influxdb table to query for this metric
$groupBy = empty($options['groupBy']) ? '' : ', ' . implode(', ', array_map(fn($groupBy) => '"' . $groupBy . '" ', $options['groupBy'])); //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
$filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status
if (!empty($filters)) {
$filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters)));
} else {
$filters = '';
}
$query = "SELECT sum(value) AS \"value\" ";
$query .= "FROM \"{$table}\" ";
$query .= "WHERE \"time\" > '{$start}' ";
$query .= "AND \"time\" < '{$end}' ";
$query .= "AND \"metric_type\"='counter' {$filters} ";
$query .= "GROUP BY time({$period['key']}), \"projectId\", \"projectInternalId\" {$groupBy} ";
$query .= "FILL(null)";
try {
$result = $this->influxDB->query($query);
$points = $result->getPoints();
foreach ($points as $point) {
$projectId = $point['projectId'];
if (!empty($projectId) && $projectId !== 'console') {
$metricUpdated = $metric;
if (!empty($groupBy)) {
foreach ($options['groupBy'] as $groupBy) {
$groupedBy = $point[$groupBy] ?? '';
if (empty($groupedBy)) {
continue;
}
$metricUpdated = str_replace($groupBy, $groupedBy, $metricUpdated);
}
}
$value = (!empty($point['value'])) ? $point['value'] : 0;
if (empty($point['projectInternalId'] ?? null)) {
continue;
}
$this->createOrUpdateMetric(
$point['projectInternalId'],
$point['time'],
$period['key'],
$metricUpdated,
$value,
0
);
$this->latestTime[$metric][$period['key']] = $point['time'];
}
}
} catch (\Exception $e) { // if projects are deleted this might fail
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "sync_metric_{$metric}_influxdb");
} else {
throw $e;
}
}
}
/**
* Collect Stats
* Collect all the stats from Influd DB to Database
*
* @return void
*/
public function collect(): void
{
foreach ($this->periods as $period) {
foreach ($this->metrics as $metric => $options) { //for each metrics
try {
$this->syncFromInfluxDB($metric, $options, $period);
} catch (\Exception $e) {
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e);
} else {
throw $e;
}
}
}
}
}
}
-225
View File
@@ -1,225 +0,0 @@
<?php
namespace Appwrite\Usage;
use Utopia\App;
class Stats
{
/**
* @var array
*/
protected $params = [];
/**
* @var mixed
*/
protected $statsd;
/**
* @var string
*/
protected $namespace = 'appwrite.usage';
/**
* Event constructor.
*
* @param mixed $statsd
*/
public function __construct($statsd)
{
$this->statsd = $statsd;
}
/**
* @param string $key
* @param mixed $value
*
* @return $this
*/
public function setParam(string $key, $value): self
{
$this->params[$key] = $value;
return $this;
}
/**
* @param string $key
*
* @return mixed|null
*/
public function getParam(string $key)
{
return (isset($this->params[$key])) ? $this->params[$key] : null;
}
/**
* @param string $namespace
*
* @return $this
*/
public function setNamespace(string $namespace): self
{
$this->namespace = $namespace;
return $this;
}
/**
* @return string
*/
public function getNamespace()
{
return $this->namespace;
}
/**
* Submit data to StatsD.
* Send various metrics to StatsD based on the parameters that are set
* @return void
*/
public function submit(): void
{
$projectId = $this->params['projectId'] ?? '';
$projectInternalId = $this->params['projectInternalId'];
$tags = ",projectInternalId={$projectInternalId},projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN');
// the global namespace is prepended to every key (optional)
$this->statsd->setNamespace($this->namespace);
$httpRequest = $this->params['project.{scope}.network.requests'] ?? 0;
$httpMethod = $this->params['httpMethod'] ?? '';
if ($httpRequest >= 1) {
$this->statsd->increment('project.{scope}.network.requests' . $tags . ',method=' . \strtolower($httpMethod));
}
$inbound = $this->params['project.{scope}.network.inbound'] ?? 0;
$outbound = $this->params['project.{scope}.network.outbound'] ?? 0;
$this->statsd->count('project.{scope}.network.inbound' . $tags, $inbound);
$this->statsd->count('project.{scope}.network.outbound' . $tags, $outbound);
$this->statsd->count('project.{scope}.network.bandwidth' . $tags, $inbound + $outbound);
$usersMetrics = [
'users.{scope}.requests.create',
'users.{scope}.requests.read',
'users.{scope}.requests.update',
'users.{scope}.requests.delete',
'users.{scope}.count.total',
];
foreach ($usersMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value === 1 || $value === -1) {
$this->statsd->count($metric . $tags, $value);
}
}
$dbMetrics = [
'databases.{scope}.requests.create',
'databases.{scope}.requests.read',
'databases.{scope}.requests.update',
'databases.{scope}.requests.delete',
'collections.{scope}.requests.create',
'collections.{scope}.requests.read',
'collections.{scope}.requests.update',
'collections.{scope}.requests.delete',
'documents.{scope}.requests.create',
'documents.{scope}.requests.read',
'documents.{scope}.requests.update',
'documents.{scope}.requests.delete',
'databases.{scope}.count.total',
'collections.{scope}.count.total',
'documents.{scope}.count.total'
];
foreach ($dbMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value === 1 || $value === -1) {
$dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? '');
$this->statsd->count($metric . $dbTags, $value);
}
}
$storageMertics = [
'buckets.{scope}.requests.create',
'buckets.{scope}.requests.read',
'buckets.{scope}.requests.update',
'buckets.{scope}.requests.delete',
'files.{scope}.requests.create',
'files.{scope}.requests.read',
'files.{scope}.requests.update',
'files.{scope}.requests.delete',
'buckets.{scope}.count.total',
'files.{scope}.count.total',
'files.{scope}.storage.size'
];
foreach ($storageMertics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value !== 0) {
$storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? '');
$this->statsd->count($metric . $storageTags, $value);
}
}
$sessionsMetrics = [
'sessions.{scope}.requests.create',
'sessions.{scope}.requests.update',
'sessions.{scope}.requests.delete',
];
foreach ($sessionsMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value >= 1) {
$sessionTags = $tags . ",provider=" . ($this->params['provider'] ?? '');
$this->statsd->count($metric . $sessionTags, $value);
}
}
$functionId = $this->params['functionId'] ?? '';
$functionExecution = $this->params['executions.{scope}.compute'] ?? 0;
$functionExecutionTime = ($this->params['executionTime'] ?? 0) * 1000; // ms
$functionExecutionStatus = $this->params['executionStatus'] ?? '';
$functionBuild = $this->params['builds.{scope}.compute'] ?? 0;
$functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms
$functionBuildStatus = $this->params['buildStatus'] ?? '';
$functionCompute = $functionExecutionTime + $functionBuildTime;
$functionTags = $tags . ',functionId=' . $functionId;
$deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0;
$storageSize = $this->params['files.{scope}.storage.size'] ?? 0;
if ($deploymentSize + $storageSize > 0 || $deploymentSize + $storageSize <= -1) {
$this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize);
}
if ($deploymentSize !== 0) {
$this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize);
}
if ($functionExecution >= 1) {
$this->statsd->increment('executions.{scope}.compute' . $functionTags . ',functionStatus=' . $functionExecutionStatus);
if ($functionExecutionTime > 0) {
$this->statsd->count('executions.{scope}.compute.time' . $functionTags, $functionExecutionTime);
}
}
if ($functionBuild >= 1) {
$this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus);
$this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime);
}
if ($functionBuild + $functionExecution >= 1) {
$this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute);
}
$this->reset();
}
public function reset(): self
{
$this->params = [];
$this->namespace = 'appwrite.usage';
return $this;
}
}
@@ -0,0 +1,27 @@
<?php
namespace Appwrite\Utopia\Database\Validator\Queries;
use Utopia\Database\Validator\Query\Select;
class Attributes extends Base
{
public const ALLOWED_ATTRIBUTES = [
'key',
'type',
'size',
'required',
'array',
'status',
'error'
];
/**
* Expression constructor
*
*/
public function __construct()
{
parent::__construct('attributes', self::ALLOWED_ATTRIBUTES);
}
}
@@ -2,13 +2,13 @@
namespace Appwrite\Utopia\Database\Validator\Queries;
use Appwrite\Extend\Exception;
use Utopia\Database\Validator\Queries;
use Utopia\Database\Validator\Query\Limit;
use Utopia\Database\Validator\Query\Offset;
use Utopia\Database\Validator\Query\Cursor;
use Utopia\Database\Validator\Query\Filter;
use Utopia\Database\Validator\Query\Order;
use Utopia\Database\Validator\Query\Select;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
@@ -24,7 +24,9 @@ class Base extends Queries
*/
public function __construct(string $collection, array $allowedAttributes)
{
$collection = Config::getParam('collections', [])[$collection];
$config = Config::getParam('collections', []);
$collections = array_merge($config['console'], $config['projects'], $config['buckets'], $config['databases']);
$collection = $collections[$collection];
// array for constant lookup time
$allowedAttributesLookup = [];
foreach ($allowedAttributes as $attribute) {
@@ -67,7 +69,6 @@ class Base extends Queries
new Cursor(),
new Filter($attributes),
new Order($attributes),
new Select($attributes),
];
parent::__construct($validators);
@@ -0,0 +1,23 @@
<?php
namespace Appwrite\Utopia\Database\Validator\Queries;
class Indexes extends Base
{
public const ALLOWED_ATTRIBUTES = [
'key',
'type',
'status',
'attributes',
'error',
];
/**
* Expression constructor
*
*/
public function __construct()
{
parent::__construct('indexes', self::ALLOWED_ATTRIBUTES);
}
}
+2
View File
@@ -239,6 +239,7 @@ class Response extends SwooleResponse
public const MODEL_HEALTH_QUEUE = 'healthQueue';
public const MODEL_HEALTH_TIME = 'healthTime';
public const MODEL_HEALTH_ANTIVIRUS = 'healthAntivirus';
public const MODEL_HEALTH_STATUS_LIST = 'healthStatusList';
// Console
public const MODEL_CONSOLE_VARIABLES = 'consoleVariables';
@@ -304,6 +305,7 @@ class Response extends SwooleResponse
->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE))
->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metrics', self::MODEL_METRIC, true, false))
->setModel(new BaseList('Variables List', self::MODEL_VARIABLE_LIST, 'variables', self::MODEL_VARIABLE))
->setModel(new BaseList('Status List', self::MODEL_HEALTH_STATUS_LIST, 'statuses', self::MODEL_HEALTH_STATUS))
->setModel(new BaseList('Locale codes list', self::MODEL_LOCALE_CODE_LIST, 'localeCodes', self::MODEL_LOCALE_CODE))
->setModel(new BaseList('Provider list', self::MODEL_PROVIDER_LIST, 'providers', self::MODEL_PROVIDER))
->setModel(new BaseList('Message list', self::MODEL_MESSAGE_LIST, 'messages', self::MODEL_MESSAGE))
@@ -3,6 +3,7 @@
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Utopia\Database\Document;
class AttributeRelationship extends Attribute
{
@@ -73,4 +74,23 @@ class AttributeRelationship extends Attribute
{
return Response::MODEL_ATTRIBUTE_RELATIONSHIP;
}
/**
* Process Document before returning it to the client
*
* @return Document
*/
public function filter(Document $document): Document
{
$options = $document->getAttribute('options');
if (!\is_null($options)) {
$document->setAttribute('relatedCollection', $options['relatedCollection']);
$document->setAttribute('relationType', $options['relationType']);
$document->setAttribute('twoWay', $options['twoWay']);
$document->setAttribute('twoWayKey', $options['twoWayKey']);
$document->setAttribute('side', $options['side']);
$document->setAttribute('onDelete', $options['onDelete']);
}
return $document;
}
}
@@ -81,18 +81,6 @@ class Func extends Model
'default' => '',
'example' => '5 4 * * *',
])
->addRule('scheduleNext', [
'type' => self::TYPE_DATETIME,
'description' => 'Function\'s next scheduled execution time in ISO 8601 format.',
'default' => '',
'example' => self::TYPE_DATETIME_EXAMPLE,
])
->addRule('schedulePrevious', [
'type' => self::TYPE_DATETIME,
'description' => 'Function\'s previous scheduled execution time in ISO 8601 format.',
'default' => '',
'example' => self::TYPE_DATETIME_EXAMPLE,
])
->addRule('timeout', [
'type' => self::TYPE_INTEGER,
'description' => 'Function execution timeout in seconds.',
@@ -10,6 +10,12 @@ class HealthStatus extends Model
public function __construct()
{
$this
->addRule('name', [
'type' => self::TYPE_STRING,
'description' => 'Name of the service.',
'default' => '',
'example' => 'database',
])
->addRule('ping', [
'type' => self::TYPE_INTEGER,
'description' => 'Duration in milliseconds how long the health check took.',
@@ -132,6 +132,12 @@ class Project extends Model
'default' => false,
'example' => true,
])
->addRule('authPersonalDataCheck', [
'type' => self::TYPE_BOOLEAN,
'description' => 'Whether or not to check the user password for similarity with their personal data.',
'default' => false,
'example' => true,
])
->addRule('providers', [
'type' => Response::MODEL_AUTH_PROVIDER,
'description' => 'List of Auth Providers.',
@@ -307,6 +313,7 @@ class Project extends Model
$document->setAttribute('authSessionsLimit', $authValues['maxSessions'] ?? APP_LIMIT_USER_SESSIONS_DEFAULT);
$document->setAttribute('authPasswordHistory', $authValues['passwordHistory'] ?? 0);
$document->setAttribute('authPasswordDictionary', $authValues['passwordDictionary'] ?? false);
$document->setAttribute('authPersonalDataCheck', $authValues['personalDataCheck'] ?? false);
foreach ($auth as $index => $method) {
$key = $method['key'];
@@ -16,7 +16,7 @@ class UsageBuckets extends Model
'default' => '',
'example' => '30d',
])
->addRule('filesCount', [
->addRule('filesTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of files in this bucket.',
'default' => [],
@@ -30,34 +30,6 @@ class UsageBuckets extends Model
'example' => [],
'array' => true
])
->addRule('filesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files deleted.',
'default' => [],
'example' => [],
'array' => true
])
;
}
@@ -16,41 +16,13 @@ class UsageCollection extends Model
'default' => '',
'example' => '30d',
])
->addRule('documentsCount', [
->addRule('documentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
;
}
@@ -16,72 +16,16 @@ class UsageDatabase extends Model
'default' => '',
'example' => '30d',
])
->addRule('documentsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCount', [
->addRule('collectionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
->addRule('documentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections delete.',
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
@@ -16,111 +16,27 @@ class UsageDatabases extends Model
'default' => '',
'example' => '30d',
])
->addRule('databasesCount', [
->addRule('databasesTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCount', [
->addRule('collectionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections delete.',
'default' => [],
'example' => [],
'array' => true
])
;
}
@@ -16,30 +16,16 @@ class UsageFunction extends Model
'default' => '',
'example' => '30d',
])
->addRule('executionsTotal', [
->addRule('deploymentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'description' => 'Aggregated stats for number of function deployments.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsFailure', [
->addRule('deploymentsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution successes.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution duration.',
'description' => 'Aggregated stats for function deployments storage.',
'default' => [],
'example' => [],
'array' => true
@@ -51,23 +37,31 @@ class UsageFunction extends Model
'example' => [],
'array' => true
])
->addRule('buildsFailure', [
->addRule('buildsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build successes.',
'description' => 'Aggregated stats for builds storage.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build duration.',
'description' => 'Aggregated stats for function build compute.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution compute.',
'default' => [],
'example' => [],
'array' => true
@@ -16,30 +16,23 @@ class UsageFunctions extends Model
'default' => '',
'example' => '30d',
])
->addRule('executionsTotal', [
->addRule('functionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'description' => 'Aggregated stats for number of functions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsFailure', [
->addRule('deploymentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution failures.',
'description' => 'Aggregated stats for number of function deployments.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsSuccess', [
->addRule('deploymentsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution successes.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution duration.',
'description' => 'Aggregated stats for function deployments storage.',
'default' => [],
'example' => [],
'array' => true
@@ -51,23 +44,31 @@ class UsageFunctions extends Model
'example' => [],
'array' => true
])
->addRule('buildsFailure', [
->addRule('buildsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build successes.',
'description' => 'Aggregated stats for builds storage.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build duration.',
'description' => 'Aggregated stats for function build compute.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution compute.',
'default' => [],
'example' => [],
'array' => true
@@ -16,7 +16,7 @@ class UsageProject extends Model
'default' => '',
'example' => '30d',
])
->addRule('requests', [
->addRule('requestsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of requests.',
'default' => [],
@@ -30,42 +30,42 @@ class UsageProject extends Model
'example' => [],
'array' => true
])
->addRule('executions', [
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documents', [
->addRule('documentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databases', [
->addRule('databasesTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of databases.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('users', [
->addRule('usersTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of users.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('storage', [
->addRule('filesStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buckets', [
->addRule('bucketsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of buckets.',
'default' => [],
@@ -16,79 +16,23 @@ class UsageStorage extends Model
'default' => '',
'example' => '30d',
])
->addRule('storage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of files.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsCount', [
->addRule('bucketsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of buckets.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsCreate', [
->addRule('filesTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets created.',
'description' => 'Aggregated stats for total number of files.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsRead', [
->addRule('filesStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files deleted.',
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true
@@ -16,62 +16,21 @@ class UsageUsers extends Model
'default' => '',
'example' => '30d',
])
->addRule('usersCount', [
->addRule('usersTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of users.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsCreate', [
->addRule('sessionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsProviderCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions deleted.',
'default' => [],
'example' => [],
'array' => true
])
;
}
+58 -116
View File
@@ -4,7 +4,6 @@ namespace Executor;
use Exception;
use Utopia\App;
use Utopia\CLI\Console;
class Executor
{
@@ -18,20 +17,29 @@ class Executor
public const METHOD_CONNECT = 'CONNECT';
public const METHOD_TRACE = 'TRACE';
private $endpoint;
private bool $selfSigned = false;
private $selfSigned = false;
private string $endpoint;
protected $headers = [
'content-type' => '',
];
protected array $headers;
protected int $cpus;
protected int $memory;
public function __construct(string $endpoint)
{
if (!filter_var($endpoint, FILTER_VALIDATE_URL)) {
throw new Exception('Unsupported endpoint');
}
$this->endpoint = $endpoint;
$this->cpus = \intval(App::getEnv('_APP_FUNCTIONS_CPUS', '1'));
$this->memory = intval(App::getEnv('_APP_FUNCTIONS_MEMORY', '512'));
$this->headers = [
'content-type' => 'application/json',
'authorization' => 'Bearer ' . App::getEnv('_APP_EXECUTOR_SECRET', ''),
];
}
/**
@@ -42,45 +50,41 @@ class Executor
* @param string $deploymentId
* @param string $projectId
* @param string $source
* @param string $runtime
* @param string $baseImage
* @param string $image
* @param bool $remove
* @param string $entrypoint
* @param string $workdir
* @param string $destinaction
* @param string $network
* @param array $vars
* @param string $destination
* @param array $variables
* @param array $commands
*/
public function createRuntime(
string $deploymentId,
string $projectId,
string $source,
string $runtime,
string $baseImage,
string $image,
bool $remove = false,
string $entrypoint = '',
string $workdir = '',
string $destination = '',
array $vars = [],
array $variables = [],
array $commands = []
) {
$runtimeId = "$projectId-$deploymentId";
$route = "/runtimes";
$headers = [
'content-type' => 'application/json',
'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '')
];
$headers = [ 'x-opr-runtime-id' => $runtimeId ];
$params = [
'runtimeId' => "$projectId-$deploymentId",
'runtimeId' => $runtimeId,
'source' => $source,
'destination' => $destination,
'runtime' => $runtime,
'baseImage' => $baseImage,
'image' => $image,
'entrypoint' => $entrypoint,
'workdir' => $workdir,
'vars' => $vars,
'variables' => $variables,
'remove' => $remove,
'commands' => $commands
'commands' => $commands,
'cpus' => $this->cpus,
'memory' => $this->memory,
];
$timeout = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
@@ -89,36 +93,8 @@ class Executor
$status = $response['headers']['status-code'];
if ($status >= 400) {
throw new \Exception($response['body']['message'], $status);
}
return $response['body'];
}
/**
* Delete Runtime
*
* Deletes a runtime and cleans up any containers remaining.
*
* @param string $projectId
* @param string $deploymentId
*/
public function deleteRuntime(string $projectId, string $deploymentId)
{
$runtimeId = "$projectId-$deploymentId";
$route = "/runtimes/$runtimeId";
$headers = [
'content-type' => 'application/json',
'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '')
];
$params = [];
$response = $this->call(self::METHOD_DELETE, $route, $headers, $params, true, 30);
$status = $response['headers']['status-code'];
if ($status >= 400) {
throw new \Exception($response['body']['message'], $status);
$message = \is_string($response['body']) ? $response['body'] : $response['body']['message'];
throw new \Exception($message, $status);
}
return $response['body'];
@@ -129,86 +105,52 @@ class Executor
*
* @param string $projectId
* @param string $deploymentId
* @param string $path
* @param array $vars
* @param string $entrypoint
* @param string $data
* @param string runtime
* @param string $baseImage
* @param string $payload
* @param array $variables
* @param int $timeout
* @param string $image
* @param string $source
* @param string $entrypoint
*
* @return array
*/
public function createExecution(
string $projectId,
string $deploymentId,
string $path,
array $vars,
string $payload,
array $variables,
int $timeout,
string $image,
string $source,
string $entrypoint,
string $data,
string $runtime,
string $baseImage,
$timeout
) {
$route = "/execution";
$headers = [
'content-type' => 'application/json',
'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '')
];
$runtimeId = "$projectId-$deploymentId";
$route = '/runtimes/' . $runtimeId . '/execution';
$headers = [ 'x-opr-runtime-id' => $runtimeId ];
$params = [
'runtimeId' => "$projectId-$deploymentId",
'vars' => $vars,
'data' => $data,
'runtimeId' => $runtimeId,
'variables' => $variables,
'payload' => $payload,
'timeout' => $timeout,
'image' => $image,
'source' => $source,
'entrypoint' => $entrypoint,
'cpus' => $this->cpus,
'memory' => $this->memory,
];
/* Add 2 seconds as a buffer to the actual timeout value since there can be a slight variance*/
$requestTimeout = $timeout + 2;
$timeout = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
$response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $timeout);
$response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout);
$status = $response['headers']['status-code'];
for ($attempts = 0; $attempts < 10; $attempts++) {
try {
switch (true) {
case $status < 400:
return $response['body'];
case $status === 404:
$response = $this->createRuntime(
deploymentId: $deploymentId,
projectId: $projectId,
source: $path,
runtime: $runtime,
baseImage: $baseImage,
vars: $vars,
entrypoint: $entrypoint,
commands: []
);
$response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout);
$status = $response['headers']['status-code'];
if ($status < 400) {
return $response['body'];
}
break;
case $status === 406:
$response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout);
$status = $response['headers']['status-code'];
if ($status < 400) {
return $response['body'];
}
break;
default:
throw new \Exception($response['body']['message'], $status);
}
} catch (\Exception $e) {
throw new \Exception($e->getMessage(), $e->getCode());
}
sleep(2);
if ($status >= 400) {
$message = \is_string($response['body']) ? $response['body'] : $response['body']['message'];
throw new \Exception($message, $status);
}
throw new Exception($response['body']['message'], 503);
return $response['body'];
}
/**