Merge branch 'main' into 1.5.x-db-storage-metrics

# Conflicts:
#	app/config/specs/open-api3-latest-console.json
#	app/config/specs/swagger2-latest-console.json
#	app/controllers/api/project.php
#	src/Appwrite/Utopia/Response/Model/UsageProject.php
#	tests/e2e/General/UsageTest.php
This commit is contained in:
Bradley Schofield
2024-09-19 15:37:27 +09:00
5137 changed files with 393303 additions and 1145 deletions
@@ -0,0 +1,81 @@
<?php
namespace Appwrite\Auth\Validator;
use Utopia\Validator;
use Utopia\Validator\Text;
/**
* MockNumber.
*
* Validates if a given object represents a valid phone and OTP pair
*/
class MockNumber extends Validator
{
private $message = '';
/**
* Get Description.
*
* Returns validator description
*
* @return string
*/
public function getDescription(): string
{
return $this->message;
}
/**
* Is valid.
*
* @param mixed $value
*
* @return bool
*/
public function isValid($value): bool
{
if (!\is_array($value) || !isset($value['phone']) || !isset($value['otp'])) {
$this->message = 'Invalid payload structure. Please check the "phone" and "otp" fields';
return false;
}
$phone = new Phone();
if (!$phone->isValid($value['phone'])) {
$this->message = $phone->getDescription();
return false;
}
$otp = new Text(6, 6, Text::NUMBERS);
if (!$otp->isValid($value['otp'])) {
$this->message = 'Invalid OTP. Please make sure the OTP is a 6 digit number';
return false;
}
return true;
}
/**
* Is array
*
* Function will return true if object is array.
*
* @return bool
*/
public function isArray(): bool
{
return false;
}
/**
* Get Type
*
* Returns validator type.
*
* @return string
*/
public function getType(): string
{
return self::TYPE_OBJECT;
}
}
+22
View File
@@ -54,6 +54,7 @@ class Event
protected array $context = [];
protected ?Document $project = null;
protected ?Document $user = null;
protected ?string $userId = null;
protected bool $paused = false;
/**
@@ -145,6 +146,18 @@ class Event
return $this;
}
/**
* Set user ID for this event.
*
* @return self
*/
public function setUserId(string $userId): self
{
$this->userId = $userId;
return $this;
}
/**
* Get user responsible for triggering this event.
*
@@ -155,6 +168,14 @@ class Event
return $this->user;
}
/**
* Get user responsible for triggering this event.
*/
public function getUserId(): ?string
{
return $this->userId;
}
/**
* Set payload for this event.
*
@@ -303,6 +324,7 @@ class Event
return $client->enqueue([
'project' => $this->project,
'user' => $this->user,
'userId' => $this->userId,
'payload' => $this->payload,
'context' => $this->context,
'events' => Event::generateEvents($this->getEvent(), $this->getParams())
+25
View File
@@ -14,6 +14,7 @@ class Func extends Event
protected string $path = '';
protected string $method = '';
protected array $headers = [];
protected ?string $functionId = null;
protected ?Document $function = null;
protected ?Document $execution = null;
@@ -49,6 +50,28 @@ class Func extends Event
return $this->function;
}
/**
* Sets function id for the function event.
*
* @param string $functionId
*/
public function setFunctionId(string $functionId): self
{
$this->functionId = $functionId;
return $this;
}
/**
* Returns set function id for the function event.
*
* @return string|null
*/
public function getFunctionId(): ?string
{
return $this->functionId;
}
/**
* Sets execution for the function event.
*
@@ -199,7 +222,9 @@ class Func extends Event
return $client->enqueue([
'project' => $this->project,
'user' => $this->user,
'userId' => $this->userId,
'function' => $this->function,
'functionId' => $this->functionId,
'execution' => $this->execution,
'type' => $this->type,
'jwt' => $this->jwt,
+7 -1
View File
@@ -107,6 +107,8 @@ class Exception extends \Exception
public const USER_TARGET_ALREADY_EXISTS = 'user_target_already_exists';
public const USER_API_KEY_AND_SESSION_SET = 'user_key_and_session_set';
public const API_KEY_EXPIRED = 'api_key_expired';
/** Teams */
public const TEAM_NOT_FOUND = 'team_not_found';
public const TEAM_INVITE_ALREADY_EXISTS = 'team_invite_already_exists';
@@ -154,6 +156,7 @@ class Exception extends \Exception
public const FUNCTION_RUNTIME_UNSUPPORTED = 'function_runtime_unsupported';
public const FUNCTION_ENTRYPOINT_MISSING = 'function_entrypoint_missing';
public const FUNCTION_SYNCHRONOUS_TIMEOUT = 'function_synchronous_timeout';
public const FUNCTION_TEMPLATE_NOT_FOUND = 'function_template_not_found';
/** Deployments */
public const DEPLOYMENT_NOT_FOUND = 'deployment_not_found';
@@ -162,9 +165,11 @@ class Exception extends \Exception
public const BUILD_NOT_FOUND = 'build_not_found';
public const BUILD_NOT_READY = 'build_not_ready';
public const BUILD_IN_PROGRESS = 'build_in_progress';
public const BUILD_ALREADY_COMPLETED = 'build_already_completed';
/** Execution */
public const EXECUTION_NOT_FOUND = 'execution_not_found';
public const EXECUTION_IN_PROGRESS = 'execution_in_progress';
/** Databases */
public const DATABASE_NOT_FOUND = 'database_not_found';
@@ -195,6 +200,7 @@ class Exception extends \Exception
public const ATTRIBUTE_LIMIT_EXCEEDED = 'attribute_limit_exceeded';
public const ATTRIBUTE_VALUE_INVALID = 'attribute_value_invalid';
public const ATTRIBUTE_TYPE_INVALID = 'attribute_type_invalid';
public const ATTRIBUTE_INVALID_RESIZE = 'attribute_invalid_resize';
/** Relationship */
public const RELATIONSHIP_VALUE_INVALID = 'relationship_value_invalid';
@@ -308,7 +314,7 @@ class Exception extends \Exception
$this->code = $code ?? $this->errors[$type]['code'];
// Mark string errors like HY001 from PDO as 500 errors
if(\is_string($this->code)) {
if (\is_string($this->code)) {
if (\is_numeric($this->code)) {
$this->code = (int) $this->code;
} else {
+16
View File
@@ -0,0 +1,16 @@
<?php
namespace Appwrite\Functions;
class Specification
{
public const S_05VCPU_512MB = 's-0.5vcpu-512mb';
public const S_1VCPU_512MB = 's-1vcpu-512mb';
public const S_1VCPU_1GB = 's-1vcpu-1gb';
public const S_2VCPU_2GB = 's-2vcpu-2gb';
public const S_2VCPU_4GB = 's-2vcpu-4gb';
public const S_4VCPU_4GB = 's-4vcpu-4gb';
public const S_4VCPU_8GB = 's-4vcpu-8gb';
public const S_8VCPU_4GB = 's-8vcpu-4gb';
public const S_8VCPU_8GB = 's-8vcpu-8gb';
}
@@ -0,0 +1,107 @@
<?php
namespace Appwrite\Functions\Validator;
use Utopia\Validator;
/**
* Headers.
*
* Validates user provided headers
*/
class Headers extends Validator
{
public function __construct(protected bool $allowEmpty = true, protected int $maxKeys = 100, protected int $maxSize = 16384)
{
}
/**
* Get Description.
*
* Returns validator description
*
* @return string
*/
public function getDescription(): string
{
return 'Invalid headers: Alphanumeric characters or hyphens only, cannot start with "x-appwrite", maximum ' . $this->maxKeys . ' keys, and total size ' . $this->maxSize . '.';
}
/**
* Is valid.
*
* @param mixed $value
*
* @return bool
*/
public function isValid($value): bool
{
if ($this->allowEmpty && empty($value)) {
return true;
}
if (!\is_array($value)) {
return false;
}
if (\count($value) > $this->maxKeys) {
return false;
}
$size = 0;
foreach ($value as $key => $val) {
$length = \strlen($key);
// Reject non-string keys
if (!\is_string($key) || $length === 0) {
return false;
}
$size += $length + \strlen($val);
if ($size >= $this->maxSize) {
return false;
}
// Check first and last character
if (!ctype_alnum($key[0]) || !ctype_alnum($key[$length - 1])) {
return false;
}
// Check middle characters
for ($i = 1; $i < $length - 1; $i++) {
if (!ctype_alnum($key[$i]) && $key[$i] !== '-') {
return false;
}
}
// Check for x-appwrite- prefix
if (str_starts_with($key, 'x-appwrite-')) {
return false;
}
}
return true;
}
/**
* Is array
*
* Function will return true if object is array.
*
* @return bool
*/
public function isArray(): bool
{
return false;
}
/**
* Get Type
*
* Returns validator type.
*
* @return string
*/
public function getType(): string
{
return self::TYPE_OBJECT;
}
}
@@ -0,0 +1,18 @@
<?php
namespace Appwrite\Functions\Validator;
use Utopia\Validator\Text;
class Payload extends Text
{
public function __construct(int $length, int $min = 1)
{
parent::__construct($length, $min);
}
public function getType(): string
{
return self::TYPE_STRING;
}
}
@@ -0,0 +1,112 @@
<?php
namespace Appwrite\Functions\Validator;
use Utopia\Validator;
class RuntimeSpecification extends Validator
{
private array $plan;
private array $specifications;
private float $maxCpus;
private int $maxMemory;
public function __construct(array $plan, array $specifications, float $maxCpus, int $maxMemory)
{
$this->plan = $plan;
$this->specifications = $specifications;
$this->maxCpus = $maxCpus;
$this->maxMemory = $maxMemory;
}
/**
* Get Allowed Specifications.
*
* Get allowed specifications taking into account the limits set by the environment variables and the plan.
*
* @return array
*/
public function getAllowedSpecifications(): array
{
$allowedSpecifications = [];
foreach ($this->specifications as $size => $values) {
if ($values['cpus'] <= $this->maxCpus && $values['memory'] <= $this->maxMemory) {
if (!empty($this->plan) && array_key_exists('runtimeSpecifications', $this->plan)) {
if (!\in_array($size, $this->plan['runtimeSpecifications'])) {
continue;
}
}
$allowedSpecifications[] = $size;
}
}
return $allowedSpecifications;
}
/**
* Get Description.
*
* Returns validator description.
*
* @return string
*/
public function getDescription(): string
{
return 'Specification must be one of: ' . implode(', ', $this->getAllowedSpecifications());
}
/**
* Is valid.
*
* Returns true if valid or false if not.
*
* @param mixed $value
*
* @return bool
*/
public function isValid($value): bool
{
if (empty($value)) {
return false;
}
if (!\is_string($value)) {
return false;
}
if (!\in_array($value, $this->getAllowedSpecifications())) {
return false;
}
return true;
}
/**
* Is array.
*
* Function will return true if object is array.
*
* @return bool
*/
public function isArray(): bool
{
return false;
}
/**
* Get Type.
*
* Returns validator type.
*
* @return string
*/
public function getType(): string
{
return self::TYPE_STRING;
}
}
+22 -20
View File
@@ -87,6 +87,10 @@ abstract class Migration
'1.5.6' => 'V20',
'1.5.7' => 'V20',
'1.5.8' => 'V20',
'1.5.9' => 'V20',
'1.5.10' => 'V20',
'1.5.11' => 'V20',
'1.6.0' => 'V21',
];
/**
@@ -171,29 +175,27 @@ abstract class Migration
Console::log('Migrating Collection ' . $collection['$id'] . ':');
\Co\run(function (array $collection, callable $callback) {
foreach ($this->documentsIterator($collection['$id']) as $document) {
go(function (Document $document, callable $callback) {
if (empty($document->getId()) || empty($document->getCollection())) {
return;
}
foreach ($this->documentsIterator($collection['$id']) as $document) {
go(function (Document $document, callable $callback) {
if (empty($document->getId()) || empty($document->getCollection())) {
return;
}
$old = $document->getArrayCopy();
$new = call_user_func($callback, $document);
$old = $document->getArrayCopy();
$new = call_user_func($callback, $document);
if (is_null($new) || $new->getArrayCopy() == $old) {
return;
}
if (is_null($new) || $new->getArrayCopy() == $old) {
return;
}
try {
$this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document);
} catch (\Throwable $th) {
Console::error('Failed to update document: ' . $th->getMessage());
return;
}
}, $document, $callback);
}
}, $collection, $callback);
try {
$this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document);
} catch (\Throwable $th) {
Console::error('Failed to update document: ' . $th->getMessage());
return;
}
}, $document, $callback);
}
}
}
+18 -20
View File
@@ -816,29 +816,27 @@ class V19 extends Migration
Console::log('Migrating Collection ' . $collection['$id'] . ':');
\Co\run(function (array $collection, callable $callback) {
foreach ($this->documentsIterator($collection['$id']) as $document) {
go(function (Document $document, callable $callback) {
if (empty($document->getId()) || empty($document->getCollection())) {
return;
}
foreach ($this->documentsIterator($collection['$id']) as $document) {
go(function (Document $document, callable $callback) {
if (empty($document->getId()) || empty($document->getCollection())) {
return;
}
$old = $document->getArrayCopy();
$new = call_user_func($callback, $document);
$old = $document->getArrayCopy();
$new = call_user_func($callback, $document);
if (is_null($new) || $new->getArrayCopy() == $old) {
return;
}
if (is_null($new) || $new->getArrayCopy() == $old) {
return;
}
try {
$this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document);
} catch (\Throwable $th) {
Console::error('Failed to update document: ' . $th->getMessage());
return;
}
}, $document, $callback);
}
}, $collection, $callback);
try {
$this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document);
} catch (\Throwable $th) {
Console::error('Failed to update document: ' . $th->getMessage());
return;
}
}, $document, $callback);
}
}
}
}
+213
View File
@@ -0,0 +1,213 @@
<?php
namespace Appwrite\Migration\Version;
use Appwrite\Migration\Migration;
use Exception;
use Throwable;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
class V21 extends Migration
{
/**
* @throws Throwable
*/
public function execute(): void
{
/**
* Disable SubQueries for Performance.
*/
foreach (['subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships', 'subQueryVariables', 'subQueryChallenges', 'subQueryProjectVariables', 'subQueryTargets', 'subQueryTopicTargets'] as $name) {
Database::addFilter(
$name,
fn () => null,
fn () => []
);
}
Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
$this->projectDB->setNamespace("_{$this->project->getInternalId()}");
Console::info('Migrating Collections');
$this->migrateCollections();
if ($this->project->getInternalId() !== 'console') {
Console::info('Migrating Buckets');
$this->migrateBuckets();
}
Console::info('Migrating Documents');
$this->forEachDocument([$this, 'fixDocument']);
}
/**
* Migrate Collections.
*
* @return void
* @throws Exception|Throwable
*/
private function migrateCollections(): void
{
$internalProjectId = $this->project->getInternalId();
$collectionType = match ($internalProjectId) {
'console' => 'console',
default => 'projects',
};
$collections = $this->collections[$collectionType];
foreach ($collections as $collection) {
$id = $collection['$id'];
Console::log("Migrating Collection \"{$id}\"");
$this->projectDB->setNamespace("_$internalProjectId");
switch ($id) {
case 'projects':
// Create accessedAt attribute
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'accessedAt');
} catch (Throwable $th) {
Console::warning("'accessedAt' from {$id}: {$th->getMessage()}");
}
break;
case 'platforms':
// Increase 'type' length to 255
try {
$this->projectDB->updateAttribute($id, 'type', size: 255);
} catch (Throwable $th) {
Console::warning("'type' from {$id}: {$th->getMessage()}");
}
break;
case 'schedules':
// Create data attribute
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'data');
} catch (Throwable $th) {
Console::warning("'data' from {$id}: {$th->getMessage()}");
}
break;
case 'functions':
// Create scopes attribute
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'scopes');
} catch (Throwable $th) {
Console::warning("'scopes' from {$id}: {$th->getMessage()}");
}
// Create specification attribute
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'specification');
} catch (Throwable $th) {
Console::warning("'specification' from {$id}: {$th->getMessage()}");
}
break;
case 'executions':
// Create requestMethod index
try {
$this->createIndexFromCollection($this->projectDB, $id, '_key_requestMethod');
} catch (\Throwable $th) {
Console::warning("'_key_requestMethod' from {$id}: {$th->getMessage()}");
}
// Create requestPath index
try {
$this->createIndexFromCollection($this->projectDB, $id, '_key_requestPath');
} catch (\Throwable $th) {
Console::warning("'_key_requestPath' from {$id}: {$th->getMessage()}");
}
// Create deployment index
try {
$this->createIndexFromCollection($this->projectDB, $id, '_key_deployment');
} catch (\Throwable $th) {
Console::warning("'_key_deployment' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'scheduledAt' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'scheduledAt');
} catch (\Throwable $th) {
Console::warning("'scheduledAt' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'scheduleInternalId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'scheduleInternalId');
} catch (\Throwable $th) {
Console::warning("'scheduleInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Create 'scheduleId' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'scheduleId');
} catch (\Throwable $th) {
Console::warning("'scheduleId' from {$id}: {$th->getMessage()}");
}
break;
}
usleep(50000);
}
}
/**
* Fix run on each document
*
* @param Document $document
* @return Document
*/
protected function fixDocument(Document $document): Document
{
switch ($document->getCollection()) {
case 'projects':
/**
* Bump version number.
*/
$document->setAttribute('version', '1.6.0');
// Add accessedAt attribute
$document->setAttribute('accessedAt', DateTime::now());
break;
case 'functions':
// Add scopes attribute
$document->setAttribute('scopes', []);
// Add size attribute
$document->setAttribute('specification', APP_FUNCTION_SPECIFICATION_DEFAULT);
}
return $document;
}
/**
* Migrating Buckets.
*
* @return void
*/
private function migrateBuckets(): void
{
foreach ($this->documentsIterator('buckets') as $bucket) {
$bucketId = 'bucket_' . $bucket['$internalId'];
try {
$this->projectDB->updateAttribute($bucketId, 'metadata', size: 65534);
$this->projectDB->purgeCachedCollection($bucketId);
} catch (\Throwable $th) {
Console::warning("'bucketId' from {$bucketId}: {$th->getMessage()}");
}
}
}
}
@@ -21,6 +21,8 @@ class Origin extends Validator
public const CLIENT_TYPE_APPLE_TVOS = 'apple-tvos';
public const CLIENT_TYPE_ANDROID = 'android';
public const CLIENT_TYPE_UNITY = 'unity';
public const CLIENT_TYPE_REACT_NATIVE_IOS = 'react-native-ios';
public const CLIENT_TYPE_REACT_NATIVE_ANDROID = 'react-native-android';
public const SCHEME_TYPE_HTTP = 'http';
@@ -88,6 +90,8 @@ class Origin extends Validator
case self::CLIENT_TYPE_APPLE_MACOS:
case self::CLIENT_TYPE_APPLE_WATCHOS:
case self::CLIENT_TYPE_APPLE_TVOS:
case self::CLIENT_TYPE_REACT_NATIVE_IOS:
case self::CLIENT_TYPE_REACT_NATIVE_ANDROID:
$this->clients[] = (isset($platform['key'])) ? $platform['key'] : '';
break;
+2
View File
@@ -8,6 +8,7 @@ use Appwrite\Platform\Tasks\Maintenance;
use Appwrite\Platform\Tasks\Migrate;
use Appwrite\Platform\Tasks\QueueCount;
use Appwrite\Platform\Tasks\QueueRetry;
use Appwrite\Platform\Tasks\ScheduleExecutions;
use Appwrite\Platform\Tasks\ScheduleFunctions;
use Appwrite\Platform\Tasks\ScheduleMessages;
use Appwrite\Platform\Tasks\SDKs;
@@ -33,6 +34,7 @@ class Tasks extends Service
->addAction(SDKs::getName(), new SDKs())
->addAction(SSL::getName(), new SSL())
->addAction(ScheduleFunctions::getName(), new ScheduleFunctions())
->addAction(ScheduleExecutions::getName(), new ScheduleExecutions())
->addAction(ScheduleMessages::getName(), new ScheduleMessages())
->addAction(Specs::getName(), new Specs())
->addAction(Upgrade::getName(), new Upgrade())
+13 -5
View File
@@ -7,6 +7,7 @@ use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Domains\Domain;
use Utopia\DSN\DSN;
use Utopia\Logger\Logger;
use Utopia\Platform\Action;
use Utopia\Registry\Registry;
@@ -100,13 +101,20 @@ class Doctor extends Action
Console::log('🟢 HTTPS force option is enabled for function domains');
}
$providerName = System::getEnv('_APP_LOGGING_PROVIDER', '');
$providerConfig = System::getEnv('_APP_LOGGING_CONFIG', '');
if (empty($providerName) || empty($providerConfig) || !Logger::hasProvider($providerName)) {
Console::log('🔴 Logging adapter is disabled');
} else {
Console::log('🟢 Logging adapter is enabled (' . $providerName . ')');
try {
$loggingProvider = new DSN($providerConfig ?? '');
$providerName = $loggingProvider->getScheme();
if (empty($providerName) || !Logger::hasProvider($providerName)) {
Console::log('🔴 Logging adapter is disabled');
} else {
Console::log('🟢 Logging adapter is enabled (' . $providerName . ')');
}
} catch (\Throwable $th) {
Console::log('🔴 Logging adapter is misconfigured');
}
\usleep(200 * 1000); // Sleep for 0.2 seconds
+33 -8
View File
@@ -3,8 +3,8 @@
namespace Appwrite\Platform\Tasks;
use Appwrite\Migration\Migration;
use Redis;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
@@ -12,10 +12,13 @@ use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Action;
use Utopia\Registry\Registry;
use Utopia\System\System;
use Utopia\Validator\Text;
class Migrate extends Action
{
protected Redis $redis;
public static function getName(): string
{
return 'migrate';
@@ -27,31 +30,53 @@ class Migrate extends Action
->desc('Migrate Appwrite to new version')
/** @TODO APP_VERSION_STABLE needs to be defined */
->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true)
->inject('cache')
->inject('dbForConsole')
->inject('getProjectDB')
->inject('register')
->callback(fn ($version, $cache, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $cache, $dbForConsole, $getProjectDB, $register));
->callback(function ($version, $dbForConsole, $getProjectDB, Registry $register) {
\Co\run(function () use ($version, $dbForConsole, $getProjectDB, $register) {
$this->action($version, $dbForConsole, $getProjectDB, $register);
});
});
}
private function clearProjectsCache(Cache $cache, Document $project)
private function clearProjectsCache(Document $project)
{
try {
$cache->purge("cache-_{$project->getInternalId()}:*");
$iterator = null;
do {
$pattern = "default-cache-_{$project->getInternalId()}:*";
$keys = $this->redis->scan($iterator, $pattern, 1000);
if ($keys !== false) {
foreach ($keys as $key) {
$this->redis->del($key);
}
}
} while ($iterator > 0);
} catch (\Throwable $th) {
Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage());
}
}
public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register)
public function action(string $version, Database $dbForConsole, callable $getProjectDB, Registry $register)
{
Authorization::disable();
if (!array_key_exists($version, Migration::$versions)) {
Console::error("Version {$version} not found.");
Console::exit(1);
return;
}
$this->redis = new Redis();
$this->redis->connect(
System::getEnv('_APP_REDIS_HOST', null),
System::getEnv('_APP_REDIS_PORT', 6379),
3,
null,
10
);
$app = new App('UTC');
Console::success('Starting Data Migration to version ' . $version);
@@ -87,7 +112,7 @@ class Migrate extends Action
continue;
}
$this->clearProjectsCache($cache, $project);
$this->clearProjectsCache($project);
try {
// TODO: Iterate through all project DBs
@@ -103,7 +128,7 @@ class Migrate extends Action
throw $th;
}
$this->clearProjectsCache($cache, $project);
$this->clearProjectsCache($project);
}
$sum = \count($projects);
+2 -2
View File
@@ -50,7 +50,7 @@ class SDKs extends Action
$production = ($git) ? (Console::confirm('Type "Appwrite" to push code to production git repos') == 'Appwrite') : false;
$message = ($git) ? Console::confirm('Please enter your commit message:') : '';
if (!in_array($version, ['0.6.x', '0.7.x', '0.8.x', '0.9.x', '0.10.x', '0.11.x', '0.12.x', '0.13.x', '0.14.x', '0.15.x', '1.0.x', '1.1.x', '1.2.x', '1.3.x', '1.4.x', '1.5.x', 'latest'])) {
if (!in_array($version, ['0.6.x', '0.7.x', '0.8.x', '0.9.x', '0.10.x', '0.11.x', '0.12.x', '0.13.x', '0.14.x', '0.15.x', '1.0.x', '1.1.x', '1.2.x', '1.3.x', '1.4.x', '1.5.x', '1.6.x', 'latest'])) {
throw new \Exception('Unknown version given');
}
@@ -235,7 +235,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
->setTwitter(APP_SOCIAL_TWITTER_HANDLE)
->setDiscord(APP_SOCIAL_DISCORD_CHANNEL, APP_SOCIAL_DISCORD)
->setDefaultHeaders([
'X-Appwrite-Response-Format' => '1.5.0',
'X-Appwrite-Response-Format' => '1.6.0',
]);
// Make sure we have a clean slate.
+8 -5
View File
@@ -64,7 +64,8 @@ abstract class ScheduleBase extends Action
$collectionId = match ($schedule->getAttribute('resourceType')) {
'function' => 'functions',
'message' => 'messages'
'message' => 'messages',
'execution' => 'executions'
};
$resource = $getProjectDB($project)->getDocument(
@@ -73,6 +74,7 @@ abstract class ScheduleBase extends Action
);
return [
'$internalId' => $schedule->getInternalId(),
'$id' => $schedule->getId(),
'resourceId' => $schedule->getAttribute('resourceId'),
'schedule' => $schedule->getAttribute('schedule'),
@@ -109,11 +111,12 @@ abstract class ScheduleBase extends Action
foreach ($results as $document) {
try {
$this->schedules[$document['resourceId']] = $getSchedule($document);
$this->schedules[$document->getInternalId()] = $getSchedule($document);
} catch (\Throwable $th) {
$collectionId = match ($document->getAttribute('resourceType')) {
'function' => 'functions',
'message' => 'messages'
'message' => 'messages',
'execution' => 'executions'
};
Console::error("Failed to load schedule for project {$document['projectId']} {$collectionId} {$document['resourceId']}");
@@ -170,10 +173,10 @@ abstract class ScheduleBase extends Action
if (!$document['active']) {
Console::info("Removing: {$document['resourceId']}");
unset($this->schedules[$document['resourceId']]);
unset($this->schedules[$document->getInternalId()]);
} elseif ($new !== $org) {
Console::info("Updating: {$document['resourceId']}");
$this->schedules[$document['resourceId']] = $getSchedule($document);
$this->schedules[$document->getInternalId()] = $getSchedule($document);
}
}
@@ -0,0 +1,82 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Event\Func;
use Swoole\Coroutine as Co;
use Utopia\Database\Database;
use Utopia\Pools\Group;
class ScheduleExecutions extends ScheduleBase
{
public const UPDATE_TIMER = 3; // seconds
public const ENQUEUE_TIMER = 4; // seconds
public static function getName(): string
{
return 'schedule-executions';
}
public static function getSupportedResource(): string
{
return 'execution';
}
protected function enqueueResources(Group $pools, Database $dbForConsole): void
{
$queue = $pools->get('queue')->pop();
$connection = $queue->getResource();
$queueForFunctions = new Func($connection);
$intervalEnd = (new \DateTime())->modify('+' . self::ENQUEUE_TIMER . ' seconds');
foreach ($this->schedules as $schedule) {
if (!$schedule['active']) {
$dbForConsole->deleteDocument(
'schedules',
$schedule['$id'],
);
unset($this->schedules[$schedule['$internalId']]);
continue;
}
$scheduledAt = new \DateTime($schedule['schedule']);
if ($scheduledAt <= $intervalEnd) {
continue;
}
$data = $dbForConsole->getDocument(
'schedules',
$schedule['$id'],
)->getAttribute('data', []);
$delay = $scheduledAt->getTimestamp() - (new \DateTime())->getTimestamp();
\go(function () use ($queueForFunctions, $schedule, $delay, $data) {
Co::sleep($delay);
$queueForFunctions->setType('schedule')
// Set functionId instead of function as we don't have $dbForProject
// TODO: Refactor to use function instead of functionId
->setFunctionId($schedule['resource']['functionId'])
->setExecution($schedule['resource'])
->setMethod($data['method'] ?? 'POST')
->setPath($data['path'] ?? '/')
->setHeaders($data['headers'] ?? [])
->setBody($data['body'] ?? '')
->setProject($schedule['project'])
->setUserId($data['userId'] ?? '')
->trigger();
});
$dbForConsole->deleteDocument(
'schedules',
$schedule['$id'],
);
unset($this->schedules[$schedule['$internalId']]);
}
$queue->reclaim();
}
}
@@ -35,7 +35,7 @@ class ScheduleMessages extends ScheduleBase
continue;
}
\go(function () use ($now, $schedule, $pools, $dbForConsole) {
\go(function () use ($schedule, $pools, $dbForConsole) {
$queue = $pools->get('queue')->pop();
$connection = $queue->getResource();
$queueForMessaging = new Messaging($connection);
@@ -53,7 +53,7 @@ class ScheduleMessages extends ScheduleBase
$queue->reclaim();
unset($this->schedules[$schedule['resourceId']]);
unset($this->schedules[$schedule['$internalId']]);
});
}
}
+3 -3
View File
@@ -259,7 +259,7 @@ class Specs extends Action
$specs = new Specification($formatInstance);
$endpoint = System::getEnv('_APP_HOME', '[HOSTNAME]');
$email = System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM);
$email = System::getEnv('_APP_SYSTEM_TEAM_EMAIL', APP_EMAIL_TEAM);
$formatInstance
->setParam('name', APP_NAME)
@@ -280,7 +280,7 @@ class Specs extends Action
if ($mocks) {
$path = __DIR__ . '/../../../../app/config/specs/' . $format . '-mocks-' . $platform . '.json';
if (!file_put_contents($path, json_encode($specs->parse()))) {
if (!file_put_contents($path, json_encode($specs->parse(), JSON_PRETTY_PRINT))) {
throw new Exception('Failed to save mocks spec file: ' . $path);
}
@@ -291,7 +291,7 @@ class Specs extends Action
$path = __DIR__ . '/../../../../app/config/specs/' . $format . '-' . $version . '-' . $platform . '.json';
if (!file_put_contents($path, json_encode($specs->parse()))) {
if (!file_put_contents($path, json_encode($specs->parse(), JSON_PRETTY_PRINT))) {
throw new Exception('Failed to save spec file: ' . $path);
}
+214 -33
View File
@@ -2,12 +2,14 @@
namespace Appwrite\Platform\Workers;
use Ahc\Jwt\JWT;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Utopia\Response\Model\Deployment;
use Appwrite\Vcs\Comment;
use Exception;
use Executor\Executor;
use Swoole\Coroutine as Co;
use Utopia\Cache\Cache;
@@ -140,6 +142,7 @@ class Builds extends Action
}
$version = $function->getAttribute('version', 'v2');
$spec = Config::getParam('runtime-specifications')[$function->getAttribute('specifications', APP_FUNCTION_SPECIFICATION_DEFAULT)];
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);
$key = $function->getAttribute('runtime');
$runtime = $runtimes[$key] ?? null;
@@ -156,9 +159,9 @@ class Builds extends Action
$startTime = DateTime::now();
$durationStart = \microtime(true);
$buildId = $deployment->getAttribute('buildId', '');
$build = $dbForProject->getDocument('builds', $buildId);
$isNewBuild = empty($buildId);
if ($isNewBuild) {
if ($build->isEmpty()) {
$buildId = ID::unique();
$build = $dbForProject->createDocument('builds', new Document([
'$id' => $buildId,
@@ -180,6 +183,9 @@ class Builds extends Action
$deployment->setAttribute('buildId', $build->getId());
$deployment->setAttribute('buildInternalId', $build->getInternalId());
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
} elseif ($build->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
} else {
$build = $dbForProject->getDocument('builds', $buildId);
}
@@ -202,7 +208,60 @@ class Builds extends Action
}
try {
if ($isNewBuild && $isVcsEnabled) {
if ($isNewBuild && !$isVcsEnabled) {
// Non-vcs+Template
$templateRepositoryName = $template->getAttribute('repositoryName', '');
$templateOwnerName = $template->getAttribute('ownerName', '');
$templateVersion = $template->getAttribute('version', '');
$templateRootDirectory = $template->getAttribute('rootDirectory', '');
$templateRootDirectory = \rtrim($templateRootDirectory, '/');
$templateRootDirectory = \ltrim($templateRootDirectory, '.');
$templateRootDirectory = \ltrim($templateRootDirectory, '/');
if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateVersion)) {
$stdout = '';
$stderr = '';
// Clone template repo
$tmpTemplateDirectory = '/tmp/builds/' . $buildId . '-template';
$gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateVersion, GitHub::CLONE_TYPE_TAG, $tmpTemplateDirectory, $templateRootDirectory);
$exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr);
if ($exit !== 0) {
throw new \Exception('Unable to clone code repository: ' . $stderr);
}
// Ensure directories
Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr);
$tmpPathFile = $tmpTemplateDirectory . '/code.tar.gz';
$localDevice = new Local();
if (substr($tmpTemplateDirectory, -1) !== '/') {
$tmpTemplateDirectory .= '/';
}
$tarParamDirectory = \escapeshellarg($tmpTemplateDirectory . (empty($templateRootDirectory) ? '' : '/' . $templateRootDirectory));
Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax
$source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$result = $localDevice->transfer($tmpPathFile, $source, $deviceForFunctions);
if (!$result) {
throw new \Exception("Unable to move file");
}
Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr);
$directorySize = $deviceForFunctions->getFileSize($source);
$build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source));
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize));
}
} elseif ($isNewBuild && $isVcsEnabled) {
// VCS and VCS+Temaplte
$tmpDirectory = '/tmp/builds/' . $buildId . '/code';
$rootDirectory = $function->getAttribute('providerRootDirectory', '');
$rootDirectory = \rtrim($rootDirectory, '/');
@@ -217,30 +276,60 @@ class Builds extends Action
$branchName = $deployment->getAttribute('providerBranch');
$commitHash = $deployment->getAttribute('providerCommitHash', '');
$gitCloneCommand = $github->generateCloneCommand($cloneOwner, $cloneRepository, $branchName, $tmpDirectory, $rootDirectory, $commitHash);
$cloneVersion = $branchName;
$cloneType = GitHub::CLONE_TYPE_BRANCH;
if (!empty($commitHash)) {
$cloneVersion = $commitHash;
$cloneType = GitHub::CLONE_TYPE_COMMIT;
}
$gitCloneCommand = $github->generateCloneCommand($cloneOwner, $cloneRepository, $cloneVersion, $cloneType, $tmpDirectory, $rootDirectory);
$stdout = '';
$stderr = '';
Console::execute('mkdir -p /tmp/builds/' . \escapeshellcmd($buildId), '', $stdout, $stderr);
Console::execute('mkdir -p ' . \escapeshellarg('/tmp/builds/' . $buildId), '', $stdout, $stderr);
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
}
$exit = Console::execute($gitCloneCommand, '', $stdout, $stderr);
if ($exit !== 0) {
throw new \Exception('Unable to clone code repository: ' . $stderr);
}
// Local refactoring for function folder with spaces
if (str_contains($rootDirectory, ' ')) {
$rootDirectoryWithoutSpaces = str_replace(' ', '', $rootDirectory);
$from = $tmpDirectory . '/' . $rootDirectory;
$to = $tmpDirectory . '/' . $rootDirectoryWithoutSpaces;
$exit = Console::execute('mv "' . \escapeshellarg($from) . '" "' . \escapeshellarg($to) . '"', '', $stdout, $stderr);
if ($exit !== 0) {
throw new \Exception('Unable to move function with spaces' . $stderr);
}
$rootDirectory = $rootDirectoryWithoutSpaces;
}
// Build from template
$templateRepositoryName = $template->getAttribute('repositoryName', '');
$templateOwnerName = $template->getAttribute('ownerName', '');
$templateBranch = $template->getAttribute('branch', '');
$templateVersion = $template->getAttribute('version', '');
$templateRootDirectory = $template->getAttribute('rootDirectory', '');
$templateRootDirectory = \rtrim($templateRootDirectory, '/');
$templateRootDirectory = \ltrim($templateRootDirectory, '.');
$templateRootDirectory = \ltrim($templateRootDirectory, '/');
if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateBranch)) {
if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateVersion)) {
// Clone template repo
$tmpTemplateDirectory = '/tmp/builds/' . \escapeshellcmd($buildId) . '/template';
$gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateBranch, $tmpTemplateDirectory, $templateRootDirectory);
$tmpTemplateDirectory = '/tmp/builds/' . $buildId . '/template';
$gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateVersion, GitHub::CLONE_TYPE_TAG, $tmpTemplateDirectory, $templateRootDirectory);
$exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr);
if ($exit !== 0) {
@@ -248,20 +337,20 @@ class Builds extends Action
}
// Ensure directories
Console::execute('mkdir -p ' . $tmpTemplateDirectory . '/' . $templateRootDirectory, '', $stdout, $stderr);
Console::execute('mkdir -p ' . $tmpDirectory . '/' . $rootDirectory, '', $stdout, $stderr);
Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr);
Console::execute('mkdir -p ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr);
// Merge template into user repo
Console::execute('rsync -av --exclude \'.git\' ' . $tmpTemplateDirectory . '/' . $templateRootDirectory . '/ ' . $tmpDirectory . '/' . $rootDirectory, '', $stdout, $stderr);
Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr);
// Commit and push
$exit = Console::execute('git config --global user.email "team@appwrite.io" && git config --global user.name "Appwrite" && cd ' . $tmpDirectory . ' && git add . && git commit -m "Create \'' . \escapeshellcmd($function->getAttribute('name', '')) . '\' function" && git push origin ' . \escapeshellcmd($branchName), '', $stdout, $stderr);
$exit = Console::execute('git config --global user.email "team@appwrite.io" && git config --global user.name "Appwrite" && cd ' . \escapeshellarg($tmpDirectory) . ' && git add . && git commit -m "Create ' . \escapeshellarg($function->getAttribute('name', '')) . ' function" && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr);
if ($exit !== 0) {
throw new \Exception('Unable to push code repository: ' . $stderr);
}
$exit = Console::execute('cd ' . $tmpDirectory . ' && git rev-parse HEAD', '', $stdout, $stderr);
$exit = Console::execute('cd ' . \escapeshellarg($tmpDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr);
if ($exit !== 0) {
throw new \Exception('Unable to get vcs commit SHA: ' . $stderr);
@@ -295,7 +384,7 @@ class Builds extends Action
);
}
$tmpPath = '/tmp/builds/' . \escapeshellcmd($buildId);
$tmpPath = '/tmp/builds/' . $buildId;
$tmpPathFile = $tmpPath . '/code.tar.gz';
$localDevice = new Local();
@@ -304,26 +393,28 @@ class Builds extends Action
}
$directorySize = $localDevice->getDirectorySize($tmpDirectory);
$functionsSizeLimit = (int) System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000');
$functionsSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000');
if ($directorySize > $functionsSizeLimit) {
throw new \Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / 1048576, 2) . ' MBs.');
}
Console::execute('tar --exclude code.tar.gz -czf ' . $tmpPathFile . ' -C /tmp/builds/' . \escapeshellcmd($buildId) . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory) . ' .', '', $stdout, $stderr);
$tarParamDirectory = '/tmp/builds/' . $buildId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory);
Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax
$path = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$result = $localDevice->transfer($tmpPathFile, $path, $deviceForFunctions);
$source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$result = $localDevice->transfer($tmpPathFile, $source, $deviceForFunctions);
if (!$result) {
throw new \Exception("Unable to move file");
}
Console::execute('rm -rf ' . $tmpPath, '', $stdout, $stderr);
$source = $path;
Console::execute('rm -rf ' . \escapeshellarg($tmpPath), '', $stdout, $stderr);
$build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source));
$directorySize = $deviceForFunctions->getFileSize($source);
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize));
$this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
}
@@ -345,8 +436,7 @@ class Builds extends Action
->setEvent('functions.[functionId].deployments.[deploymentId].update')
->setParam('functionId', $function->getId())
->setParam('deploymentId', $deployment->getId())
->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules())))
;
->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules())));
$deploymentUpdate->trigger();
@@ -383,14 +473,47 @@ class Builds extends Action
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
}
$cpus = $spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT;
$memory = max($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT, 1024); // We have a minimum of 1024MB here because some runtimes can't compile with less memory than this.
$jwtExpiry = (int)System::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
$jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0);
$apiKey = $jwtObj->encode([
'projectId' => $project->getId(),
'scopes' => $function->getAttribute('scopes', [])
]);
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https';
$hostname = System::getEnv('_APP_DOMAIN');
$endpoint = $protocol . '://' . $hostname . "/v1";
// Appwrite vars
$vars = \array_merge($vars, [
'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint,
'APPWRITE_FUNCTION_API_KEY' => API_KEY_DYNAMIC . '_' . $apiKey,
'APPWRITE_FUNCTION_ID' => $function->getId(),
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'),
'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(),
'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
'APPWRITE_FUNCTION_CPUS' => $cpus,
'APPWRITE_FUNCTION_MEMORY' => $memory,
'APPWRITE_VERSION' => APP_VERSION_STABLE,
'APPWRITE_REGION' => $project->getAttribute('region'),
'APPWRITE_DEPLOYMENT_TYPE' => $deployment->getAttribute('type', ''),
'APPWRITE_VCS_REPOSITORY_ID' => $deployment->getAttribute('providerRepositoryId', ''),
'APPWRITE_VCS_REPOSITORY_NAME' => $deployment->getAttribute('providerRepositoryName', ''),
'APPWRITE_VCS_REPOSITORY_OWNER' => $deployment->getAttribute('providerRepositoryOwner', ''),
'APPWRITE_VCS_REPOSITORY_URL' => $deployment->getAttribute('providerRepositoryUrl', ''),
'APPWRITE_VCS_REPOSITORY_BRANCH' => $deployment->getAttribute('providerBranch', ''),
'APPWRITE_VCS_REPOSITORY_BRANCH_URL' => $deployment->getAttribute('providerBranchUrl', ''),
'APPWRITE_VCS_COMMIT_HASH' => $deployment->getAttribute('providerCommitHash', ''),
'APPWRITE_VCS_COMMIT_MESSAGE' => $deployment->getAttribute('providerCommitMessage', ''),
'APPWRITE_VCS_COMMIT_URL' => $deployment->getAttribute('providerCommitUrl', ''),
'APPWRITE_VCS_COMMIT_AUTHOR_NAME' => $deployment->getAttribute('providerCommitAuthor', ''),
'APPWRITE_VCS_COMMIT_AUTHOR_URL' => $deployment->getAttribute('providerCommitAuthorUrl', ''),
'APPWRITE_VCS_ROOT_DIRECTORY' => $deployment->getAttribute('providerRootDirectory', ''),
]);
$command = $deployment->getAttribute('commands', '');
@@ -398,8 +521,15 @@ class Builds extends Action
$response = null;
$err = null;
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
}
$isCanceled = false;
Co::join([
Co\go(function () use ($executor, &$response, $project, $deployment, $source, $function, $runtime, $vars, $command, &$err) {
Co\go(function () use ($executor, &$response, $project, $deployment, $source, $function, $runtime, $vars, $command, $cpus, $memory, &$err) {
try {
$version = $function->getAttribute('version', 'v2');
$command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . \trim(\escapeshellarg($command), "\'") . '"';
@@ -410,6 +540,8 @@ class Builds extends Action
source: $source,
image: $runtime['image'],
version: $version,
cpus: $cpus,
memory: $memory,
remove: true,
entrypoint: $deployment->getAttribute('entrypoint'),
destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}",
@@ -420,27 +552,38 @@ class Builds extends Action
$err = $error;
}
}),
Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err) {
Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err, &$isCanceled) {
try {
$executor->getLogs(
deploymentId: $deployment->getId(),
projectId: $project->getId(),
callback: function ($logs) use (&$response, &$build, $dbForProject, $allEvents, $project) {
if ($response === null) {
callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $allEvents, $project, &$isCanceled) {
if ($isCanceled) {
return;
}
// If we have response or error from concurrent coroutine, we already have latest logs
if ($response === null && $err === null) {
$build = $dbForProject->getDocument('builds', $build->getId());
if ($build->isEmpty()) {
throw new \Exception('Build not found', 404);
}
if ($build->getAttribute('status') === 'canceled') {
$isCanceled = true;
Console::info('Ignoring realtime logs because build has been canceled');
return;
}
$logs = \mb_substr($logs, 0, null, 'UTF-8'); // Get only valid UTF8 part - removes leftover half-multibytes causing SQL errors
$build = $build->setAttribute('logs', $build->getAttribute('logs', '') . $logs);
$build = $dbForProject->updateDocument('builds', $build->getId(), $build);
/**
* Send realtime Event
*/
* Send realtime Event
*/
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
event: $allEvents[0],
@@ -465,6 +608,11 @@ class Builds extends Action
}),
]);
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
}
if ($err) {
throw $err;
}
@@ -472,6 +620,11 @@ class Builds extends Action
$endTime = DateTime::now();
$durationEnd = \microtime(true);
$buildSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_BUILD_SIZE_LIMIT', '2000000000');
if ($response['size'] > $buildSizeLimit) {
throw new \Exception('Build size should be less than ' . number_format($buildSizeLimit / 1048576, 2) . ' MBs.');
}
/** Update the build document */
$build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime']))));
$build->setAttribute('endTime', $endTime);
@@ -481,6 +634,8 @@ class Builds extends Action
$build->setAttribute('size', $response['size']);
$build->setAttribute('logs', $response['output']);
$build = $dbForProject->updateDocument('builds', $buildId, $build);
if ($isVcsEnabled) {
$this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
}
@@ -495,6 +650,11 @@ class Builds extends Action
$function = $dbForProject->updateDocument('functions', $function->getId(), $function);
}
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
}
/** Update function schedule */
// Inform scheduler if function is still active
@@ -505,19 +665,24 @@ class Builds extends Action
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
} catch (\Throwable $th) {
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
}
$endTime = DateTime::now();
$durationEnd = \microtime(true);
$build->setAttribute('endTime', $endTime);
$build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart)));
$build->setAttribute('status', 'failed');
$build->setAttribute('logs', $th->getMessage() . "\n" . $th->getFile() . ':' . $th->getLine() . "\n" . $th->getTraceAsString());
$build->setAttribute('logs', $th->getMessage());
$build = $dbForProject->updateDocument('builds', $buildId, $build);
if ($isVcsEnabled) {
$this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
}
} finally {
$build = $dbForProject->updateDocument('builds', $buildId, $build);
/**
* Send realtime Event
*/
@@ -536,13 +701,29 @@ class Builds extends Action
);
/** Trigger usage queue */
if ($build->getAttribute('status') === 'ready') {
$queueForUsage
->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project
->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000);
} elseif ($build->getAttribute('status') === 'failed') {
$queueForUsage
->addMetric(METRIC_BUILDS_FAILED, 1) // per project
->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED), 1) // per function
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED), (int)$build->getAttribute('duration', 0) * 1000);
}
$queueForUsage
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->setProject($project)
->trigger();
}
+13 -9
View File
@@ -7,7 +7,7 @@ use Appwrite\Extend\Exception;
use Executor\Executor;
use Throwable;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\Abuse\Adapters\Database\TimeLimit;
use Utopia\Audit\Audit;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
@@ -95,12 +95,6 @@ class Deletes extends Action
case DELETE_TYPE_USERS:
$this->deleteUser($getProjectDB, $document, $project);
break;
case DELETE_TYPE_TEAMS:
$this->deleteMemberships($getProjectDB, $document, $project);
if ($project->getId() === 'console') {
$this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document);
}
break;
case DELETE_TYPE_BUCKETS:
$this->deleteBucket($getProjectDB, $deviceForFiles, $document, $project);
break;
@@ -115,6 +109,9 @@ class Deletes extends Action
break;
}
break;
case DELETE_TYPE_TEAM_PROJECTS:
$this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $document);
break;
case DELETE_TYPE_EXECUTIONS:
$this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
break;
@@ -419,7 +416,7 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteMemberships(callable $getProjectDB, Document $document, Document $project): void
public function deleteMemberships(callable $getProjectDB, Document $document, Document $project): void
{
$dbForProject = $getProjectDB($project);
$teamInternalId = $document->getInternalId();
@@ -447,14 +444,21 @@ class Deletes extends Action
* @throws Conflict
* @throws Restricted
* @throws Structure
* @throws Exception
*/
private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void
private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Document $document): void
{
$projects = $dbForConsole->find('projects', [
Query::equal('teamInternalId', [$document->getInternalId()])
]);
foreach ($projects as $project) {
$deviceForFiles = getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId());
$deviceForFunctions = getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId());
$deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId());
$deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId());
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project);
$dbForConsole->deleteDocument('projects', $project->getId());
}
+72 -25
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Workers;
use Ahc\Jwt\JWT;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
@@ -70,27 +71,40 @@ class Functions extends Action
throw new Exception('Missing payload');
}
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
$type = $payload['type'] ?? '';
$events = $payload['events'] ?? [];
$data = $payload['body'] ?? '';
$eventData = $payload['payload'] ?? '';
$project = new Document($payload['project'] ?? []);
$function = new Document($payload['function'] ?? []);
$functionId = $payload['functionId'] ?? '';
$user = new Document($payload['user'] ?? []);
$userId = $payload['userId'] ?? '';
$method = $payload['method'] ?? 'POST';
$headers = $payload['headers'] ?? [];
$path = $payload['path'] ?? '/';
$jwt = $payload['jwt'] ?? '';
if ($user->isEmpty() && !empty($userId)) {
$user = $dbForProject->getDocument('users', $userId);
}
if (empty($jwt) && !$user->isEmpty()) {
$jwtExpiry = $function->getAttribute('timeout', 900);
$jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0);
$jwt = $jwtObj->encode([
'userId' => $user->getId(),
]);
}
if ($project->getId() === 'console') {
return;
}
if ($function->isEmpty() && !empty($functionId)) {
$function = $dbForProject->getDocument('functions', $functionId);
}
$log->addTag('functionId', $function->getId());
$log->addTag('projectId', $project->getId());
$log->addTag('type', $type);
@@ -151,7 +165,6 @@ class Functions extends Action
*/
switch ($type) {
case 'http':
$jwt = $payload['jwt'] ?? '';
$execution = new Document($payload['execution'] ?? []);
$user = new Document($payload['user'] ?? []);
$this->execute(
@@ -175,6 +188,7 @@ class Functions extends Action
);
break;
case 'schedule':
$execution = new Document($payload['execution'] ?? []);
$this->execute(
log: $log,
dbForProject: $dbForProject,
@@ -187,12 +201,12 @@ class Functions extends Action
path: $path,
method: $method,
headers: $headers,
data: null,
user: null,
jwt: null,
data: $data,
user: $user,
jwt: $jwt,
event: null,
eventData: null,
executionId: null,
executionId: $execution->getId() ?? null
);
break;
}
@@ -253,9 +267,7 @@ class Functions extends Action
'search' => implode(' ', [$function->getId(), $executionId]),
]);
if ($function->getAttribute('logging')) {
$execution = $dbForProject->createDocument('executions', $execution);
}
$execution = $dbForProject->createDocument('executions', $execution);
if ($execution->isEmpty()) {
throw new Exception('Failed to create execution');
@@ -308,6 +320,7 @@ class Functions extends Action
$user ??= new Document();
$functionId = $function->getId();
$deploymentId = $function->getAttribute('deployment', '');
$spec = Config::getParam('runtime-specifications')[$function->getAttribute('specification', APP_FUNCTION_SPECIFICATION_DEFAULT)];
$log->addTag('deploymentId', $deploymentId);
@@ -354,10 +367,21 @@ class Functions extends Action
$runtime = $runtimes[$function->getAttribute('runtime')];
$jwtExpiry = $function->getAttribute('timeout', 900);
$jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0);
$apiKey = $jwtObj->encode([
'projectId' => $project->getId(),
'scopes' => $function->getAttribute('scopes', [])
]);
$headers['x-appwrite-key'] = API_KEY_DYNAMIC . '_' . $apiKey;
$headers['x-appwrite-trigger'] = $trigger;
$headers['x-appwrite-event'] = $event ?? '';
$headers['x-appwrite-user-id'] = $user->getId() ?? '';
$headers['x-appwrite-user-jwt'] = $jwt ?? '';
$headers['x-appwrite-country-code'] = '';
$headers['x-appwrite-continent-code'] = '';
$headers['x-appwrite-continent-eu'] = 'false';
/** Create execution or update execution status */
$execution = $dbForProject->getDocument('executions', $executionId ?? '');
@@ -390,9 +414,7 @@ class Functions extends Action
'search' => implode(' ', [$functionId, $executionId]),
]);
if ($function->getAttribute('logging')) {
$execution = $dbForProject->createDocument('executions', $execution);
}
$execution = $dbForProject->createDocument('executions', $execution);
// TODO: @Meldiron Trigger executions.create event here
@@ -404,9 +426,7 @@ class Functions extends Action
if ($execution->getAttribute('status') !== 'processing') {
$execution->setAttribute('status', 'processing');
if ($function->getAttribute('logging')) {
$execution = $dbForProject->updateDocument('executions', $executionId, $execution);
}
$execution = $dbForProject->updateDocument('executions', $executionId, $execution);
}
$durationStart = \microtime(true);
@@ -440,14 +460,36 @@ class Functions extends Action
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
}
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https';
$hostname = System::getEnv('_APP_DOMAIN');
$endpoint = $protocol . '://' . $hostname . "/v1";
// Appwrite vars
$vars = \array_merge($vars, [
'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint,
'APPWRITE_FUNCTION_ID' => $functionId,
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'),
'APPWRITE_FUNCTION_DEPLOYMENT' => $deploymentId,
'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
'APPWRITE_FUNCTION_CPUS' => ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT),
'APPWRITE_FUNCTION_MEMORY' => ($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT),
'APPWRITE_VERSION' => APP_VERSION_STABLE,
'APPWRITE_REGION' => $project->getAttribute('region'),
'APPWRITE_DEPLOYMENT_TYPE' => $deployment->getAttribute('type', ''),
'APPWRITE_VCS_REPOSITORY_ID' => $deployment->getAttribute('providerRepositoryId', ''),
'APPWRITE_VCS_REPOSITORY_NAME' => $deployment->getAttribute('providerRepositoryName', ''),
'APPWRITE_VCS_REPOSITORY_OWNER' => $deployment->getAttribute('providerRepositoryOwner', ''),
'APPWRITE_VCS_REPOSITORY_URL' => $deployment->getAttribute('providerRepositoryUrl', ''),
'APPWRITE_VCS_REPOSITORY_BRANCH' => $deployment->getAttribute('providerBranch', ''),
'APPWRITE_VCS_REPOSITORY_BRANCH_URL' => $deployment->getAttribute('providerBranchUrl', ''),
'APPWRITE_VCS_COMMIT_HASH' => $deployment->getAttribute('providerCommitHash', ''),
'APPWRITE_VCS_COMMIT_MESSAGE' => $deployment->getAttribute('providerCommitMessage', ''),
'APPWRITE_VCS_COMMIT_URL' => $deployment->getAttribute('providerCommitUrl', ''),
'APPWRITE_VCS_COMMIT_AUTHOR_NAME' => $deployment->getAttribute('providerCommitAuthor', ''),
'APPWRITE_VCS_COMMIT_AUTHOR_URL' => $deployment->getAttribute('providerCommitAuthorUrl', ''),
'APPWRITE_VCS_ROOT_DIRECTORY' => $deployment->getAttribute('providerRootDirectory', ''),
]);
/** Execute function */
@@ -469,10 +511,13 @@ class Functions extends Action
path: $path,
method: $method,
headers: $headers,
runtimeEntrypoint: $command
runtimeEntrypoint: $command,
cpus: $spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT,
memory: $spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT,
logging: $function->getAttribute('logging', true),
);
$status = $executionResponse['statusCode'] >= 400 ? 'failed' : 'completed';
$status = $executionResponse['statusCode'] >= 500 ? 'failed' : 'completed';
$headersFiltered = [];
foreach ($executionResponse['headers'] as $key => $value) {
@@ -507,13 +552,15 @@ class Functions extends Action
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->trigger()
;
}
if ($function->getAttribute('logging')) {
$execution = $dbForProject->updateDocument('executions', $executionId, $execution);
}
$execution = $dbForProject->updateDocument('executions', $executionId, $execution);
/** Trigger Webhook */
$executionModel = new Execution();
$queueForEvents
+24 -8
View File
@@ -101,7 +101,7 @@ class Messaging extends Action
case MESSAGE_SEND_TYPE_EXTERNAL:
$message = $dbForProject->getDocument('messages', $payload['messageId']);
$this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project);
$this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $queueForUsage);
break;
default:
throw new \Exception('Unknown message type: ' . $type);
@@ -113,6 +113,7 @@ class Messaging extends Action
Document $message,
Device $deviceForFiles,
Document $project,
Usage $queueForUsage
): void {
$topicIds = $message->getAttribute('topics', []);
$targetIds = $message->getAttribute('targets', []);
@@ -218,8 +219,8 @@ class Messaging extends Action
/**
* @var array<array> $results
*/
$results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project) {
return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project) {
$results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
if (\array_key_exists($providerId, $providers)) {
$provider = $providers[$providerId];
} else {
@@ -246,8 +247,8 @@ class Messaging extends Action
$adapter->getMaxMessagesPerRequest()
);
return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project) {
return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project) {
return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
$deliveredTotal = 0;
$deliveryErrors = [];
$messageData = clone $message;
@@ -286,6 +287,20 @@ class Messaging extends Action
} catch (\Throwable $e) {
$deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage();
} finally {
$errorTotal = count($deliveryErrors);
$queueForUsage
->setProject($project)
->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal))
->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal)
->addMetric(METRIC_MESSAGES_FAILED, $errorTotal)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), ($deliveredTotal + $errorTotal))
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $deliveredTotal)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal))
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal)
->trigger();
return [
'deliveredTotal' => $deliveredTotal,
'deliveryErrors' => $deliveryErrors,
@@ -318,6 +333,7 @@ class Messaging extends Action
$message->setAttribute('status', MessageStatus::SENT);
}
$message->removeAttribute('to');
foreach ($providers as $provider) {
@@ -452,10 +468,10 @@ class Messaging extends Action
$countryCode = $adapter->getCountryCode($message['to'][0] ?? '');
if (!empty($countryCode)) {
$queueForUsage
->addMetric(str_replace('{countryCode}', $countryCode, METRIC_MESSAGES_COUNTRY_CODE), 1);
->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1);
}
$queueForUsage
->addMetric(METRIC_MESSAGES, 1)
->addMetric(METRIC_AUTH_METHOD_PHONE, 1)
->setProject($project)
->trigger();
} catch (\Throwable $th) {
@@ -669,7 +685,7 @@ class Messaging extends Action
private function getLocalDevice($project): Local
{
if($this->localDevice === null) {
if ($this->localDevice === null) {
$this->localDevice = new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId());
}
+32
View File
@@ -184,8 +184,12 @@ class Usage extends Action
$deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS)));
$buildsFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED)));
$buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$buildsComputeSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS)));
$buildsComputeFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED)));
$executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
@@ -210,6 +214,20 @@ class Usage extends Action
];
}
if (!empty($buildsSuccess['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_SUCCESS,
'value' => ($buildsSuccess['value'] * -1),
];
}
if (!empty($buildsFailed['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_FAILED,
'value' => ($buildsFailed['value'] * -1),
];
}
if (!empty($buildsStorage['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_STORAGE,
@@ -224,6 +242,20 @@ class Usage extends Action
];
}
if (!empty($buildsComputeSuccess['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_COMPUTE_SUCCESS,
'value' => ($buildsComputeSuccess['value'] * -1),
];
}
if (!empty($buildsComputeFailed['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_COMPUTE_FAILED,
'value' => ($buildsComputeFailed['value'] * -1),
];
}
if (!empty($executions['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS,
+12 -2
View File
@@ -286,7 +286,13 @@ class Swagger2 extends Format
$validator = $validator->getValidator();
}
switch ((!empty($validator)) ? \get_class($validator) : '') {
$validatorClass = (!empty($validator)) ? \get_class($validator) : '';
if ($validatorClass === 'Utopia\Validator\AnyOf') {
$validator = $param['validator']->getValidators()[0];
$validatorClass = \get_class($validator);
}
switch ($validatorClass) {
case 'Utopia\Validator\Text':
case 'Utopia\Database\Validator\UID':
$node['type'] = $validator->getType();
@@ -338,6 +344,10 @@ class Swagger2 extends Format
$consumes = ['multipart/form-data'];
$node['type'] = 'file';
break;
case 'Appwrite\Functions\Validator\Payload':
$consumes = ['multipart/form-data'];
$node['type'] = 'payload';
break;
case 'Appwrite\Utopia\Database\Validator\Queries\Attributes':
case 'Appwrite\Utopia\Database\Validator\Queries\Buckets':
case 'Appwrite\Utopia\Database\Validator\Queries\Collections':
@@ -428,7 +438,7 @@ class Swagger2 extends Format
}
}
if ($allowed) {
if ($allowed && $validator->getType() === 'string') {
$node['enum'] = $validator->getList();
$node['x-enum-name'] = $this->getEnumName($route->getLabel('sdk.namespace', ''), $method, $name);
$node['x-enum-keys'] = $this->getEnumKeys($route->getLabel('sdk.namespace', ''), $method, $name);
@@ -9,7 +9,9 @@ class Deployments extends Base
'buildId',
'activate',
'entrypoint',
'commands'
'commands',
'type',
'size'
];
/**
@@ -8,7 +8,10 @@ class Executions extends Base
'trigger',
'status',
'responseStatusCode',
'duration'
'duration',
'requestMethod',
'requestPath',
'deploymentId'
];
/**
+151
View File
@@ -0,0 +1,151 @@
<?php
namespace Appwrite\Utopia\Fetch;
class BodyMultipart
{
/**
* @var array<string, mixed> $parts
*/
private array $parts = [];
private string $boundary = "";
public function __construct(string $boundary = null)
{
if (is_null($boundary)) {
$this->boundary = self::generateBoundary();
} else {
$this->boundary = $boundary;
}
}
public static function generateBoundary(): string
{
return '-----------------------------' . \uniqid();
}
public function load(string $body): self
{
$eol = "\r\n";
$sections = \explode('--' . $this->boundary, $body);
foreach ($sections as $section) {
if (empty($section)) {
continue;
}
if (strpos($section, $eol) === 0) {
$section = substr($section, \strlen($eol));
}
if (substr($section, -2) === $eol) {
$section = substr($section, 0, -1 * \strlen($eol));
}
if ($section == '--') {
continue;
}
$partChunks = \explode($eol . $eol, $section, 2);
if (\count($partChunks) < 2) {
continue; // Broken part
}
[ $partHeaders, $partBody ] = $partChunks;
$partHeaders = \explode($eol, $partHeaders);
$partName = "";
foreach ($partHeaders as $partHeader) {
if (!empty($partName)) {
break;
}
$partHeaderArray = \explode(':', $partHeader, 2);
$partHeaderName = \strtolower($partHeaderArray[0] ?? '');
$partHeaderValue = $partHeaderArray[1] ?? '';
if ($partHeaderName == "content-disposition") {
$dispositionChunks = \explode("; ", $partHeaderValue);
foreach ($dispositionChunks as $dispositionChunk) {
$dispositionChunkValues = \explode("=", $dispositionChunk, 2);
if (\count($dispositionChunkValues) >= 2) {
if ($dispositionChunkValues[0] === "name") {
$partName = \trim($dispositionChunkValues[1], "\"");
break;
}
}
}
}
}
if (!empty($partName)) {
$this->parts[$partName] = $partBody;
}
}
return $this;
}
/**
* @return array<string, mixed>
*/
public function getParts(): array
{
return $this->parts ?? [];
}
public function getPart(string $key, mixed $default = ''): mixed
{
return $this->parts[$key] ?? $default;
}
public function setPart(string $key, mixed $value): self
{
$this->parts[$key] = $value;
return $this;
}
public function getBoundary(): string
{
return $this->boundary;
}
public function setBoundary(string $boundary): self
{
$this->boundary = $boundary;
return $this;
}
public function exportHeader(): string
{
return 'multipart/form-data; boundary=' . $this->boundary;
}
public function exportBody(): string
{
$eol = "\r\n";
$query = '--' . $this->boundary;
foreach ($this->parts as $key => $value) {
$query .= $eol . 'Content-Disposition: form-data; name="' . $key . '"';
if (\is_array($value)) {
$query .= $eol . 'Content-Type: application/json';
$value = \json_encode($value);
}
$query .= $eol . $eol;
if ($value === false) {
$query .= 0 . $eol;
} else {
$query .= $value . $eol;
}
$query .= '--' . $this->boundary;
}
$query .= "--" . $eol;
return $query;
}
}
@@ -0,0 +1,24 @@
<?php
namespace Appwrite\Utopia\Request\Filters;
use Appwrite\Utopia\Request\Filter;
class V18 extends Filter
{
// Convert 1.5 params to 1.6
public function parse(array $content, string $model): array
{
switch ($model) {
case 'account.deleteMfaAuthenticator':
unset($content['otp']);
break;
case 'functions.create':
$content['templateVersion'] = $content['templateBranch'] ?? "";
unset($content['templateBranch']);
break;
}
return $content;
}
}
+80 -1
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Utopia;
use Appwrite\Utopia\Fetch\BodyMultipart;
use Appwrite\Utopia\Response\Filter;
use Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response\Model\Account;
@@ -72,6 +73,7 @@ use Appwrite\Utopia\Response\Model\Migration;
use Appwrite\Utopia\Response\Model\MigrationFirebaseProject;
use Appwrite\Utopia\Response\Model\MigrationReport;
use Appwrite\Utopia\Response\Model\Mock;
use Appwrite\Utopia\Response\Model\MockNumber;
use Appwrite\Utopia\Response\Model\None;
use Appwrite\Utopia\Response\Model\Phone;
use Appwrite\Utopia\Response\Model\Platform;
@@ -82,11 +84,15 @@ use Appwrite\Utopia\Response\Model\ProviderRepository;
use Appwrite\Utopia\Response\Model\Rule;
use Appwrite\Utopia\Response\Model\Runtime;
use Appwrite\Utopia\Response\Model\Session;
use Appwrite\Utopia\Response\Model\Specification;
use Appwrite\Utopia\Response\Model\Subscriber;
use Appwrite\Utopia\Response\Model\Target;
use Appwrite\Utopia\Response\Model\Team;
use Appwrite\Utopia\Response\Model\TemplateEmail;
use Appwrite\Utopia\Response\Model\TemplateFunction;
use Appwrite\Utopia\Response\Model\TemplateRuntime;
use Appwrite\Utopia\Response\Model\TemplateSMS;
use Appwrite\Utopia\Response\Model\TemplateVariable;
use Appwrite\Utopia\Response\Model\Token;
use Appwrite\Utopia\Response\Model\Topic;
use Appwrite\Utopia\Response\Model\UsageBuckets;
@@ -100,8 +106,10 @@ use Appwrite\Utopia\Response\Model\UsageStorage;
use Appwrite\Utopia\Response\Model\UsageUsers;
use Appwrite\Utopia\Response\Model\User;
use Appwrite\Utopia\Response\Model\Variable;
use Appwrite\Utopia\Response\Model\VcsContent;
use Appwrite\Utopia\Response\Model\Webhook;
use Exception;
use JsonException;
use Swoole\Http\Response as SwooleHTTPResponse;
// Keep last
use Utopia\Database\Document;
@@ -233,6 +241,8 @@ class Response extends SwooleResponse
public const MODEL_BRANCH = 'branch';
public const MODEL_BRANCH_LIST = 'branchList';
public const MODEL_DETECTION = 'detection';
public const MODEL_VCS_CONTENT = 'vcsContent';
public const MODEL_VCS_CONTENT_LIST = 'vcsContentList';
// Functions
public const MODEL_FUNCTION = 'function';
@@ -247,6 +257,12 @@ class Response extends SwooleResponse
public const MODEL_BUILD_LIST = 'buildList'; // Not used anywhere yet
public const MODEL_FUNC_PERMISSIONS = 'funcPermissions';
public const MODEL_HEADERS = 'headers';
public const MODEL_SPECIFICATION = 'specification';
public const MODEL_SPECIFICATION_LIST = 'specificationList';
public const MODEL_TEMPLATE_FUNCTION = 'templateFunction';
public const MODEL_TEMPLATE_FUNCTION_LIST = 'templateFunctionList';
public const MODEL_TEMPLATE_RUNTIME = 'templateRuntime';
public const MODEL_TEMPLATE_VARIABLE = 'templateVariable';
// Proxy
public const MODEL_PROXY_RULE = 'proxyRule';
@@ -266,6 +282,7 @@ class Response extends SwooleResponse
public const MODEL_WEBHOOK_LIST = 'webhookList';
public const MODEL_KEY = 'key';
public const MODEL_KEY_LIST = 'keyList';
public const MODEL_MOCK_NUMBER = 'mockNumber';
public const MODEL_AUTH_PROVIDER = 'authProvider';
public const MODEL_AUTH_PROVIDER_LIST = 'authProviderList';
public const MODEL_PLATFORM = 'platform';
@@ -335,6 +352,7 @@ class Response extends SwooleResponse
->setModel(new BaseList('Teams List', self::MODEL_TEAM_LIST, 'teams', self::MODEL_TEAM))
->setModel(new BaseList('Memberships List', self::MODEL_MEMBERSHIP_LIST, 'memberships', self::MODEL_MEMBERSHIP))
->setModel(new BaseList('Functions List', self::MODEL_FUNCTION_LIST, 'functions', self::MODEL_FUNCTION))
->setModel(new BaseList('Function Templates List', self::MODEL_TEMPLATE_FUNCTION_LIST, 'templates', self::MODEL_TEMPLATE_FUNCTION))
->setModel(new BaseList('Installations List', self::MODEL_INSTALLATION_LIST, 'installations', self::MODEL_INSTALLATION))
->setModel(new BaseList('Provider Repositories List', self::MODEL_PROVIDER_REPOSITORY_LIST, 'providerRepositories', self::MODEL_PROVIDER_REPOSITORY))
->setModel(new BaseList('Branches List', self::MODEL_BRANCH_LIST, 'branches', self::MODEL_BRANCH))
@@ -364,6 +382,8 @@ class Response extends SwooleResponse
->setModel(new BaseList('Target list', self::MODEL_TARGET_LIST, 'targets', self::MODEL_TARGET))
->setModel(new BaseList('Migrations List', self::MODEL_MIGRATION_LIST, 'migrations', self::MODEL_MIGRATION))
->setModel(new BaseList('Migrations Firebase Projects List', self::MODEL_MIGRATION_FIREBASE_PROJECT_LIST, 'projects', self::MODEL_MIGRATION_FIREBASE_PROJECT))
->setModel(new BaseList('Specifications List', self::MODEL_SPECIFICATION_LIST, 'specifications', self::MODEL_SPECIFICATION))
->setModel(new BaseList('VCS Content List', self::MODEL_VCS_CONTENT_LIST, 'contents', self::MODEL_VCS_CONTENT))
// Entities
->setModel(new Database())
->setModel(new Collection())
@@ -403,9 +423,13 @@ class Response extends SwooleResponse
->setModel(new Team())
->setModel(new Membership())
->setModel(new Func())
->setModel(new TemplateFunction())
->setModel(new TemplateRuntime())
->setModel(new TemplateVariable())
->setModel(new Installation())
->setModel(new ProviderRepository())
->setModel(new Detection())
->setModel(new VcsContent())
->setModel(new Branch())
->setModel(new Runtime())
->setModel(new Deployment())
@@ -414,6 +438,7 @@ class Response extends SwooleResponse
->setModel(new Project())
->setModel(new Webhook())
->setModel(new Key())
->setModel(new MockNumber())
->setModel(new AuthProvider())
->setModel(new Platform())
->setModel(new Variable())
@@ -440,6 +465,7 @@ class Response extends SwooleResponse
->setModel(new UsageFunction())
->setModel(new UsageProject())
->setModel(new Headers())
->setModel(new Specification())
->setModel(new Rule())
->setModel(new TemplateSMS())
->setModel(new TemplateEmail())
@@ -467,6 +493,7 @@ class Response extends SwooleResponse
*/
public const CONTENT_TYPE_YAML = 'application/x-yaml';
public const CONTENT_TYPE_NULL = 'null';
public const CONTENT_TYPE_MULTIPART = 'multipart/form-data';
/**
* List of defined output objects
@@ -537,7 +564,11 @@ class Response extends SwooleResponse
switch ($this->getContentType()) {
case self::CONTENT_TYPE_JSON:
$this->json(!empty($output) ? $output : new \stdClass());
try {
$this->json(!empty($output) ? $output : new \stdClass());
} catch (JsonException $e) {
throw new Exception('Failed to parse response: ' . $e->getMessage(), 400);
}
break;
case self::CONTENT_TYPE_YAML:
@@ -547,6 +578,10 @@ class Response extends SwooleResponse
case self::CONTENT_TYPE_NULL:
break;
case self::CONTENT_TYPE_MULTIPART:
$this->multipart(!empty($output) ? $output : new \stdClass());
break;
default:
if ($model === self::MODEL_NONE) {
$this->noContent();
@@ -678,6 +713,50 @@ class Response extends SwooleResponse
->send(\yaml_emit($data, YAML_UTF8_ENCODING));
}
/**
* Multipart
*
* This helper is for sending multipart/form-data HTTP response.
* It sets relevant content type header ('multipart/form-data') and convert a PHP array ($data) to valid Multipart using BodyMultipart
*
* @param array $data
*
* @return void
*/
public function multipart(array $data): void
{
$multipart = new BodyMultipart();
foreach ($data as $key => $value) {
$multipart->setPart($key, $value);
}
$this
->setContentType($multipart->exportHeader())
->send($multipart->exportBody());
}
/**
* JSON
*
* This helper is for sending JSON HTTP response.
* It sets relevant content type header ('application/json') and convert a PHP array ($data) to valid JSON using native json_encode
*
* @see http://en.wikipedia.org/wiki/JSON
*
* @param mixed $data
* @return void
*/
public function json($data): void
{
if (!is_array($data) && !$data instanceof \stdClass) {
throw new \Exception('Response body is not a valid JSON object.');
}
$this
->setContentType(Response::CONTENT_TYPE_JSON, self::CHARSET_UTF8)
->send(\json_encode($data, JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR));
}
/**
* @return array
*/
+11 -3
View File
@@ -33,9 +33,17 @@ class V16 extends Filter
protected function parseDeployment(array $content)
{
$content['buildStderr'] = '';
$content['buildStdout'] = $content['buildLogs'];
unset($content['buildLogs']);
if (isset($content['buildLogs'])) {
$content['buildStderr'] = '';
$content['buildStdout'] = $content['buildLogs'];
unset($content['buildLogs']);
}
if (isset($content['buildSize'])) {
$content['size'] += + $content['buildSize'] ?? 0;
unset($content['buildSize']);
}
return $content;
}
@@ -0,0 +1,57 @@
<?php
namespace Appwrite\Utopia\Response\Filters;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Filter;
class V18 extends Filter
{
// Convert 1.6 Data format to 1.5 format
public function parse(array $content, string $model): array
{
$parsedResponse = $content;
$parsedResponse = match($model) {
Response::MODEL_FUNCTION => $this->parseFunction($content),
Response::MODEL_EXECUTION => $this->parseExecution($content),
Response::MODEL_PROJECT => $this->parseProject($content),
Response::MODEL_RUNTIME => $this->parseRuntime($content),
default => $parsedResponse,
};
return $parsedResponse;
}
protected function parseExecution(array $content)
{
if (!empty($content['status']) && !empty($content['statusCode'])) {
if ($content['status'] === 'completed' && $content['statusCode'] >= 400 && $content['statusCode'] < 500) {
$content['status'] = 'failed';
}
}
unset($content['scheduledAt']);
return $content;
}
protected function parseFunction(array $content)
{
unset($content['scopes']);
unset($content['specification']);
return $content;
}
protected function parseProject(array $content)
{
unset($content['authMockNumbers']);
unset($content['authSessionAlerts']);
return $content;
}
protected function parseRuntime(array $content)
{
unset($content['key']);
return $content;
}
}
@@ -58,6 +58,12 @@ class Deployment extends Model
'default' => 0,
'example' => 128,
])
->addRule('buildSize', [
'type' => self::TYPE_INTEGER,
'description' => 'The build output size in bytes.',
'default' => 0,
'example' => 128,
])
->addRule('buildId', [
'type' => self::TYPE_STRING,
'description' => 'The current build ID.',
@@ -72,7 +72,8 @@ class Document extends Any
public function filter(DatabaseDocument $document): DatabaseDocument
{
$document->removeAttribute('$internalId');
$document->removeAttribute('$collection'); // $collection is the internal collection ID
$document->removeAttribute('$collection');
$document->removeAttribute('$tenant');
foreach ($document->getAttributes() as $attribute) {
if (\is_array($attribute)) {
@@ -4,6 +4,7 @@ namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use Utopia\Database\DateTime;
use Utopia\Database\Helpers\Role;
class Execution extends Model
@@ -110,6 +111,13 @@ class Execution extends Model
'default' => 0,
'example' => 0.400,
])
->addRule('scheduledAt', [
'type' => self::TYPE_DATETIME,
'description' => 'The scheduled time for execution. If left empty, execution will be queued immediately.',
'required' => false,
'default' => DateTime::now(),
'example' => self::TYPE_DATETIME_EXAMPLE,
])
;
}
+14 -1
View File
@@ -71,6 +71,13 @@ class Func extends Model
'default' => '',
'example' => '5e5ea5c16897e',
])
->addRule('scopes', [
'type' => self::TYPE_STRING,
'description' => 'Allowed permission scopes.',
'default' => [],
'example' => 'users.read',
'array' => true,
])
->addRule('vars', [
'type' => Response::MODEL_VARIABLE,
'description' => 'Function variables.',
@@ -112,7 +119,7 @@ class Func extends Model
->addRule('version', [
'type' => self::TYPE_STRING,
'description' => 'Version of Open Runtimes used for the function.',
'default' => 'v3',
'default' => 'v4',
'example' => 'v2',
])
->addRule('installationId', [
@@ -145,6 +152,12 @@ class Func extends Model
'default' => false,
'example' => false,
])
->addRule('specification', [
'type' => self::TYPE_STRING,
'description' => 'Machine specification for builds and executions.',
'default' => APP_FUNCTION_SPECIFICATION_DEFAULT,
'example' => APP_FUNCTION_SPECIFICATION_DEFAULT,
])
;
}
+1 -1
View File
@@ -60,7 +60,7 @@ class Key extends Model
])
->addRule('accessedAt', [
'type' => self::TYPE_DATETIME,
'description' => 'Most recent access date in ISO 8601 format. This attribute is only updated again after ' . APP_KEY_ACCCESS / 60 / 60 . ' hours.',
'description' => 'Most recent access date in ISO 8601 format. This attribute is only updated again after ' . APP_KEY_ACCESS / 60 / 60 . ' hours.',
'default' => '',
'example' => self::TYPE_DATETIME_EXAMPLE
])
@@ -0,0 +1,47 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class MockNumber extends Model
{
public function __construct()
{
$this
->addRule('phone', [
'type' => self::TYPE_STRING,
'description' => 'Mock phone number for testing phone authentication. Useful for testing phone authentication without sending an SMS.',
'default' => '',
'example' => '+1612842323',
])
->addRule('otp', [
'type' => self::TYPE_STRING,
'description' => 'Mock OTP for the number. ',
'default' => '',
'example' => '123456',
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName(): string
{
return 'Mock Number';
}
/**
* Get Type
*
* @return string
*/
public function getType(): string
{
return Response::MODEL_MOCK_NUMBER;
}
}
@@ -138,6 +138,19 @@ class Project extends Model
'default' => false,
'example' => true,
])
->addRule('authMockNumbers', [
'type' => Response::MODEL_MOCK_NUMBER,
'description' => 'An array of mock numbers and their corresponding verification codes (OTPs).',
'default' => [],
'array' => true,
'example' => [new \stdClass()],
])
->addRule('authSessionAlerts', [
'type' => self::TYPE_BOOLEAN,
'description' => 'Whether or not to send session alert emails to users.',
'default' => false,
'example' => true,
])
->addRule('oAuthProviders', [
'type' => Response::MODEL_AUTH_PROVIDER,
'description' => 'List of Auth Providers.',
@@ -321,6 +334,8 @@ class Project extends Model
$document->setAttribute('authPasswordHistory', $authValues['passwordHistory'] ?? 0);
$document->setAttribute('authPasswordDictionary', $authValues['passwordDictionary'] ?? false);
$document->setAttribute('authPersonalDataCheck', $authValues['personalDataCheck'] ?? false);
$document->setAttribute('authMockNumbers', $authValues['mockNumbers'] ?? []);
$document->setAttribute('authSessionAlerts', $authValues['sessionAlerts'] ?? false);
foreach ($auth as $index => $method) {
$key = $method['key'];
@@ -16,6 +16,12 @@ class Runtime extends Model
'default' => '',
'example' => 'python-3.8',
])
->addRule('key', [
'type' => self::TYPE_STRING,
'description' => 'Parent runtime key.',
'default' => '',
'example' => 'python',
])
->addRule('name', [
'type' => self::TYPE_STRING,
'description' => 'Runtime Name.',
@@ -0,0 +1,58 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class Specification extends Model
{
public function __construct()
{
$this->addRule('memory', [
'type' => self::TYPE_INTEGER,
'description' => 'Memory size in MB.',
'default' => 0,
'example' => 512
]);
$this->addRule('cpus', [
'type' => self::TYPE_FLOAT,
'description' => 'Number of CPUs.',
'default' => 0,
'example' => 1
]);
$this->addRule('enabled', [
'type' => self::TYPE_BOOLEAN,
'description' => 'Is size enabled.',
'default' => false,
'example' => true
]);
$this->addRule('slug', [
'type' => self::TYPE_STRING,
'description' => 'Size slug.',
'default' => '',
'example' => APP_FUNCTION_SPECIFICATION_DEFAULT
]);
}
/**
* Get Name
*
* @return string
*/
public function getName(): string
{
return 'Specification';
}
/**
* Get Type
*
* @return string
*/
public function getType(): string
{
return Response::MODEL_SPECIFICATION;
}
}
@@ -0,0 +1,142 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class TemplateFunction extends Model
{
public function __construct()
{
$this
->addRule('icon', [
'type' => self::TYPE_STRING,
'description' => 'Function Template Icon.',
'default' => '',
'example' => 'icon-lightning-bolt',
])
->addRule('id', [
'type' => self::TYPE_STRING,
'description' => 'Function Template ID.',
'default' => '',
'example' => 'starter',
])
->addRule('name', [
'type' => self::TYPE_STRING,
'description' => 'Function Template Name.',
'default' => '',
'example' => 'Starter function',
])
->addRule('tagline', [
'type' => self::TYPE_STRING,
'description' => 'Function Template Tagline.',
'default' => '',
'example' => 'A simple function to get started.',
])
->addRule('permissions', [
'type' => self::TYPE_STRING,
'description' => 'Execution permissions.',
'default' => [],
'example' => 'any',
'array' => true,
])
->addRule('events', [
'type' => self::TYPE_STRING,
'description' => 'Function trigger events.',
'default' => [],
'example' => 'account.create',
'array' => true,
])
->addRule('cron', [
'type' => self::TYPE_STRING,
'description' => 'Function execution schedult in CRON format.',
'default' => '',
'example' => '0 0 * * *',
])
->addRule('timeout', [
'type' => self::TYPE_INTEGER,
'description' => 'Function execution timeout in seconds.',
'default' => 15,
'example' => 300,
])
->addRule('useCases', [
'type' => self::TYPE_STRING,
'description' => 'Function use cases.',
'default' => [],
'example' => 'Starter',
'array' => true,
])
->addRule('runtimes', [
'type' => Response::MODEL_TEMPLATE_RUNTIME,
'description' => 'List of runtimes that can be used with this template.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('instructions', [
'type' => self::TYPE_STRING,
'description' => 'Function Template Instructions.',
'default' => '',
'example' => 'For documentation and instructions check out <link>.',
])
->addRule('vcsProvider', [
'type' => self::TYPE_STRING,
'description' => 'VCS (Version Control System) Provider.',
'default' => '',
'example' => 'github',
])
->addRule('providerRepositoryId', [
'type' => self::TYPE_STRING,
'description' => 'VCS (Version Control System) Repository ID',
'default' => '',
'example' => 'templates',
])
->addRule('providerOwner', [
'type' => self::TYPE_STRING,
'description' => 'VCS (Version Control System) Owner.',
'default' => '',
'example' => 'appwrite',
])
->addRule('providerVersion', [
'type' => self::TYPE_STRING,
'description' => 'VCS (Version Control System) branch version (tag).',
'default' => '',
'example' => 'main',
])
->addRule('variables', [
'type' => Response::MODEL_TEMPLATE_VARIABLE,
'description' => 'Function variables.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('scopes', [
'type' => self::TYPE_STRING,
'description' => 'Function scopes.',
'default' => [],
'example' => 'users.read',
'array' => true,
]);
}
/**
* Get Name
*
* @return string
*/
public function getName(): string
{
return 'Template Function';
}
/**
* Get Type
*
* @return string
*/
public function getType(): string
{
return Response::MODEL_TEMPLATE_FUNCTION;
}
}
@@ -0,0 +1,58 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class TemplateRuntime extends Model
{
public function __construct()
{
$this
->addRule('name', [
'type' => self::TYPE_STRING,
'description' => 'Runtime Name.',
'default' => '',
'example' => 'node-19.0',
])
->addRule('commands', [
'type' => self::TYPE_STRING,
'description' => 'The build command used to build the deployment.',
'default' => '',
'example' => 'npm install',
])
->addRule('entrypoint', [
'type' => self::TYPE_STRING,
'description' => 'The entrypoint file used to execute the deployment.',
'default' => '',
'example' => 'index.js',
])
->addRule('providerRootDirectory', [
'type' => self::TYPE_STRING,
'description' => 'Path to function in VCS (Version Control System) repository',
'default' => '',
'example' => 'node/starter',
]);
}
/**
* Get Name
*
* @return string
*/
public function getName(): string
{
return 'Template Runtime';
}
/**
* Get Type
*
* @return string
*/
public function getType(): string
{
return Response::MODEL_TEMPLATE_RUNTIME;
}
}
@@ -0,0 +1,70 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class TemplateVariable extends Model
{
public function __construct()
{
$this
->addRule('name', [
'type' => self::TYPE_STRING,
'description' => 'Variable Name.',
'default' => '',
'example' => 'APPWRITE_DATABASE_ID',
])
->addRule('description', [
'type' => self::TYPE_STRING,
'description' => 'Variable Description.',
'default' => '',
'example' => 'The ID of the Appwrite database that contains the collection to sync.',
])
->addRule('value', [
'type' => self::TYPE_STRING,
'description' => 'Variable Value.',
'default' => '',
'example' => '512',
])
->addRule('placeholder', [
'type' => self::TYPE_STRING,
'description' => 'Variable Placeholder.',
'default' => '',
'example' => '64a55...7b912',
])
->addRule('required', [
'type' => self::TYPE_BOOLEAN,
'description' => 'Is the variable required?',
'default' => false,
'example' => false,
])
->addRule('type', [
'type' => self::TYPE_STRING,
'description' => 'Variable Type.',
'default' => '',
'example' => 'password',
]);
}
/**
* Get Name
*
* @return string
*/
public function getName(): string
{
return 'Template Variable';
}
/**
* Get Type
*
* @return string
*/
public function getType(): string
{
return Response::MODEL_TEMPLATE_VARIABLE;
}
}
@@ -46,6 +46,12 @@ class UsageFunction extends Model
'default' => 0,
'example' => 0,
])
->addRule('buildsMbSecondsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of function builds mbSeconds.',
'default' => 0,
'example' => 0,
])
->addRule('executionsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of function executions.',
@@ -58,6 +64,12 @@ class UsageFunction extends Model
'default' => 0,
'example' => 0,
])
->addRule('executionsMbSecondsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of function executions mbSeconds.',
'default' => 0,
'example' => 0,
])
->addRule('deployments', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of function deployments per period.',
@@ -93,6 +105,13 @@ class UsageFunction extends Model
'example' => [],
'array' => true
])
->addRule('buildsMbSeconds', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of function builds mbSeconds per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executions', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of function executions per period.',
@@ -100,7 +119,6 @@ class UsageFunction extends Model
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of function executions compute time per period.',
@@ -108,6 +126,13 @@ class UsageFunction extends Model
'example' => [],
'array' => true
])
->addRule('executionsMbSeconds', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of function mbSeconds per period.',
'default' => [],
'example' => [],
'array' => true
])
;
}
@@ -52,6 +52,12 @@ class UsageFunctions extends Model
'default' => 0,
'example' => 0,
])
->addRule('buildsMbSecondsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of functions build mbSeconds.',
'default' => 0,
'example' => 0,
])
->addRule('executionsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of functions execution.',
@@ -64,6 +70,12 @@ class UsageFunctions extends Model
'default' => 0,
'example' => 0,
])
->addRule('executionsMbSecondsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of functions execution mbSeconds.',
'default' => 0,
'example' => 0,
])
->addRule('functions', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of functions per period.',
@@ -106,6 +118,13 @@ class UsageFunctions extends Model
'example' => [],
'array' => true
])
->addRule('buildsMbSeconds', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated sum of functions build mbSeconds per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executions', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of functions execution per period.',
@@ -113,7 +132,6 @@ class UsageFunctions extends Model
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of functions execution compute time per period.',
@@ -121,6 +139,13 @@ class UsageFunctions extends Model
'example' => [],
'array' => true
])
->addRule('executionsMbSeconds', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of functions mbSeconds per period.',
'default' => [],
'example' => [],
'array' => true
])
;
}
@@ -46,6 +46,18 @@ class UsageProject extends Model
'default' => 0,
'example' => 0,
])
->addRule('functionsStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of functions storage size (in bytes).',
'default' => 0,
'example' => 0,
])
->addRule('buildsStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of builds storage size (in bytes).',
'default' => 0,
'example' => 0,
])
->addRule('deploymentsStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of deployments storage size (in bytes).',
@@ -58,6 +70,18 @@ class UsageProject extends Model
'default' => 0,
'example' => 0,
])
->addRule('executionsMbSecondsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of function executions mbSeconds.',
'default' => 0,
'example' => 0,
])
->addRule('buildsMbSecondsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of function builds mbSeconds.',
'default' => 0,
'example' => 0,
])
->addRule('requests', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of requests per period.',
@@ -107,9 +131,23 @@ class UsageProject extends Model
'example' => [],
'array' => true
])
->addRule('deploymentsStorageBreakdown', [
->addRule('executionsMbSecondsBreakdown', [
'type' => Response::MODEL_METRIC_BREAKDOWN,
'description' => 'Aggregated breakdown in totals of deployments storage size (in bytes).',
'description' => 'Aggregated breakdown in totals of execution mbSeconds by functions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsMbSecondsBreakdown', [
'type' => Response::MODEL_METRIC_BREAKDOWN,
'description' => 'Aggregated breakdown in totals of build mbSeconds by functions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('functionsStorageBreakdown', [
'type' => Response::MODEL_METRIC_BREAKDOWN,
'description' => 'Aggregated breakdown in totals of functions storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true
+1 -1
View File
@@ -135,7 +135,7 @@ class User extends Model
])
->addRule('accessedAt', [
'type' => self::TYPE_DATETIME,
'description' => 'Most recent access date in ISO 8601 format. This attribute is only updated again after ' . APP_USER_ACCCESS / 60 / 60 . ' hours.',
'description' => 'Most recent access date in ISO 8601 format. This attribute is only updated again after ' . APP_USER_ACCESS / 60 / 60 . ' hours.',
'default' => '',
'example' => self::TYPE_DATETIME_EXAMPLE,
])
@@ -0,0 +1,55 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class VcsContent extends Model
{
public function __construct()
{
$this
->addRule('size', [
'type' => self::TYPE_INTEGER,
'description' => 'Content size in bytes. Only files have size, and for directories, 0 is returned.',
'default' => 0,
'required' => false,
'example' => 1523,
])
->addRule('isDirectory', [
'type' => self::TYPE_BOOLEAN,
'description' => 'If a content is a directory. Directories can be used to check nested contents.',
'default' => false,
'required' => false,
'example' => true
])
->addRule('name', [
'type' => self::TYPE_STRING,
'description' => 'Name of directory or file.',
'default' => "",
'example' => 'Main.java',
'array' => false,
]);
}
/**
* Get Name
*
* @return string
*/
public function getName(): string
{
return 'VcsContents';
}
/**
* Get Type
*
* @return string
*/
public function getType(): string
{
return Response::MODEL_VCS_CONTENT;
}
}
+53 -15
View File
@@ -3,6 +3,7 @@
namespace Executor;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Utopia\Fetch\BodyMultipart;
use Exception;
use Utopia\System\System;
@@ -24,10 +25,6 @@ class Executor
protected array $headers;
protected int $cpus;
protected int $memory;
public function __construct(string $endpoint)
{
if (!filter_var($endpoint, FILTER_VALIDATE_URL)) {
@@ -35,8 +32,6 @@ class Executor
}
$this->endpoint = $endpoint;
$this->cpus = \intval(System::getEnv('_APP_FUNCTIONS_CPUS', '1'));
$this->memory = \intval(System::getEnv('_APP_FUNCTIONS_MEMORY', '512'));
$this->headers = [
'content-type' => 'application/json',
'authorization' => 'Bearer ' . System::getEnv('_APP_EXECUTOR_SECRET', ''),
@@ -65,6 +60,8 @@ class Executor
string $source,
string $image,
string $version,
float $cpus,
int $memory,
bool $remove = false,
string $entrypoint = '',
string $destination = '',
@@ -74,6 +71,12 @@ class Executor
$runtimeId = "$projectId-$deploymentId-build";
$route = "/runtimes";
$timeout = (int) System::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
// Remove after migration
if ($version == 'v3') {
$version = 'v4';
}
$params = [
'runtimeId' => $runtimeId,
'source' => $source,
@@ -83,8 +86,8 @@ class Executor
'variables' => $variables,
'remove' => $remove,
'command' => $command,
'cpus' => $this->cpus,
'memory' => $this->memory,
'cpus' => $cpus,
'memory' => $memory,
'version' => $version,
'timeout' => $timeout,
];
@@ -161,6 +164,7 @@ class Executor
* @param string $source
* @param string $entrypoint
* @param string $runtimeEntrypoint
* @param bool $logging
*
* @return array
*/
@@ -177,7 +181,10 @@ class Executor
string $path,
string $method,
array $headers,
float $cpus,
int $memory,
string $runtimeEntrypoint = null,
bool $logging,
int $requestTimeout = null
) {
if (empty($headers['host'])) {
@@ -185,11 +192,16 @@ class Executor
}
$runtimeId = "$projectId-$deploymentId";
$route = '/runtimes/' . $runtimeId . '/execution';
$route = '/runtimes/' . $runtimeId . '/executions';
// Remove after migration
if ($version == 'v3') {
$version = 'v4';
}
$params = [
'runtimeId' => $runtimeId,
'variables' => $variables,
'body' => $body,
'timeout' => $timeout,
'path' => $path,
'method' => $method,
@@ -197,19 +209,25 @@ class Executor
'image' => $image,
'source' => $source,
'entrypoint' => $entrypoint,
'cpus' => $this->cpus,
'memory' => $this->memory,
'cpus' => $cpus,
'memory' => $memory,
'version' => $version,
'runtimeEntrypoint' => $runtimeEntrypoint,
'logging' => $logging,
'restartPolicy' => 'always' // Once utopia/orchestration has it, use DockerAPI::ALWAYS (0.13+)
];
if (!empty($body)) {
$params['body'] = $body;
}
// Safety timeout. Executor has timeout, and open runtime has soft timeout.
// This one shouldn't really happen, but prevents from unexpected networking behaviours.
if ($requestTimeout == null) {
$requestTimeout = $timeout + 15;
}
$response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $requestTimeout);
$response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId, 'content-type' => 'multipart/form-data', 'accept' => 'multipart/form-data' ], $params, true, $requestTimeout);
$status = $response['headers']['status-code'];
if ($status >= 400) {
@@ -217,6 +235,11 @@ class Executor
throw new \Exception($message, $status);
}
$response['body']['headers'] = \json_decode($response['body']['headers'] ?? '{}', true);
$response['body']['statusCode'] = \intval($response['body']['statusCode'] ?? 500);
$response['body']['duration'] = \floatval($response['body']['duration'] ?? 0);
$response['body']['startTime'] = \floatval($response['body']['startTime'] ?? \microtime(true));
return $response['body'];
}
@@ -248,7 +271,13 @@ class Executor
break;
case 'multipart/form-data':
$query = $this->flatten($params);
$multipart = new BodyMultipart();
foreach ($params as $key => $value) {
$multipart->setPart($key, $value);
}
$headers['content-type'] = $multipart->exportHeader();
$query = $multipart->exportBody();
break;
default:
@@ -315,7 +344,16 @@ class Executor
$curlErrorMessage = curl_error($ch);
if ($decode) {
switch (substr($responseType, 0, strpos($responseType, ';'))) {
$strpos = strpos($responseType, ';');
$strpos = \is_bool($strpos) ? \strlen($responseType) : $strpos;
switch (substr($responseType, 0, $strpos)) {
case 'multipart/form-data':
$boundary = \explode('boundary=', $responseHeaders['content-type'] ?? '')[1] ?? '';
$multipartResponse = new BodyMultipart($boundary);
$multipartResponse->load(\is_bool($responseBody) ? '' : $responseBody);
$responseBody = $multipartResponse->getParts();
break;
case 'application/json':
$json = json_decode($responseBody, true);