Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '1.5.x' into feat-ssr-dx

Browse Source
This commit is contained in:
Luke B. Silver
2024-02-22 13:22:41 +00:00
committed by GitHub
parent a3b88f2015 dc3608633b
commit f285173f4e
285 changed files with 1768 additions and 1138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Appwrite/Auth/Validator/Phone.php
+1 -1
View File
@@ -47,7 +47,7 @@ class Phone extends Validator
return true;
}
return !!\preg_match('/^\+[1-9]\d{1,14}$/', $value);
return !!\preg_match('/^\+[1-9]\d{6,14}$/', $value);
}
/**
src/Appwrite/Event/Delete.php
+15
View File
@@ -10,6 +10,7 @@ class Delete extends Event
{
protected string $type = '';
protected ?Document $document = null;
protected ?string $resourceType = null;
protected ?string $resource = null;
protected ?string $datetime = null;
protected ?string $hourlyUsageRetentionDatetime = null;
@@ -107,6 +108,19 @@ class Delete extends Event
return $this;
}
/**
* Sets the resource type for the delete event.
*
* @param string $resourceType
* @return self
*/
public function setResourceType(string $resourceType): self
{
$this->resourceType = $resourceType;
return $this;
}
/**
* Returns the set document for the delete event.
*
@@ -133,6 +147,7 @@ class Delete extends Event
'type' => $this->type,
'document' => $this->document,
'resource' => $this->resource,
'resourceType' => $this->resourceType,
'datetime' => $this->datetime,
'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime
]);
src/Appwrite/Event/Event.php
+3
View File
@@ -27,6 +27,9 @@ class Event
public const USAGE_QUEUE_NAME = 'v1-usage';
public const USAGE_CLASS_NAME = 'UsageV1';
public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump';
public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1';
public const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
public const WEBHOOK_CLASS_NAME = 'WebhooksV1';
src/Appwrite/Event/Messaging.php
+25 -1
View File
@@ -8,13 +8,13 @@ use Utopia\Queue\Client;
class Messaging extends Event
{
protected string $type = '';
protected ?string $messageId = null;
protected ?Document $message = null;
protected ?array $recipients = null;
protected ?string $scheduledAt = null;
protected ?string $providerType = null;
public function __construct(protected Connection $connection)
{
parent::__construct($connection);
@@ -24,6 +24,29 @@ class Messaging extends Event
->setClass(Event::MESSAGING_CLASS_NAME);
}
/**
* Sets type for the build event.
*
* @param string $type Can be `MESSAGE_TYPE_INTERNAL` or `MESSAGE_TYPE_EXTERNAL`.
* @return self
*/
public function setType(string $type): self
{
$this->type = $type;
return $this;
}
/**
* Returns set type for the function event.
*
* @return string
*/
public function getType(): string
{
return $this->type;
}
/**
* Sets recipient for the messaging event.
*
@@ -162,6 +185,7 @@ class Messaging extends Event
$client = new Client($this->queue, $this->connection);
return $client->enqueue([
'type' => $this->type,
'project' => $this->project,
'user' => $this->user,
'messageId' => $this->messageId,
src/Appwrite/Event/Usage.php
+1
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Event;
use Utopia\CLI\Console;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
use Utopia\Database\Document;
src/Appwrite/Event/UsageDump.php
+47
View File
@@ -0,0 +1,47 @@
<?php
namespace Appwrite\Event;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
class UsageDump extends Event
{
protected array $stats;
public function __construct(protected Connection $connection)
{
parent::__construct($connection);
$this
->setQueue(Event::USAGE_DUMP_QUEUE_NAME)
->setClass(Event::USAGE_DUMP_CLASS_NAME);
}
/**
* Add Stats.
*
* @param array $stats
* @return self
*/
public function setStats(array $stats): self
{
$this->stats = $stats;
return $this;
}
/**
* Sends metrics to the usage worker.
*
* @return string|bool
*/
public function trigger(): string|bool
{
$client = new Client($this->queue, $this->connection);
return $client->enqueue([
'stats' => $this->stats,
]);
}
}
src/Appwrite/Extend/Exception.php
+43 -42
View File
@@ -92,8 +92,9 @@ class Exception extends \Exception
public const USER_OAUTH2_BAD_REQUEST = 'user_oauth2_bad_request';
public const USER_OAUTH2_UNAUTHORIZED = 'user_oauth2_unauthorized';
public const USER_OAUTH2_PROVIDER_ERROR = 'user_oauth2_provider_error';
public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_alread_verified';
public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified';
public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_already_verified';
public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified';
public const USER_DELETION_PROHIBITED = 'user_deletion_prohibited';
public const USER_TARGET_NOT_FOUND = 'user_target_not_found';
public const USER_TARGET_ALREADY_EXISTS = 'user_target_already_exists';
public const USER_API_KEY_AND_SESSION_SET = 'user_key_and_session_set';
@@ -131,18 +132,19 @@ class Exception extends \Exception
public const STORAGE_INVALID_CONTENT_RANGE = 'storage_invalid_content_range';
public const STORAGE_INVALID_RANGE = 'storage_invalid_range';
public const STORAGE_INVALID_APPWRITE_ID = 'storage_invalid_appwrite_id';
public const STORAGE_FILE_NOT_PUBLIC = 'storage_file_not_public';
/** VCS */
public const INSTALLATION_NOT_FOUND = 'installation_not_found';
public const PROVIDER_REPOSITORY_NOT_FOUND = 'provider_repository_not_found';
public const REPOSITORY_NOT_FOUND = 'repository_not_found';
public const PROVIDER_CONTRIBUTION_CONFLICT = 'provider_contribution_conflict';
public const GENERAL_PROVIDER_FAILURE = 'general_provider_failure';
public const INSTALLATION_NOT_FOUND = 'installation_not_found';
public const PROVIDER_REPOSITORY_NOT_FOUND = 'provider_repository_not_found';
public const REPOSITORY_NOT_FOUND = 'repository_not_found';
public const PROVIDER_CONTRIBUTION_CONFLICT = 'provider_contribution_conflict';
public const GENERAL_PROVIDER_FAILURE = 'general_provider_failure';
/** Functions */
public const FUNCTION_NOT_FOUND = 'function_not_found';
public const FUNCTION_RUNTIME_UNSUPPORTED = 'function_runtime_unsupported';
public const FUNCTION_ENTRYPOINT_MISSING = 'function_entrypoint_missing';
public const FUNCTION_ENTRYPOINT_MISSING = 'function_entrypoint_missing';
/** Deployments */
public const DEPLOYMENT_NOT_FOUND = 'deployment_not_found';
@@ -215,10 +217,10 @@ class Exception extends \Exception
public const ROUTER_DOMAIN_NOT_CONFIGURED = 'router_domain_not_configured';
/** Proxy */
public const RULE_RESOURCE_NOT_FOUND = 'rule_resource_not_found';
public const RULE_NOT_FOUND = 'rule_not_found';
public const RULE_ALREADY_EXISTS = 'rule_already_exists';
public const RULE_VERIFICATION_FAILED = 'rule_verification_failed';
public const RULE_RESOURCE_NOT_FOUND = 'rule_resource_not_found';
public const RULE_NOT_FOUND = 'rule_not_found';
public const RULE_ALREADY_EXISTS = 'rule_already_exists';
public const RULE_VERIFICATION_FAILED = 'rule_verification_failed';
/** Keys */
public const KEY_NOT_FOUND = 'key_not_found';
@@ -235,53 +237,52 @@ class Exception extends \Exception
public const GRAPHQL_TOO_MANY_QUERIES = 'graphql_too_many_queries';
/** Migrations */
public const MIGRATION_NOT_FOUND = 'migration_not_found';
public const MIGRATION_ALREADY_EXISTS = 'migration_already_exists';
public const MIGRATION_IN_PROGRESS = 'migration_in_progress';
public const MIGRATION_PROVIDER_ERROR = 'migration_provider_error';
public const MIGRATION_NOT_FOUND = 'migration_not_found';
public const MIGRATION_ALREADY_EXISTS = 'migration_already_exists';
public const MIGRATION_IN_PROGRESS = 'migration_in_progress';
public const MIGRATION_PROVIDER_ERROR = 'migration_provider_error';
/** Realtime */
public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid';
public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages';
public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation';
public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid';
public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages';
public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation';
/** Health */
public const HEALTH_QUEUE_SIZE_EXCEEDED = 'health_queue_size_exceeded';
public const HEALTH_CERTIFICATE_EXPIRED = 'health_certificate_expired';
public const HEALTH_INVALID_HOST = 'health_invalid_host';
public const HEALTH_QUEUE_SIZE_EXCEEDED = 'health_queue_size_exceeded';
public const HEALTH_CERTIFICATE_EXPIRED = 'health_certificate_expired';
public const HEALTH_INVALID_HOST = 'health_invalid_host';
/** Provider */
public const PROVIDER_NOT_FOUND = 'provider_not_found';
public const PROVIDER_ALREADY_EXISTS = 'provider_already_exists';
public const PROVIDER_INCORRECT_TYPE = 'provider_incorrect_type';
public const PROVIDER_MISSING_CREDENTIALS = 'provider_missing_credentials';
public const PROVIDER_NOT_FOUND = 'provider_not_found';
public const PROVIDER_ALREADY_EXISTS = 'provider_already_exists';
public const PROVIDER_INCORRECT_TYPE = 'provider_incorrect_type';
public const PROVIDER_MISSING_CREDENTIALS = 'provider_missing_credentials';
/** Topic */
public const TOPIC_NOT_FOUND = 'topic_not_found';
public const TOPIC_ALREADY_EXISTS = 'topic_already_exists';
public const TOPIC_NOT_FOUND = 'topic_not_found';
public const TOPIC_ALREADY_EXISTS = 'topic_already_exists';
/** Subscriber */
public const SUBSCRIBER_NOT_FOUND = 'subscriber_not_found';
public const SUBSCRIBER_ALREADY_EXISTS = 'subscriber_already_exists';
public const SUBSCRIBER_NOT_FOUND = 'subscriber_not_found';
public const SUBSCRIBER_ALREADY_EXISTS = 'subscriber_already_exists';
/** Message */
public const MESSAGE_NOT_FOUND = 'message_not_found';
public const MESSAGE_MISSING_TARGET = 'message_missing_target';
public const MESSAGE_ALREADY_SENT = 'message_already_sent';
public const MESSAGE_ALREADY_PROCESSING = 'message_already_processing';
public const MESSAGE_ALREADY_FAILED = 'message_already_failed';
public const MESSAGE_ALREADY_SCHEDULED = 'message_already_scheduled';
public const MESSAGE_TARGET_NOT_EMAIL = 'message_target_not_email';
public const MESSAGE_TARGET_NOT_SMS = 'message_target_not_sms';
public const MESSAGE_TARGET_NOT_PUSH = 'message_target_not_push';
public const MESSAGE_MISSING_SCHEDULE = 'message_missing_schedule';
public const MESSAGE_NOT_FOUND = 'message_not_found';
public const MESSAGE_MISSING_TARGET = 'message_missing_target';
public const MESSAGE_ALREADY_SENT = 'message_already_sent';
public const MESSAGE_ALREADY_PROCESSING = 'message_already_processing';
public const MESSAGE_ALREADY_FAILED = 'message_already_failed';
public const MESSAGE_ALREADY_SCHEDULED = 'message_already_scheduled';
public const MESSAGE_TARGET_NOT_EMAIL = 'message_target_not_email';
public const MESSAGE_TARGET_NOT_SMS = 'message_target_not_sms';
public const MESSAGE_TARGET_NOT_PUSH = 'message_target_not_push';
public const MESSAGE_MISSING_SCHEDULE = 'message_missing_schedule';
/** Targets */
public const TARGET_PROVIDER_INVALID_TYPE = 'target_provider_invalid_type';
/** Schedules */
public const SCHEDULE_NOT_FOUND = 'schedule_not_found';
public const SCHEDULE_NOT_FOUND = 'schedule_not_found';
protected string $type = '';
src/Appwrite/Platform/Services/Tasks.php
-1
View File
@@ -42,7 +42,6 @@ class Tasks extends Service
->addAction(Install::getName(), new Install())
->addAction(Maintenance::getName(), new Maintenance())
->addAction(Migrate::getName(), new Migrate())
->addAction(Migrate::getName(), new Migrate())
->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
->addAction(QueueCount::getName(), new QueueCount())
->addAction(QueueRetry::getName(), new QueueRetry())
src/Appwrite/Platform/Services/Workers.php
+2 -2
View File
@@ -14,7 +14,7 @@ use Appwrite\Platform\Workers\Builds;
use Appwrite\Platform\Workers\Deletes;
use Appwrite\Platform\Workers\Hamster;
use Appwrite\Platform\Workers\Usage;
use Appwrite\Platform\Workers\UsageHook;
use Appwrite\Platform\Workers\UsageDump;
use Appwrite\Platform\Workers\Migrations;
class Workers extends Service
@@ -33,7 +33,7 @@ class Workers extends Service
->addAction(Messaging::getName(), new Messaging())
->addAction(Webhooks::getName(), new Webhooks())
->addAction(Hamster::getName(), new Hamster())
->addAction(UsageHook::getName(), new UsageHook())
->addAction(UsageDump::getName(), new UsageDump())
->addAction(Usage::getName(), new Usage())
->addAction(Migrations::getName(), new Migrations())
src/Appwrite/Platform/Tasks/ScheduleBase.php
+12 -2
View File
@@ -64,8 +64,13 @@ abstract class ScheduleBase extends Action
$getSchedule = function (Document $schedule) use ($dbForConsole, $getProjectDB): array {
$project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId'));
$collectionId = match ($schedule->getAttribute('resourceType')) {
'function' => 'functions',
'message' => 'messages'
};
$resource = $getProjectDB($project)->getDocument(
$schedule->getAttribute('resourceCollection'),
$collectionId,
$schedule->getAttribute('resourceId')
);
@@ -108,7 +113,12 @@ abstract class ScheduleBase extends Action
try {
$this->schedules[$document['resourceId']] = $getSchedule($document);
} catch (\Throwable $th) {
Console::error("Failed to load schedule for project {$document['projectId']} {$document['resourceCollection']} {$document['resourceId']}");
$collectionId = match ($document->getAttribute('resourceType')) {
'function' => 'functions',
'message' => 'messages'
};
Console::error("Failed to load schedule for project {$document['projectId']} {$collectionId} {$document['resourceId']}");
Console::error($th->getMessage());
}
}
src/Appwrite/Platform/Tasks/ScheduleMessages.php
+1
View File
@@ -50,6 +50,7 @@ class ScheduleMessages extends ScheduleBase
$queueForMessaging = new Messaging($connection);
$queueForMessaging
->setType(MESSAGE_SEND_TYPE_EXTERNAL)
->setMessageId($schedule['resourceId'])
->setProject($schedule['project'])
->trigger();
src/Appwrite/Platform/Workers/Builds.php
+12 -14
View File
@@ -25,6 +25,7 @@ use Utopia\Database\Helpers\ID;
use Utopia\Logger\Log;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\VCS\Adapter\Git\GitHub;
@@ -49,9 +50,9 @@ class Builds extends Action
->inject('queueForUsage')
->inject('cache')
->inject('dbForProject')
->inject('getFunctionsDevice')
->inject('deviceForFunctions')
->inject('log')
->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice, Log $log) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $getFunctionsDevice, $log));
->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log));
}
/**
@@ -62,12 +63,12 @@ class Builds extends Action
* @param Usage $queueForUsage
* @param Cache $cache
* @param Database $dbForProject
* @param callable $getFunctionsDevice
* @param Device $deviceForFunctions
* @param Log $log
* @return void
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice, Log $log): void
public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void
{
$payload = $message->getPayload() ?? [];
@@ -89,7 +90,7 @@ class Builds extends Action
case BUILD_TYPE_RETRY:
Console::info('Creating build for deployment: ' . $deployment->getId());
$github = new GitHub($cache);
$this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template, $log);
$this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template, $log);
break;
default:
@@ -98,7 +99,7 @@ class Builds extends Action
}
/**
* @param callable $getFunctionsDevice
* @param Device $deviceForFunctions
* @param Func $queueForFunctions
* @param Event $queueForEvents
* @param Usage $queueForUsage
@@ -114,7 +115,7 @@ class Builds extends Action
* @throws \Utopia\Database\Exception
* @throws Exception
*/
protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void
protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void
{
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
@@ -156,7 +157,6 @@ class Builds extends Action
$durationStart = \microtime(true);
$buildId = $deployment->getAttribute('buildId', '');
$isNewBuild = empty($buildId);
$deviceFunctions = $getFunctionsDevice($project->getId());
if ($isNewBuild) {
$buildId = ID::unique();
@@ -170,7 +170,7 @@ class Builds extends Action
'path' => '',
'runtime' => $function->getAttribute('runtime'),
'source' => $deployment->getAttribute('path', ''),
'sourceType' => strtolower($deviceFunctions->getType()),
'sourceType' => strtolower($deviceForFunctions->getType()),
'logs' => '',
'endTime' => null,
'duration' => 0,
@@ -188,7 +188,7 @@ class Builds extends Action
$installationId = $deployment->getAttribute('installationId', '');
$providerRepositoryId = $deployment->getAttribute('providerRepositoryId', '');
$providerCommitHash = $deployment->getAttribute('providerCommitHash', '');
$isVcsEnabled = $providerRepositoryId ? true : false;
$isVcsEnabled = !empty($providerRepositoryId);
$owner = '';
$repositoryName = '';
@@ -311,10 +311,8 @@ class Builds extends Action
Console::execute('tar --exclude code.tar.gz -czf ' . $tmpPathFile . ' -C /tmp/builds/' . \escapeshellcmd($buildId) . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory) . ' .', '', $stdout, $stderr);
$deviceFunctions = $getFunctionsDevice($project->getId());
$path = $deviceFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$result = $localDevice->transfer($tmpPathFile, $path, $deviceFunctions);
$path = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$result = $localDevice->transfer($tmpPathFile, $path, $deviceForFunctions);
if (!$result) {
throw new \Exception("Unable to move file");
src/Appwrite/Platform/Workers/Deletes.php
+73 -147
View File
@@ -44,22 +44,22 @@ class Deletes extends Action
->inject('message')
->inject('dbForConsole')
->inject('getProjectDB')
->inject('getFilesDevice')
->inject('getFunctionsDevice')
->inject('getBuildsDevice')
->inject('getCacheDevice')
->inject('deviceForFiles')
->inject('deviceForFunctions')
->inject('deviceForBuilds')
->inject('deviceForCache')
->inject('abuseRetention')
->inject('executionRetention')
->inject('auditRetention')
->inject('log')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $abuseRetention, $executionRetention, $auditRetention, $log));
->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log));
}
/**
* @throws Exception
* @throws Throwable
*/
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void
{
$payload = $message->getPayload() ?? [];
@@ -71,6 +71,7 @@ class Deletes extends Action
$datetime = $payload['datetime'] ?? null;
$hourlyUsageRetentionDatetime = $payload['hourlyUsageRetentionDatetime'] ?? null;
$resource = $payload['resource'] ?? null;
$resourceType = $payload['resourceType'] ?? null;
$document = new Document($payload['document'] ?? []);
$project = new Document($payload['project'] ?? []);
@@ -80,20 +81,14 @@ class Deletes extends Action
switch (\strval($type)) {
case DELETE_TYPE_DOCUMENT:
switch ($document->getCollection()) {
case DELETE_TYPE_DATABASES:
$this->deleteDatabase($getProjectDB, $document, $project);
break;
case DELETE_TYPE_COLLECTIONS:
$this->deleteCollection($getProjectDB, $document, $project);
break;
case DELETE_TYPE_PROJECTS:
$this->deleteProject($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $document);
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document);
break;
case DELETE_TYPE_FUNCTIONS:
$this->deleteFunction($dbForConsole, $getProjectDB, $getFunctionsDevice, $getBuildsDevice, $document, $project);
$this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project);
break;
case DELETE_TYPE_DEPLOYMENTS:
$this->deleteDeployment($getProjectDB, $getFunctionsDevice, $getBuildsDevice, $document, $project);
$this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project);
break;
case DELETE_TYPE_USERS:
$this->deleteUser($getProjectDB, $document, $project);
@@ -101,11 +96,11 @@ class Deletes extends Action
case DELETE_TYPE_TEAMS:
$this->deleteMemberships($getProjectDB, $document, $project);
if ($project->getId() === 'console') {
$this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $document);
$this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document);
}
break;
case DELETE_TYPE_BUCKETS:
$this->deleteBucket($getProjectDB, $getFilesDevice, $document, $project);
$this->deleteBucket($getProjectDB, $deviceForFiles, $document, $project);
break;
case DELETE_TYPE_INSTALLATIONS:
$this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project);
@@ -114,10 +109,6 @@ class Deletes extends Action
$this->deleteRule($dbForConsole, $document);
break;
default:
if (\str_starts_with($document->getCollection(), 'database_')) {
$this->deleteCollection($getProjectDB, $document, $project);
break;
}
Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
break;
}
@@ -147,7 +138,7 @@ class Deletes extends Action
$this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
break;
case DELETE_TYPE_CACHE_BY_RESOURCE:
$this->deleteCacheByResource($project, $getProjectDB, $resource);
$this->deleteCacheByResource($project, $getProjectDB, $resource, $resourceType);
break;
case DELETE_TYPE_CACHE_BY_TIMESTAMP:
$this->deleteCacheByDate($project, $getProjectDB, $datetime);
@@ -203,8 +194,13 @@ class Deletes extends Action
return;
}
$collectionId = match ($document->getAttribute('resourceType')) {
'function' => 'functions',
'message' => 'messages'
};
$resource = $getProjectDB($project)->getDocument(
$document->getAttribute('resourceCollection'),
$collectionId,
$document->getAttribute('resourceId')
);
@@ -327,32 +323,37 @@ class Deletes extends Action
* @param string $resource
* @return void
* @throws Authorization
* @param string|null $resourceType
* @throws Exception
*/
private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource): void
private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource, string $resourceType = null): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$document = $dbForProject->findOne('cache', [Query::equal('resource', [$resource])]);
if ($document) {
$cache = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
);
$cache = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
);
$this->deleteById(
$document,
$dbForProject,
function ($document) use ($cache, $projectId) {
$path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
if ($cache->purge($document->getId())) {
Console::success('Deleting cache file: ' . $path);
} else {
Console::error('Failed to delete cache file: ' . $path);
}
}
);
$query[] = Query::equal('resource', [$resource]);
if (!empty($resourceType)) {
$query[] = Query::equal('resourceType', [$resourceType]);
}
$this->deleteByGroup(
'cache',
$query,
$dbForProject,
function (Document $document) use ($cache, $projectId) {
$path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
if ($cache->purge($document->getId())) {
Console::success('Deleting cache file: ' . $path);
} else {
Console::error('Failed to delete cache file: ' . $path);
}
}
);
}
/**
@@ -392,72 +393,6 @@ class Deletes extends Action
);
}
/**
* @param callable $getProjectDB
* @param Document $document
* @param Document $project
* @return void
* @throws Exception
*/
private function deleteDatabase(callable $getProjectDB, Document $document, Document $project): void
{
$databaseId = $document->getId();
$dbForProject = $getProjectDB($project);
$this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($getProjectDB, $project) {
$this->deleteCollection($getProjectDB, $document, $project);
});
$dbForProject->deleteCollection('database_' . $document->getInternalId());
$this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId, $project);
}
/**
* @param callable $getProjectDB
* @param Document $document teams document
* @param Document $project
* @return void
* @throws Exception
*/
private function deleteCollection(callable $getProjectDB, Document $document, Document $project): void
{
$collectionId = $document->getId();
$collectionInternalId = $document->getInternalId();
$databaseId = $document->getAttribute('databaseId');
$databaseInternalId = $document->getAttribute('databaseInternalId');
$dbForProject = $getProjectDB($project);
$relationships = \array_filter(
$document->getAttribute('attributes'),
fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP
);
foreach ($relationships as $relationship) {
if (!$relationship['twoWay']) {
continue;
}
$relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']);
$dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']);
$dbForProject->purgeCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId());
$dbForProject->purgeCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId());
}
$dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId());
$this->deleteByGroup('attributes', [
Query::equal('databaseInternalId', [$databaseInternalId]),
Query::equal('collectionInternalId', [$collectionInternalId])
], $dbForProject);
$this->deleteByGroup('indexes', [
Query::equal('databaseInternalId', [$databaseInternalId]),
Query::equal('collectionInternalId', [$collectionInternalId])
], $dbForProject);
$this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId . '/collection/' . $collectionId, $project);
}
/**
* @param Database $dbForConsole
* @param callable $getProjectDB
@@ -511,14 +446,14 @@ class Deletes extends Action
* @throws Restricted
* @throws Structure
*/
private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Document $document): void
private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void
{
$projects = $dbForConsole->find('projects', [
Query::equal('teamInternalId', [$document->getInternalId()])
]);
foreach ($projects as $project) {
$this->deleteProject($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $project);
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project);
$dbForConsole->deleteDocument('projects', $project->getId());
}
}
@@ -526,17 +461,17 @@ class Deletes extends Action
/**
* @param Database $dbForConsole
* @param callable $getProjectDB
* @param callable $getFilesDevice
* @param callable $getFunctionsDevice
* @param callable $getBuildsDevice
* @param callable $getCacheDevice
* @param Device $deviceForFiles
* @param Device $deviceForFunctions
* @param Device $deviceForBuilds
* @param Device $deviceForCache
* @param Document $document
* @return void
* @throws Exception
* @throws Authorization
* @throws \Utopia\Database\Exception
*/
private function deleteProject(Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Document $document): void
private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void
{
$projectId = $document->getId();
$projectInternalId = $document->getInternalId();
@@ -602,15 +537,10 @@ class Deletes extends Action
}
// Delete all storage directories
$uploads = $getFilesDevice($projectId);
$functions = $getFunctionsDevice($projectId);
$builds = $getBuildsDevice($projectId);
$cache = $getCacheDevice($projectId);
$uploads->delete($uploads->getRoot(), true);
$functions->delete($functions->getRoot(), true);
$builds->delete($builds->getRoot(), true);
$cache->delete($cache->getRoot(), true);
$deviceForFiles->delete($deviceForFiles->getRoot(), true);
$deviceForFunctions->delete($deviceForFunctions->getRoot(), true);
$deviceForBuilds->delete($deviceForBuilds->getRoot(), true);
$deviceForCache->delete($deviceForCache->getRoot(), true);
}
/**
@@ -772,14 +702,14 @@ class Deletes extends Action
/**
* @param callable $getProjectDB
* @param callable $getFunctionsDevice
* @param callable $getBuildsDevice
* @param Device $deviceForFunctions
* @param Device $deviceForBuilds
* @param Document $document function document
* @param Document $project
* @return void
* @throws Exception
*/
private function deleteFunction(Database $dbForConsole, callable $getProjectDB, callable $getFunctionsDevice, callable $getBuildsDevice, Document $document, Document $project): void
private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
@@ -811,25 +741,25 @@ class Deletes extends Action
* Delete Deployments
*/
Console::info("Deleting deployments for function " . $functionId);
$functionsStorage = $getFunctionsDevice($projectId);
$deploymentInternalIds = [];
$this->deleteByGroup('deployments', [
Query::equal('resourceInternalId', [$functionInternalId])
], $dbForProject, function (Document $document) use ($functionsStorage, &$deploymentInternalIds) {
], $dbForProject, function (Document $document) use ($deviceForFunctions, &$deploymentInternalIds) {
$deploymentInternalIds[] = $document->getInternalId();
$this->deleteDeploymentFiles($functionsStorage, $document);
$this->deleteDeploymentFiles($deviceForFunctions, $document);
});
/**
* Delete builds
*/
Console::info("Deleting builds for function " . $functionId);
$buildsStorage = $getBuildsDevice($projectId);
foreach ($deploymentInternalIds as $deploymentInternalId) {
$this->deleteByGroup('builds', [
Query::equal('deploymentInternalId', [$deploymentInternalId])
], $dbForProject, function (Document $document) use ($buildsStorage) {
$this->deleteBuildFiles($buildsStorage, $document);
], $dbForProject, function (Document $document) use ($deviceForBuilds) {
$this->deleteBuildFiles($deviceForBuilds, $document);
});
}
@@ -929,14 +859,14 @@ class Deletes extends Action
/**
* @param callable $getProjectDB
* @param callable $getFunctionsDevice
* @param callable $getBuildsDevice
* @param Device $deviceForFunctions
* @param Device $deviceForBuilds
* @param Document $document
* @param Document $project
* @return void
* @throws Exception
*/
private function deleteDeployment(callable $getProjectDB, callable $getFunctionsDevice, callable $getBuildsDevice, Document $document, Document $project): void
private function deleteDeployment(callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
@@ -946,18 +876,17 @@ class Deletes extends Action
/**
* Delete deployment files
*/
$functionsStorage = $getFunctionsDevice($projectId);
$this->deleteDeploymentFiles($functionsStorage, $document);
$this->deleteDeploymentFiles($deviceForFunctions, $document);
/**
* Delete builds
*/
Console::info("Deleting builds for deployment " . $deploymentId);
$buildsStorage = $getBuildsDevice($projectId);
$this->deleteByGroup('builds', [
Query::equal('deploymentInternalId', [$deploymentInternalId])
], $dbForProject, function (Document $document) use ($buildsStorage) {
$this->deleteBuildFiles($buildsStorage, $document);
], $dbForProject, function (Document $document) use ($deviceForBuilds) {
$this->deleteBuildFiles($deviceForBuilds, $document);
});
/**
@@ -1101,21 +1030,18 @@ class Deletes extends Action
/**
* @param callable $getProjectDB
* @param callable $getFilesDevice
* @param Device $deviceForFiles
* @param Document $document
* @param Document $project
* @return void
*/
private function deleteBucket(callable $getProjectDB, callable $getFilesDevice, Document $document, Document $project): void
private function deleteBucket(callable $getProjectDB, Device $deviceForFiles, Document $document, Document $project): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$dbForProject->deleteCollection('bucket_' . $document->getInternalId());
$device = $getFilesDevice($projectId);
$device->deletePath($document->getId());
$deviceForFiles->deletePath($document->getId());
}
/**
src/Appwrite/Platform/Workers/Messaging.php
+171 -50
View File
@@ -2,11 +2,14 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Auth\Auth;
use Appwrite\Event\Usage;
use Appwrite\Extend\Exception;
use Appwrite\Messaging\Status as MessageStatus;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Validator\Authorization;
use Utopia\DSN\DSN;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
@@ -29,10 +32,13 @@ use Utopia\Messaging\Adapter\SMS\Textmagic;
use Utopia\Messaging\Adapter\SMS\Twilio;
use Utopia\Messaging\Adapter\SMS\Vonage;
use Utopia\Messaging\Messages\Email;
use Utopia\Messaging\Messages\Email\Attachment;
use Utopia\Messaging\Messages\Push;
use Utopia\Messaging\Messages\SMS;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\Storage\Device;
use Utopia\Storage\Storage;
use function Swoole\Coroutine\batch;
@@ -44,7 +50,7 @@ class Messaging extends Action
}
/**
* @throws Exception
* @throws \Exception
*/
public function __construct()
{
@@ -53,49 +59,62 @@ class Messaging extends Action
->inject('message')
->inject('log')
->inject('dbForProject')
->inject('deviceForFiles')
->inject('deviceForLocalFiles')
->inject('queueForUsage')
->callback(fn(Message $message, Log $log, Database $dbForProject, Usage $queueForUsage) => $this->action($message, $log, $dbForProject, $queueForUsage));
->callback(fn(Message $message, Log $log, Database $dbForProject, Device $deviceForFiles, Device $deviceForLocalFiles, Usage $queueForUsage) => $this->action($message, $log, $dbForProject, $deviceForFiles, $deviceForLocalFiles, $queueForUsage));
}
/**
* @param Message $message
* @param Log $log
* @param Database $dbForProject
* @param Device $deviceForFiles
* @param Device $deviceForLocalFiles
* @param Usage $queueForUsage
* @return void
* @throws Exception
* @throws \Exception
*/
public function action(Message $message, Log $log, Database $dbForProject, Usage $queueForUsage): void
{
public function action(
Message $message,
Log $log,
Database $dbForProject,
Device $deviceForFiles,
Device $deviceForLocalFiles,
Usage $queueForUsage
): void {
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
$type = $payload['type'] ?? '';
$project = new Document($payload['project'] ?? []);
if (
!\is_null($payload['message'])
&& !\is_null($payload['recipients'])
&& $payload['providerType'] === MESSAGE_TYPE_SMS
) {
// Message was triggered internally
$this->processInternalSMSMessage(
new Document($payload['message']),
new Document($payload['project'] ?? []),
$payload['recipients'],
$queueForUsage,
$log,
);
} else {
$message = $dbForProject->getDocument('messages', $payload['messageId']);
switch ($type) {
case MESSAGE_SEND_TYPE_INTERNAL:
$message = new Document($payload['message'] ?? []);
$recipients = $payload['recipients'] ?? [];
$this->processMessage($dbForProject, $message);
$this->sendInternalSMSMessage($message, $project, $recipients, $queueForUsage, $log);
break;
case MESSAGE_SEND_TYPE_EXTERNAL:
$message = $dbForProject->getDocument('messages', $payload['messageId']);
$this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $deviceForLocalFiles,);
break;
default:
throw new Exception('Unknown message type: ' . $type);
}
}
private function processMessage(Database $dbForProject, Document $message): void
{
private function sendExternalMessage(
Database $dbForProject,
Document $message,
Device $deviceForFiles,
Device $deviceForLocalFiles,
): void {
$topicIds = $message->getAttribute('topics', []);
$targetIds = $message->getAttribute('targets', []);
$userIds = $message->getAttribute('users', []);
@@ -199,8 +218,8 @@ class Messaging extends Action
/**
* @var array<array> $results
*/
$results = batch(\array_map(function ($providerId) use ($identifiers, $providers, $fallback, $message, $dbForProject) {
return function () use ($providerId, $identifiers, $providers, $fallback, $message, $dbForProject) {
$results = batch(\array_map(function ($providerId) use ($identifiers, $providers, $fallback, $message, $dbForProject, $deviceForFiles, $deviceForLocalFiles) {
return function () use ($providerId, $identifiers, $providers, $fallback, $message, $dbForProject, $deviceForFiles, $deviceForLocalFiles) {
if (\array_key_exists($providerId, $providers)) {
$provider = $providers[$providerId];
} else {
@@ -216,9 +235,9 @@ class Messaging extends Action
$identifiers = $identifiers[$providerId];
$adapter = match ($provider->getAttribute('type')) {
MESSAGE_TYPE_SMS => $this->sms($provider),
MESSAGE_TYPE_PUSH => $this->push($provider),
MESSAGE_TYPE_EMAIL => $this->email($provider),
MESSAGE_TYPE_SMS => $this->getSmsAdapter($provider),
MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider),
MESSAGE_TYPE_EMAIL => $this->getEmailAdapter($provider),
default => throw new Exception(Exception::PROVIDER_INCORRECT_TYPE)
};
@@ -226,17 +245,17 @@ class Messaging extends Action
$batches = \array_chunk($identifiers, $maxBatchSize);
$batchIndex = 0;
return batch(\array_map(function ($batch) use ($message, $provider, $adapter, &$batchIndex, $dbForProject) {
return function () use ($batch, $message, $provider, $adapter, &$batchIndex, $dbForProject) {
return batch(\array_map(function ($batch) use ($message, $provider, $adapter, &$batchIndex, $dbForProject, $deviceForFiles, $deviceForLocalFiles) {
return function () use ($batch, $message, $provider, $adapter, &$batchIndex, $dbForProject, $deviceForFiles, $deviceForLocalFiles) {
$deliveredTotal = 0;
$deliveryErrors = [];
$messageData = clone $message;
$messageData->setAttribute('to', $batch);
$data = match ($provider->getAttribute('type')) {
MESSAGE_TYPE_SMS => $this->buildSMSMessage($messageData, $provider),
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider),
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $deviceForLocalFiles),
default => throw new Exception(Exception::PROVIDER_INCORRECT_TYPE)
};
@@ -310,9 +329,40 @@ class Messaging extends Action
$message->setAttribute('deliveredAt', DateTime::now());
$dbForProject->updateDocument('messages', $message->getId(), $message);
// Delete any attachments that were downloaded to the local cache
if ($provider->getAttribute('type') === MESSAGE_TYPE_EMAIL) {
if ($deviceForFiles->getType() === Storage::DEVICE_LOCAL) {
return;
}
$data = $message->getAttribute('data');
$attachments = $data['attachments'] ?? [];
foreach ($attachments as $attachment) {
$bucketId = $attachment['bucketId'];
$fileId = $attachment['fileId'];
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if ($bucket->isEmpty()) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
if ($file->isEmpty()) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
}
$path = $file->getAttribute('path', '');
if ($deviceForLocalFiles->exists($path)) {
$deviceForLocalFiles->delete($path);
}
}
}
}
private function processInternalSMSMessage(Document $message, Document $project, array $recipients, Usage $queueForUsage, Log $log): void
private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Usage $queueForUsage, Log $log): void
{
if (empty(App::getEnv('_APP_SMS_PROVIDER')) || empty(App::getEnv('_APP_SMS_FROM'))) {
throw new \Exception('Skipped SMS processing. Missing "_APP_SMS_PROVIDER" or "_APP_SMS_FROM" environment variables.');
@@ -375,7 +425,7 @@ class Messaging extends Action
]
]);
$adapter = $this->sms($provider);
$adapter = $this->getSmsAdapter($provider);
$maxBatchSize = $adapter->getMaxMessagesPerRequest();
$batches = \array_chunk($recipients, $maxBatchSize);
@@ -385,7 +435,7 @@ class Messaging extends Action
return function () use ($batch, $message, $provider, $adapter, $batchIndex, $project, $queueForUsage) {
$message->setAttribute('to', $batch);
$data = $this->buildSMSMessage($message, $provider);
$data = $this->buildSmsMessage($message, $provider);
try {
$adapter->send($data);
@@ -401,11 +451,7 @@ class Messaging extends Action
}, $batches));
}
public function shutdown(): void
{
}
private function sms(Document $provider): ?SMSAdapter
private function getSmsAdapter(Document $provider): ?SMSAdapter
{
$credentials = $provider->getAttribute('credentials');
@@ -420,7 +466,7 @@ class Messaging extends Action
};
}
private function push(Document $provider): ?PushAdapter
private function getPushAdapter(Document $provider): ?PushAdapter
{
$credentials = $provider->getAttribute('credentials');
@@ -437,7 +483,7 @@ class Messaging extends Action
};
}
private function email(Document $provider): ?EmailAdapter
private function getEmailAdapter(Document $provider): ?EmailAdapter
{
$credentials = $provider->getAttribute('credentials', []);
$options = $provider->getAttribute('options', []);
@@ -463,8 +509,13 @@ class Messaging extends Action
};
}
private function buildEmailMessage(Database $dbForProject, Document $message, Document $provider): Email
{
private function buildEmailMessage(
Database $dbForProject,
Document $message,
Document $provider,
Device $deviceForFiles,
Device $deviceForLocalFiles,
): Email {
$fromName = $provider['options']['fromName'] ?? null;
$fromEmail = $provider['options']['fromEmail'] ?? null;
$replyToEmail = $provider['options']['replyToEmail'] ?? null;
@@ -474,8 +525,9 @@ class Messaging extends Action
$bccTargets = $data['bcc'] ?? [];
$cc = [];
$bcc = [];
$attachments = $data['attachments'] ?? [];
if (\count($ccTargets) > 0) {
if (!empty($ccTargets)) {
$ccTargets = $dbForProject->find('targets', [
Query::equal('$id', $ccTargets),
Query::limit(\count($ccTargets)),
@@ -485,7 +537,7 @@ class Messaging extends Action
}
}
if (\count($bccTargets) > 0) {
if (!empty($bccTargets)) {
$bccTargets = $dbForProject->find('targets', [
Query::equal('$id', $bccTargets),
Query::limit(\count($bccTargets)),
@@ -495,21 +547,77 @@ class Messaging extends Action
}
}
if (!empty($attachments)) {
foreach ($attachments as &$attachment) {
$bucketId = $attachment['bucketId'];
$fileId = $attachment['fileId'];
$bucket = $dbForProject->getDocument('buckets', $bucketId);
if ($bucket->isEmpty()) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
if ($file->isEmpty()) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
}
$mimes = Config::getParam('storage-mimes');
$path = $file->getAttribute('path', '');
if (!$deviceForFiles->exists($path)) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path);
}
$contentType = 'text/plain';
if (\in_array($file->getAttribute('mimeType'), $mimes)) {
$contentType = $file->getAttribute('mimeType');
}
if ($deviceForFiles->getType() !== Storage::DEVICE_LOCAL) {
$deviceForFiles->transfer($path, $path, $deviceForLocalFiles);
}
$attachment = new Attachment(
$file->getAttribute('name'),
$path,
$contentType
);
}
}
$to = $message['to'];
$subject = $data['subject'];
$content = $data['content'];
$html = $data['html'] ?? false;
return new Email($to, $subject, $content, $fromName, $fromEmail, $replyToName, $replyToEmail, $cc, $bcc, null, $html);
return new Email(
$to,
$subject,
$content,
$fromName,
$fromEmail,
$replyToName,
$replyToEmail,
$cc,
$bcc,
$attachments,
$html
);
}
private function buildSMSMessage(Document $message, Document $provider): SMS
private function buildSmsMessage(Document $message, Document $provider): SMS
{
$to = $message['to'];
$content = $message['data']['content'];
$from = $provider['options']['from'];
return new SMS($to, $content, $from);
return new SMS(
$to,
$content,
$from
);
}
private function buildPushMessage(Document $message): Push
@@ -519,12 +627,25 @@ class Messaging extends Action
$body = $message['data']['body'];
$data = $message['data']['data'] ?? null;
$action = $message['data']['action'] ?? null;
$image = $message['data']['image'] ?? null;
$sound = $message['data']['sound'] ?? null;
$icon = $message['data']['icon'] ?? null;
$color = $message['data']['color'] ?? null;
$tag = $message['data']['tag'] ?? null;
$badge = $message['data']['badge'] ?? null;
return new Push($to, $title, $body, $data, $action, $sound, $icon, $color, $tag, $badge);
return new Push(
$to,
$title,
$body,
$data,
$action,
$sound,
$image,
$icon,
$color,
$tag,
$badge
);
}
}
src/Appwrite/Platform/Workers/Usage.php
+38 -17
View File
@@ -3,8 +3,9 @@
namespace Appwrite\Platform\Workers;
use Exception;
use Appwrite\Event\UsageDump;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Platform\Action;
@@ -12,14 +13,12 @@ use Utopia\Queue\Message;
class Usage extends Action
{
protected static array $stats = [];
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
'inf' => '0000-00-00 00:00'
];
private array $stats = [];
private int $lastTriggeredTime = 0;
private int $keys = 0;
private const INFINITY_PERIOD = '_inf_';
private const KEYS_THRESHOLD = 10000;
protected const INFINITY_PERIOD = '_inf_';
public static function getName(): string
{
return 'usage';
@@ -35,26 +34,31 @@ class Usage extends Action
->desc('Usage worker')
->inject('message')
->inject('getProjectDB')
->callback(function (Message $message, callable $getProjectDB) {
$this->action($message, $getProjectDB);
->inject('queueForUsageDump')
->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) {
$this->action($message, $getProjectDB, $queueForUsageDump);
});
$this->lastTriggeredTime = time();
}
/**
* @param Message $message
* @param callable $getProjectDB
* @param UsageDump $queueForUsageDump
* @return void
* @throws \Utopia\Database\Exception
* @throws Exception
*/
public function action(Message $message, callable $getProjectDB): void
public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
//Todo Figure out way to preserve keys when the container is being recreated @shimonewman
$payload = $message->getPayload() ?? [];
$aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
$project = new Document($payload['project'] ?? []);
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
@@ -70,13 +74,31 @@ class Usage extends Action
);
}
self::$stats[$projectId]['project'] = $project;
$this->stats[$projectId]['project'] = $project;
foreach ($payload['metrics'] ?? [] as $metric) {
if (!isset(self::$stats[$projectId]['keys'][$metric['key']])) {
self::$stats[$projectId]['keys'][$metric['key']] = $metric['value'];
$this->keys++;
if (!isset($this->stats[$projectId]['keys'][$metric['key']])) {
$this->stats[$projectId]['keys'][$metric['key']] = $metric['value'];
continue;
}
self::$stats[$projectId]['keys'][$metric['key']] += $metric['value'];
$this->stats[$projectId]['keys'][$metric['key']] += $metric['value'];
}
// If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
if (
$this->keys >= self::KEYS_THRESHOLD ||
(time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0)
) {
Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
$queueForUsageDump
->setStats($this->stats)
->trigger();
$this->stats = [];
$this->keys = 0;
$this->lastTriggeredTime = time();
}
}
@@ -91,7 +113,6 @@ class Usage extends Action
*/
private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void
{
$dbForProject = $getProjectDB($project);
try {
src/Appwrite/Platform/Workers/UsageDump.php
+110
View File
@@ -0,0 +1,110 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Extend\Exception;
use Utopia\App;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Queue\Message;
class UsageDump extends Action
{
protected array $stats = [];
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
'inf' => '0000-00-00 00:00'
];
public static function getName(): string
{
return 'usage-dump';
}
/**
* @throws \Exception
*/
public function __construct()
{
$this
->inject('message')
->inject('getProjectDB')
->callback(function (Message $message, callable $getProjectDB) {
$this->action($message, $getProjectDB);
});
}
/**
* @param Message $message
* @param callable $getProjectDB
* @return void
* @throws Exception
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, callable $getProjectDB): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
// TODO: rename both usage workers @shimonewman
foreach ($payload['stats'] ?? [] as $stats) {
$project = new Document($stats['project'] ?? []);
$numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
if ($numberOfKeys === 0) {
continue;
}
console::log('[' . DateTime::now() . '] ProjectId [' . $project->getInternalId() . '] Database [' . $project['database'] . '] ' . $numberOfKeys . ' keys');
try {
$dbForProject = $getProjectDB($project);
foreach ($stats['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => App::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats',
$id,
'value',
$value
);
}
}
}
}
} catch (\Exception $e) {
console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage());
}
}
}
}
src/Appwrite/Platform/Workers/UsageHook.php
-103
View File
@@ -1,103 +0,0 @@
<?php
namespace Appwrite\Platform\Workers;
use Utopia\App;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Swoole\Timer;
use Utopia\Database\DateTime;
class UsageHook extends Usage
{
public static function getName(): string
{
return 'usageHook';
}
public function __construct()
{
$this
->setType(Action::TYPE_WORKER_START)
->inject('register')
->inject('getProjectDB')
->callback(function ($register, callable $getProjectDB) {
$this->action($register, $getProjectDB);
})
;
}
/**
* @param $register
* @param $getProjectDB
* @return void
*/
public function action($register, $getProjectDB): void
{
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60000');
Timer::tick($interval, function () use ($register, $getProjectDB) {
$offset = count(self::$stats);
$projects = array_slice(self::$stats, 0, $offset, true);
array_splice(self::$stats, 0, $offset);
foreach ($projects as $data) {
$numberOfKeys = !empty($data['keys']) ? count($data['keys']) : 0;
$projectInternalId = $data['project']->getInternalId();
$database = $data['project']['database'] ?? '';
console::warning('Ticker started ' . DateTime::now());
if ($numberOfKeys === 0) {
continue;
}
try {
$dbForProject = $getProjectDB($data['project']);
foreach ($data['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => App::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats',
$id,
'value',
$value
);
}
}
}
}
} catch (\Throwable $e) {
console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage());
}
}
});
}
}
src/Appwrite/Specification/Format.php
+5 -5
View File
@@ -208,10 +208,10 @@ abstract class Format
return 'MessagingUsageRange';
}
break;
case 'createSMS':
case 'createSms':
case 'createPush':
case 'createEmail':
case 'updateSMS':
case 'updateSms':
case 'updatePush':
case 'updateEmail':
switch ($param) {
@@ -255,9 +255,9 @@ abstract class Format
case 'deleteSmsTemplate':
switch ($param) {
case 'type':
return 'SMSTemplateType';
return 'SmsTemplateType';
case 'locale':
return 'SMSTemplateLocale';
return 'SmsTemplateLocale';
}
break;
case 'createPlatform':
@@ -288,7 +288,7 @@ abstract class Format
case 'updateServiceStatus':
switch ($param) {
case 'service':
return 'APIService';
return 'ApiService';
}
break;
}
src/Appwrite/Specification/Format/OpenAPI3.php
+6 -2
View File
@@ -302,7 +302,7 @@ class OpenAPI3 extends Format
case 'Utopia\Database\Validator\UID':
case 'Utopia\Validator\Text':
$node['schema']['type'] = $validator->getType();
$node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Validator\Boolean':
$node['schema']['type'] = $validator->getType();
@@ -313,7 +313,7 @@ class OpenAPI3 extends Format
$node['schema']['x-upload-id'] = true;
}
$node['schema']['type'] = $validator->getType();
$node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Database\Validator\DatetimeValidator':
$node['schema']['type'] = $validator->getType();
@@ -451,6 +451,10 @@ class OpenAPI3 extends Format
$node['format'] = 'int32';
}
break;
case 'Appwrite\Utopia\Database\Validator\CompoundUID':
$node['schema']['type'] = $validator->getType();
$node['schema']['x-example'] = '[ID1:ID2]';
break;
default:
$node['schema']['type'] = 'string';
break;
src/Appwrite/Specification/Format/Swagger2.php
+6 -2
View File
@@ -298,7 +298,7 @@ class Swagger2 extends Format
case 'Utopia\Validator\Text':
case 'Utopia\Database\Validator\UID':
$node['type'] = $validator->getType();
$node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Validator\Boolean':
$node['type'] = $validator->getType();
@@ -309,7 +309,7 @@ class Swagger2 extends Format
$node['x-upload-id'] = true;
}
$node['type'] = $validator->getType();
$node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Database\Validator\DatetimeValidator':
$node['type'] = $validator->getType();
@@ -446,6 +446,10 @@ class Swagger2 extends Format
$node['format'] = 'int32';
}
break;
case 'Appwrite\Utopia\Database\Validator\CompoundUID':
$node['type'] = $validator->getType();
$node['x-example'] = '[ID1:ID2]';
break;
default:
$node['type'] = 'string';
break;
src/Appwrite/Utopia/Database/Validator/CompoundUID.php
+58
View File
@@ -0,0 +1,58 @@
<?php
namespace Appwrite\Utopia\Database\Validator;
use Utopia\Database\Validator\UID;
use Utopia\Validator;
class CompoundUID extends Validator
{
public function getDescription(): string
{
return 'Must consist of multiple UIDs separated by a colon. Each UID must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, and underscore. Can\'t start with a special char.';
}
public function isArray(): bool
{
return false;
}
public function isValid($value): bool
{
if (!\is_string($value)) {
return false;
}
$ids = static::parse($value);
if (\count($ids) < 2) {
return false;
}
foreach ($ids as $id) {
$validator = new UID();
if (!$validator->isValid($id)) {
return false;
}
}
return true;
}
public function getType(): string
{
return self::TYPE_STRING;
}
public static function parse(string $key): array
{
$parts = \explode(':', $key);
$result = [];
foreach ($parts as $part) {
$result[] = $part;
}
return $result;
}
}