diff --git a/Dockerfile b/Dockerfile index 13b018df0f..d5e9f1c2b0 100755 --- a/Dockerfile +++ b/Dockerfile @@ -81,6 +81,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-databases && \ chmod +x /usr/local/bin/worker-deletes && \ chmod +x /usr/local/bin/worker-functions && \ + chmod +x /usr/local/bin/worker-insights && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-messaging && \ chmod +x /usr/local/bin/worker-migrations && \ diff --git a/app/config/collections.php b/app/config/collections.php index 8847c8279a..03d3de0cd2 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -3807,6 +3807,39 @@ $projectCollections = array_merge([ 'required' => false, 'array' => false, ], + [ + '$id' => ID::custom('screenshot'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 265536, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('auditJson'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16777215, // minimum size for LONGTEXT + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('auditHtml'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16777215, // minimum size for MEDIUMTEXT + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('size'), 'type' => Database::VAR_INTEGER, diff --git a/app/worker.php b/app/worker.php index 2928522bac..8756a1e68b 100644 --- a/app/worker.php +++ b/app/worker.php @@ -9,6 +9,7 @@ use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Insight; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; @@ -16,7 +17,6 @@ use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; -use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -244,6 +244,10 @@ Server::setResource('queueForCertificates', function (Connection $queue) { return new Certificate($queue); }, ['queue']); +Server::setResource('queueForInsights', function (Connection $queue) { + return new Insight($queue); +}, ['queue']); + Server::setResource('queueForMigrations', function (Connection $queue) { return new Migration($queue); }, ['queue']); diff --git a/bin/worker-insights b/bin/worker-insights new file mode 100644 index 0000000000..d7046d8b16 --- /dev/null +++ b/bin/worker-insights @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php insights $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 91f39ff944..2b389a6414 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -484,6 +484,71 @@ services: - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES + appwrite-worker-insights: + entrypoint: worker-insights + <<: *x-logging + container_name: appwrite-worker-insights + image: appwrite-dev + networks: + - appwrite + volumes: + - appwrite-functions:/storage/functions:rw + - appwrite-builds:/storage/builds:rw + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_EXECUTOR_SECRET + - _APP_EXECUTOR_HOST + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_LOGGING_CONFIG + - _APP_VCS_GITHUB_APP_NAME + - _APP_VCS_GITHUB_PRIVATE_KEY + - _APP_VCS_GITHUB_APP_ID + - _APP_FUNCTIONS_TIMEOUT + - _APP_FUNCTIONS_BUILD_TIMEOUT + - _APP_FUNCTIONS_CPUS + - _APP_FUNCTIONS_MEMORY + - _APP_FUNCTIONS_SIZE_LIMIT + - _APP_OPTIONS_FORCE_HTTPS + - _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS + - _APP_DOMAIN + - _APP_STORAGE_DEVICE + - _APP_STORAGE_S3_ACCESS_KEY + - _APP_STORAGE_S3_SECRET + - _APP_STORAGE_S3_REGION + - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_DO_SPACES_ACCESS_KEY + - _APP_STORAGE_DO_SPACES_SECRET + - _APP_STORAGE_DO_SPACES_REGION + - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET + - _APP_DATABASE_SHARED_TABLES + appwrite-worker-certificates: entrypoint: worker-certificates <<: *x-logging @@ -875,6 +940,15 @@ services: environment: - _APP_ASSISTANT_OPENAI_API_KEY + appwrite-browser: + container_name: appwrite-browser + image: appwrite/browser:0.1.0 + networks: + - appwrite + environment: + - APPWRITE_BROWSER_SECRET=123 + - PORT=1234 + openruntimes-executor: container_name: openruntimes-executor hostname: exc1 diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index e87a2fc8f9..d69c374d14 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -48,6 +48,9 @@ class Event public const MIGRATIONS_QUEUE_NAME = 'v1-migrations'; public const MIGRATIONS_CLASS_NAME = 'MigrationsV1'; + public const INSIGHTS_QUEUE_NAME = 'v1-insights'; + public const INSIGHTS_CLASS_NAME = 'InsightsV1'; + protected string $queue = ''; protected string $class = ''; protected string $event = ''; diff --git a/src/Appwrite/Event/Insight.php b/src/Appwrite/Event/Insight.php new file mode 100644 index 0000000000..92320baf43 --- /dev/null +++ b/src/Appwrite/Event/Insight.php @@ -0,0 +1,141 @@ +setQueue(Event::INSIGHTS_QUEUE_NAME) + ->setClass(Event::INSIGHTS_CLASS_NAME); + } + + /** + * Sets build for the ininsights event. + * + * @param Document $build + * @return self + */ + public function setBuild(Document $build): self + { + $this->build = $build; + + return $this; + } + + /** + * Sets resource document for the insights event. + * + * @param Document $resource + * @return self + */ + public function setResource(Document $resource): self + { + $this->resource = $resource; + + return $this; + } + + /** + * Returns set resource document for the insights event. + * + * @return null|Document + */ + public function getResource(): ?Document + { + return $this->resource; + } + + /** + * Sets deployment for the insights event. + * + * @param Document $deployment + * @return self + */ + public function setDeployment(Document $deployment): self + { + $this->deployment = $deployment; + + return $this; + } + + /** + * Returns set deployment for the build event. + * + * @return null|Document + */ + public function getDeployment(): ?Document + { + return $this->deployment; + } + + /** + * Sets type for the build event. + * + * @param string $type Can be `BUILD_TYPE_DEPLOYMENT` or `BUILD_TYPE_RETRY`. + * @return self + */ + public function setType(string $type): self + { + $this->type = $type; + + return $this; + } + + /** + * Returns set type for the function event. + * + * @return string + */ + public function getType(): string + { + return $this->type; + } + + /** + * Executes the function event and sends it to the functions worker. + * + * @return string|bool + * @throws \InvalidArgumentException + */ + public function trigger(): string|bool + { + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ + 'project' => $this->project, + 'resource' => $this->resource, + 'deployment' => $this->deployment, + 'type' => $this->type, + 'build' => $this->build + ]); + } + + /** + * Resets event. + * + * @return self + */ + public function reset(): self + { + $this->type = ''; + $this->resource = null; + $this->deployment = null; + $this->build = null; + parent::reset(); + + return $this; + } +} diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 4baca4f1bd..ec90b1f597 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -5,6 +5,7 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Insight; use Appwrite\Event\Usage; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Utopia\Response\Model\Deployment; @@ -47,6 +48,7 @@ class Builds extends Action ->desc('Builds worker') ->inject('message') ->inject('dbForConsole') + ->inject('queueForInsights') ->inject('queueForEvents') ->inject('queueForFunctions') ->inject('queueForUsage') @@ -60,6 +62,7 @@ class Builds extends Action /** * @param Message $message * @param Database $dbForConsole + * @param Insight $queueForInsights * @param Event $queueForEvents * @param Func $queueForFunctions * @param Usage $queueForUsage @@ -70,7 +73,7 @@ class Builds extends Action * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void + public function action(Message $message, Database $dbForConsole, Insight $queueForInsights, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void { $payload = $message->getPayload() ?? []; @@ -92,7 +95,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template, $log); + $this->buildDeployment($deviceForFunctions, $queueForInsights, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template, $log); break; default: @@ -102,6 +105,7 @@ class Builds extends Action /** * @param Device $deviceForFunctions + * @param Insight $queueForInsights * @param Func $queueForFunctions * @param Event $queueForEvents * @param Usage $queueForUsage @@ -117,7 +121,7 @@ class Builds extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void + protected function buildDeployment(Device $deviceForFunctions, Insight $queueForInsights, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void { $resourceKey = match($resource->getCollection()) { 'functions' => 'functionId', @@ -739,6 +743,15 @@ class Builds extends Action project: $project, queue: $queueForUsage ); + + if ($resource->getCollection() === 'sites' && $build->getAttribute('status') === 'ready') { + $queueForInsights + ->setResource($resource) + ->setBuild($build) + ->setDeployment($deployment) + ->setProject($project) + ->trigger(); + } } } diff --git a/src/Appwrite/Platform/Modules/Sites/Module.php b/src/Appwrite/Platform/Modules/Sites/Module.php index 86a6459a86..e694db18de 100644 --- a/src/Appwrite/Platform/Modules/Sites/Module.php +++ b/src/Appwrite/Platform/Modules/Sites/Module.php @@ -2,13 +2,13 @@ namespace Appwrite\Platform\Modules\Sites; -use Appwrite\Platform\Modules\Sites\Services\Http; use Utopia\Platform; class Module extends Platform\Module { public function __construct() { - $this->addService('http', new Http()); + $this->addService('http', new Services\Http()); + $this->addService('workers', new Services\Workers()); } } diff --git a/src/Appwrite/Platform/Modules/Sites/Services/Workers.php b/src/Appwrite/Platform/Modules/Sites/Services/Workers.php new file mode 100644 index 0000000000..cdaf7b6db9 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Services/Workers.php @@ -0,0 +1,15 @@ +type = Service::TYPE_WORKER; + $this->addAction(Insights::getName(), new Insights()); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Workers/Insights.php b/src/Appwrite/Platform/Modules/Sites/Workers/Insights.php new file mode 100644 index 0000000000..1ad002d9a3 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Workers/Insights.php @@ -0,0 +1,116 @@ +client = new Client(); + $this->client->addHeader('Authorization', 'Bearer 123'); + $this + ->desc('Builds worker') + ->inject('message') + ->inject('dbForConsole') + ->inject('dbForProject') + ->inject('deviceForFunctions') + ->inject('log') + ->callback([$this, 'action']); + } + + /** + * @param Message $message + * @param Database $dbForConsole + * @param Database $dbForProject + * @param Device $deviceForFunctions + * @param Log $log + * @return void + * @throws \Utopia\Database\Exception + */ + public function action(Message $message, Database $dbForConsole, Database $dbForProject, Device $deviceForFunctions, Log $log): void + { + $payload = $message->getPayload() ?? []; + $this->executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); + + if (empty($payload)) { + throw new \Exception('Missing payload'); + } + + //todo: handle preview deployments + $resource = new Document($payload['resource'] ?? []); + $deployment = new Document($payload['deployment'] ?? []); + $rule = $dbForConsole->findOne('rules', [ + Query::equal('resourceType', ['site']), + Query::equal('resourceInternalId', [$resource->getInternalId()]), + ]); + + if (!$rule) { + throw new \Exception('No rule found'); + } + + $url = "http://{$rule->getAttribute('domain')}"; + $screenshot = $this->getScreenshot($url); + // todo: figure out where to store the screenshot + $path = $deviceForFunctions->getRoot() . DIRECTORY_SEPARATOR . "screenshots" .DIRECTORY_SEPARATOR. "{$resource->getId()}.{$deployment->getId()}.png"; + $success = $deviceForFunctions->write($path, $screenshot, "image/png"); + if ($success) { + $deployment->setAttribute('screenshot', $path); + $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } + $audit = $this->getAudit($url); + if ($audit) { + $deployment->setAttribute('auditJson', $audit); + $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } + } + + protected function getScreenshot(string $url): mixed|false + { + $response = $this->client->fetch('http://appwrite-browser/screenshot', query: [ + 'url' => $url + ]); + + if ($response->getStatusCode() !== 200) { + return false; + } + + return $response->getBody(); + } + + protected function getAudit(string $url): string|false + { + $response = $this->client->fetch('http://appwrite-browser/lighthouse', query: [ + 'url' => $url + ]); + + if ($response->getStatusCode() !== 200) { + return false; + } + + return $response->text(); + + } +}