Merge branch 'feat-sites' into feat-sites-insights

Dockerfile

@@ -81,6 +81,7 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/worker-databases && \
     chmod +x /usr/local/bin/worker-deletes && \
     chmod +x /usr/local/bin/worker-functions && \
+    chmod +x /usr/local/bin/worker-insights && \
     chmod +x /usr/local/bin/worker-mails && \
     chmod +x /usr/local/bin/worker-messaging && \
     chmod +x /usr/local/bin/worker-migrations && \

app/config/collections.php

@@ -3829,6 +3829,39 @@ $projectCollections = array_merge([
                 'required' => false,
                 'array' => false,
             ],
+            [
+                '$id' => ID::custom('screenshot'),
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 265536,
+                'signed' => false,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
+            [
+                '$id' => ID::custom('auditJson'),
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 16777215, // minimum size for LONGTEXT
+                'signed' => false,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
+            [
+                '$id' => ID::custom('auditHtml'),
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 16777215, // minimum size for MEDIUMTEXT
+                'signed' => false,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
             [
                 '$id' => ID::custom('size'),
                 'type' => Database::VAR_INTEGER,

app/worker.php

@@ -9,6 +9,7 @@ use Appwrite\Event\Database as EventDatabase;
 use Appwrite\Event\Delete;
 use Appwrite\Event\Event;
 use Appwrite\Event\Func;
+use Appwrite\Event\Insight;
 use Appwrite\Event\Mail;
 use Appwrite\Event\Messaging;
 use Appwrite\Event\Migration;
@@ -16,7 +17,6 @@ use Appwrite\Event\Usage;
 use Appwrite\Event\UsageDump;
 use Appwrite\Platform\Appwrite;
 use Swoole\Runtime;
-use Utopia\App;
 use Utopia\Cache\Adapter\Sharding;
 use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
@@ -244,6 +244,10 @@ Server::setResource('queueForCertificates', function (Connection $queue) {
     return new Certificate($queue);
 }, ['queue']);
 
+Server::setResource('queueForInsights', function (Connection $queue) {
+    return new Insight($queue);
+}, ['queue']);
+
 Server::setResource('queueForMigrations', function (Connection $queue) {
     return new Migration($queue);
 }, ['queue']);

bin/worker-insights

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/worker.php insights $@

docker-compose.yml

@@ -484,6 +484,71 @@ services:
       - _APP_DATABASE_SHARED_TABLES
       - _APP_DOMAIN_SITES
 
+  appwrite-worker-insights:
+    entrypoint: worker-insights
+    <<: *x-logging
+    container_name: appwrite-worker-insights
+    image: appwrite-dev
+    networks:
+      - appwrite
+    volumes:
+      - appwrite-functions:/storage/functions:rw
+      - appwrite-builds:/storage/builds:rw
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+    depends_on:
+      - redis
+      - mariadb
+    environment:
+      - _APP_ENV
+      - _APP_WORKER_PER_CORE
+      - _APP_OPENSSL_KEY_V1
+      - _APP_EXECUTOR_SECRET
+      - _APP_EXECUTOR_HOST
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_DB_HOST
+      - _APP_DB_PORT
+      - _APP_DB_SCHEMA
+      - _APP_DB_USER
+      - _APP_DB_PASS
+      - _APP_LOGGING_CONFIG
+      - _APP_VCS_GITHUB_APP_NAME
+      - _APP_VCS_GITHUB_PRIVATE_KEY
+      - _APP_VCS_GITHUB_APP_ID
+      - _APP_FUNCTIONS_TIMEOUT
+      - _APP_FUNCTIONS_BUILD_TIMEOUT
+      - _APP_FUNCTIONS_CPUS
+      - _APP_FUNCTIONS_MEMORY
+      - _APP_FUNCTIONS_SIZE_LIMIT
+      - _APP_OPTIONS_FORCE_HTTPS
+      - _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS
+      - _APP_DOMAIN
+      - _APP_STORAGE_DEVICE
+      - _APP_STORAGE_S3_ACCESS_KEY
+      - _APP_STORAGE_S3_SECRET
+      - _APP_STORAGE_S3_REGION
+      - _APP_STORAGE_S3_BUCKET
+      - _APP_STORAGE_DO_SPACES_ACCESS_KEY
+      - _APP_STORAGE_DO_SPACES_SECRET
+      - _APP_STORAGE_DO_SPACES_REGION
+      - _APP_STORAGE_DO_SPACES_BUCKET
+      - _APP_STORAGE_BACKBLAZE_ACCESS_KEY
+      - _APP_STORAGE_BACKBLAZE_SECRET
+      - _APP_STORAGE_BACKBLAZE_REGION
+      - _APP_STORAGE_BACKBLAZE_BUCKET
+      - _APP_STORAGE_LINODE_ACCESS_KEY
+      - _APP_STORAGE_LINODE_SECRET
+      - _APP_STORAGE_LINODE_REGION
+      - _APP_STORAGE_LINODE_BUCKET
+      - _APP_STORAGE_WASABI_ACCESS_KEY
+      - _APP_STORAGE_WASABI_SECRET
+      - _APP_STORAGE_WASABI_REGION
+      - _APP_STORAGE_WASABI_BUCKET
+      - _APP_DATABASE_SHARED_TABLES
+
   appwrite-worker-certificates:
     entrypoint: worker-certificates
     <<: *x-logging
@@ -875,6 +940,15 @@ services:
     environment:
       - _APP_ASSISTANT_OPENAI_API_KEY
 
+  appwrite-browser:
+    container_name: appwrite-browser
+    image: appwrite/browser:0.1.0
+    networks:
+      - appwrite
+    environment:
+      - APPWRITE_BROWSER_SECRET=123 # TODO: Change this to a random secret
+      - PORT=1234 # TODO: Use a different port
+
   openruntimes-executor:
     container_name: openruntimes-executor
     hostname: exc1

src/Appwrite/Event/Event.php

@@ -48,6 +48,9 @@ class Event
     public const MIGRATIONS_QUEUE_NAME = 'v1-migrations';
     public const MIGRATIONS_CLASS_NAME = 'MigrationsV1';
 
+    public const INSIGHTS_QUEUE_NAME = 'v1-insights';
+    public const INSIGHTS_CLASS_NAME = 'InsightsV1';
+
     protected string $queue = '';
     protected string $class = '';
     protected string $event = '';

src/Appwrite/Event/Insight.php

@@ -0,0 +1,141 @@
+<?php
+
+namespace Appwrite\Event;
+
+use Utopia\Database\Document;
+use Utopia\Queue\Client;
+use Utopia\Queue\Connection;
+
+class Insight extends Event
+{
+    protected string $type = '';
+    protected ?Document $resource = null;
+    protected ?Document $deployment = null;
+    protected ?Document $build = null;
+
+    public function __construct(protected Connection $connection)
+    {
+        parent::__construct($connection);
+
+        $this
+            ->setQueue(Event::INSIGHTS_QUEUE_NAME)
+            ->setClass(Event::INSIGHTS_CLASS_NAME);
+    }
+
+    /**
+     * Sets build for the insights event.
+     *
+     * @param Document $build
+     * @return self
+     */
+    public function setBuild(Document $build): self
+    {
+        $this->build = $build;
+
+        return $this;
+    }
+
+    /**
+     * Sets resource document for the insights event.
+     *
+     * @param Document $resource
+     * @return self
+     */
+    public function setResource(Document $resource): self
+    {
+        $this->resource = $resource;
+
+        return $this;
+    }
+
+    /**
+     * Returns set resource document for the insights event.
+     *
+     * @return null|Document
+     */
+    public function getResource(): ?Document
+    {
+        return $this->resource;
+    }
+
+    /**
+     * Sets deployment for the insights event.
+     *
+     * @param Document $deployment
+     * @return self
+     */
+    public function setDeployment(Document $deployment): self
+    {
+        $this->deployment = $deployment;
+
+        return $this;
+    }
+
+    /**
+     * Returns set deployment for the build event.
+     *
+     * @return null|Document
+     */
+    public function getDeployment(): ?Document
+    {
+        return $this->deployment;
+    }
+
+    /**
+     * Sets type for the build event.
+     *
+     * @param string $type Can be `BUILD_TYPE_DEPLOYMENT` or `BUILD_TYPE_RETRY`.
+     * @return self
+     */
+    public function setType(string $type): self
+    {
+        $this->type = $type;
+
+        return $this;
+    }
+
+    /**
+     * Returns set type for the function event.
+     *
+     * @return string
+     */
+    public function getType(): string
+    {
+        return $this->type;
+    }
+
+    /**
+     * Executes the function event and sends it to the functions worker.
+     *
+     * @return string|bool
+     * @throws \InvalidArgumentException
+     */
+    public function trigger(): string|bool
+    {
+        $client = new Client($this->queue, $this->connection);
+
+        return $client->enqueue([
+            'project' => $this->project,
+            'resource' => $this->resource,
+            'deployment' => $this->deployment,
+            'type' => $this->type,
+            'build' => $this->build
+        ]);
+    }
+
+    /**
+     * Resets event.
+     *
+     * @return self
+     */
+    public function reset(): self
+    {
+        $this->type = '';
+        $this->resource = null;
+        $this->deployment = null;
+        $this->build = null;
+        parent::reset();
+
+        return $this;
+    }
+}

src/Appwrite/Platform/Modules/Functions/Workers/Builds.php

@@ -5,6 +5,7 @@ namespace Appwrite\Platform\Modules\Functions\Workers;
 use Ahc\Jwt\JWT;
 use Appwrite\Event\Event;
 use Appwrite\Event\Func;
+use Appwrite\Event\Insight;
 use Appwrite\Event\Usage;
 use Appwrite\Messaging\Adapter\Realtime;
 use Appwrite\Utopia\Response\Model\Deployment;
@@ -48,6 +49,7 @@ class Builds extends Action
             ->desc('Builds worker')
             ->inject('message')
             ->inject('dbForConsole')
+            ->inject('queueForInsights')
             ->inject('queueForEvents')
             ->inject('queueForFunctions')
             ->inject('queueForUsage')
@@ -61,6 +63,7 @@ class Builds extends Action
     /**
      * @param Message $message
      * @param Database $dbForConsole
+     * @param Insight $queueForInsights
      * @param Event $queueForEvents
      * @param Func $queueForFunctions
      * @param Usage $queueForUsage
@@ -71,7 +74,7 @@ class Builds extends Action
      * @return void
      * @throws \Utopia\Database\Exception
      */
-    public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void
+    public function action(Message $message, Database $dbForConsole, Insight $queueForInsights, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -93,7 +96,7 @@ class Builds extends Action
             case BUILD_TYPE_RETRY:
                 Console::info('Creating build for deployment: ' . $deployment->getId());
                 $github = new GitHub($cache);
-                $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template, $log);
+                $this->buildDeployment($deviceForFunctions, $queueForInsights, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template, $log);
                 break;
 
             default:
@@ -103,6 +106,7 @@ class Builds extends Action
 
     /**
      * @param Device $deviceForFunctions
+     * @param Insight $queueForInsights
      * @param Func $queueForFunctions
      * @param Event $queueForEvents
      * @param Usage $queueForUsage
@@ -119,7 +123,7 @@ class Builds extends Action
      *
      * @throws Exception
      */
-    protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void
+    protected function buildDeployment(Device $deviceForFunctions, Insight $queueForInsights, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void
     {
         $resourceKey = match($resource->getCollection()) {
             'functions' => 'functionId',
@@ -777,6 +781,15 @@ class Builds extends Action
                 project: $project,
                 queue: $queueForUsage
             );
+
+            if ($resource->getCollection() === 'sites' && $build->getAttribute('status') === 'ready') {
+                $queueForInsights
+                    ->setResource($resource)
+                    ->setBuild($build)
+                    ->setDeployment($deployment)
+                    ->setProject($project)
+                    ->trigger();
+            }
         }
     }
 

src/Appwrite/Platform/Modules/Sites/Module.php

@@ -2,13 +2,13 @@
 
 namespace Appwrite\Platform\Modules\Sites;
 
-use Appwrite\Platform\Modules\Sites\Services\Http;
 use Utopia\Platform;
 
 class Module extends Platform\Module
 {
     public function __construct()
     {
-        $this->addService('http', new Http());
+        $this->addService('http', new Services\Http());
+        $this->addService('workers', new Services\Workers());
     }
 }

src/Appwrite/Platform/Modules/Sites/Services/Workers.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace Appwrite\Platform\Modules\Sites\Services;
+
+use Appwrite\Platform\Modules\Sites\Workers\Insights;
+use Utopia\Platform\Service;
+
+class Workers extends Service
+{
+    public function __construct()
+    {
+        $this->type = Service::TYPE_WORKER;
+        $this->addAction(Insights::getName(), new Insights());
+    }
+}

src/Appwrite/Platform/Modules/Sites/Workers/Insights.php

@@ -0,0 +1,116 @@
+<?php
+
+namespace Appwrite\Platform\Modules\Sites\Workers;
+
+use Exception;
+use Executor\Executor;
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+use Utopia\Database\Query;
+use Utopia\Fetch\Client;
+use Utopia\Logger\Log;
+use Utopia\Platform\Action;
+use Utopia\Queue\Message;
+use Utopia\Storage\Device;
+use Utopia\System\System;
+
+class Insights extends Action
+{
+    protected Client $client;
+    protected Executor $executor;
+
+    public static function getName(): string
+    {
+        return 'insights';
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function __construct()
+    {
+        $this->client = new Client();
+        $this->client->addHeader('Authorization', 'Bearer 123');
+        $this
+            ->desc('Builds worker')
+            ->inject('message')
+            ->inject('dbForConsole')
+            ->inject('dbForProject')
+            ->inject('deviceForFunctions')
+            ->inject('log')
+            ->callback([$this, 'action']);
+    }
+
+    /**
+     * @param Message $message
+     * @param Database $dbForConsole
+     * @param Database $dbForProject
+     * @param Device $deviceForFunctions
+     * @param Log $log
+     * @return void
+     * @throws \Utopia\Database\Exception
+     */
+    public function action(Message $message, Database $dbForConsole, Database $dbForProject, Device $deviceForFunctions, Log $log): void
+    {
+        $payload = $message->getPayload() ?? [];
+        $this->executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
+
+        if (empty($payload)) {
+            throw new \Exception('Missing payload');
+        }
+
+        //todo: handle preview deployments
+        $resource = new Document($payload['resource'] ?? []);
+        $deployment = new Document($payload['deployment'] ?? []);
+        $rule = $dbForConsole->findOne('rules', [
+            Query::equal('resourceType', ['site']),
+            Query::equal('resourceInternalId', [$resource->getInternalId()]),
+        ]);
+
+        if (!$rule) {
+            throw new \Exception('No rule found');
+        }
+
+        $url = "http://{$rule->getAttribute('domain')}";
+        $screenshot = $this->getScreenshot($url);
+        // todo: figure out where to store the screenshot
+        $path = $deviceForFunctions->getRoot() . DIRECTORY_SEPARATOR . "screenshots" .DIRECTORY_SEPARATOR. "{$resource->getId()}.{$deployment->getId()}.png";
+        $success = $deviceForFunctions->write($path, $screenshot, "image/png");
+        if ($success) {
+            $deployment->setAttribute('screenshot', $path);
+            $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
+        }
+        $audit = $this->getAudit($url);
+        if ($audit) {
+            $deployment->setAttribute('auditJson', $audit);
+            $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
+        }
+    }
+
+    protected function getScreenshot(string $url): mixed|false
+    {
+        $response = $this->client->fetch('http://appwrite-browser/screenshot', query: [
+            'url' => $url
+        ]);
+
+        if ($response->getStatusCode() !== 200) {
+            return false;
+        }
+
+        return $response->getBody();
+    }
+
+    protected function getAudit(string $url): string|false
+    {
+        $response = $this->client->fetch('http://appwrite-browser/lighthouse', query: [
+            'url' => $url
+        ]);
+
+        if ($response->getStatusCode() !== 200) {
+            return false;
+        }
+
+        return $response->text();
+
+    }
+}