Merge branch '1.4.x' of github.com:appwrite/appwrite into refactor-usage-sn

 Conflicts:

	.gitmodules

	CHANGES.md

	Dockerfile

	composer.lock

	src/Appwrite/Migration/Migration.php

src/Appwrite/Extend/Exception.php

@@ -225,8 +225,13 @@ class Exception extends \Exception
     public const MIGRATION_ALREADY_EXISTS            = 'migration_already_exists';
     public const MIGRATION_IN_PROGRESS               = 'migration_in_progress';
 
-    protected $type = '';
-    protected $errors = [];
+    /** Realtime */
+    public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid';
+    public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages';
+    public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation';
+
+    protected string $type = '';
+    protected array $errors = [];
 
     public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
     {

src/Appwrite/Migration/Migration.php

@@ -73,6 +73,9 @@ abstract class Migration
         '1.4.6' => 'V19',
         '1.4.7' => 'V19',
         '1.4.8' => 'V20',
+        '1.4.8' => 'V19',
+        '1.4.9' => 'V19',
+        '1.4.10' => 'V20',
     ];
 
     /**

src/Appwrite/Platform/Services/Tasks.php

@@ -17,6 +17,7 @@ use Appwrite\Platform\Tasks\Version;
 use Appwrite\Platform\Tasks\VolumeSync;
 use Appwrite\Platform\Tasks\CalcTierStats;
 use Appwrite\Platform\Tasks\Upgrade;
+use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
 
 class Tasks extends Service
 {
@@ -38,6 +39,8 @@ class Tasks extends Service
             ->addAction(VolumeSync::getName(), new VolumeSync())
             ->addAction(Specs::getName(), new Specs())
             ->addAction(CalcTierStats::getName(), new CalcTierStats())
+            ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
+
         ;
     }
 }

src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php

@@ -0,0 +1,115 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use PHPMailer\PHPMailer\PHPMailer;
+use Utopia\App;
+use Utopia\Config\Config;
+use Utopia\Database\Query;
+use Utopia\Database\Validator\Authorization;
+use Utopia\Platform\Action;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Pools\Group;
+use Utopia\Registry\Registry;
+
+class DeleteOrphanedProjects extends Action
+{
+    public static function getName(): string
+    {
+        return 'delete-orphaned-projects';
+    }
+
+    public function __construct()
+    {
+
+        $this
+            ->desc('Get stats for projects')
+            ->inject('pools')
+            ->inject('cache')
+            ->inject('dbForConsole')
+            ->inject('register')
+            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
+                $this->action($pools, $cache, $dbForConsole, $register);
+            });
+    }
+
+
+    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
+    {
+
+        Console::title('Delete orphaned projects V1');
+        Console::success(APP_NAME . ' Delete orphaned projects started');
+
+        /** @var array $collections */
+        $collectionsConfig = Config::getParam('collections', [])['projects'] ?? [];
+
+        /* Initialise new Utopia app */
+        $app = new App('UTC');
+        $console = $app->getResource('console');
+        $projects = [$console];
+
+        /** Database connections */
+        $totalProjects = $dbForConsole->count('projects');
+        Console::success("Found a total of: {$totalProjects} projects");
+
+        $orphans = 0;
+        $count = 0;
+        $limit = 30;
+        $sum = 30;
+        $offset = 0;
+        while (!empty($projects)) {
+            foreach ($projects as $project) {
+
+                /**
+                 * Skip user projects with id 'console'
+                 */
+                if ($project->getId() === 'console') {
+                    continue;
+                }
+
+                try {
+                    $db = $project->getAttribute('database');
+                    $adapter = $pools
+                        ->get($db)
+                        ->pop()
+                        ->getResource();
+
+                    $dbForProject = new Database($adapter, $cache);
+                    $dbForProject->setDefaultDatabase('appwrite');
+                    $dbForProject->setNamespace('_' . $project->getInternalId());
+                    $collectionsCreated = $dbForProject->count(Database::METADATA);
+                    $message = ' (' . $collectionsCreated . ') collections where found on project (' . $project->getId() . '))';
+                    if ($collectionsCreated < (count($collectionsConfig) + 2)) {
+                        Console::error($message);
+                        $orphans++;
+                    } else {
+                        Console::log($message);
+                    }
+                } catch (\Throwable $th) {
+                    //$dbForConsole->deleteDocument('projects', $project->getId());
+                    //Console::success('Deleting  project (' . $project->getId() . ')');
+                    Console::error(' (0) collections where found for project (' . $project->getId() . ')');
+                    $orphans++;
+                } finally {
+                    $pools
+                        ->get($db)
+                        ->reclaim();
+                }
+            }
+
+            $sum = \count($projects);
+
+            $projects = $dbForConsole->find('projects', [
+                Query::limit($limit),
+                Query::offset($offset),
+            ]);
+
+            $offset = $offset + $limit;
+            $count = $count + $sum;
+        }
+
+        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans');
+    }
+}

src/Appwrite/Platform/Tasks/Install.php

@@ -6,9 +6,6 @@ use Appwrite\Auth\Auth;
 use Appwrite\Docker\Compose;
 use Appwrite\Docker\Env;
 use Appwrite\Utopia\View;
-use Utopia\Analytics\Adapter;
-use Utopia\Analytics\Adapter\GoogleAnalytics;
-use Utopia\Analytics\Event;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Utopia\Validator\Text;
@@ -43,12 +40,6 @@ class Install extends Action
         /** @var array<string, array<string, string>> $vars array whre key is variable name and value is variable */
         $vars = [];
 
-        /**
-         * We are using a random value every execution for identification.
-         * This allows us to collect information without invading the privacy of our users.
-         */
-        $analytics = new GoogleAnalytics('UA-26264668-9', uniqid('server.', true));
-
         foreach ($config as $category) {
             foreach ($category['variables'] ?? [] as $var) {
                 $vars[$var['name']] = $var;
@@ -82,7 +73,7 @@ class Install extends Action
             file_put_contents($this->path . '/docker-compose.yml.' . $time . '.backup', $data);
             $compose = new Compose($data);
             $appwrite = $compose->getService('appwrite');
-            $oldVersion = ($appwrite) ? $appwrite->getImageVersion() : null;
+            $oldVersion = $appwrite?->getImageVersion();
             try {
                 $ports = $compose->getService('traefik')->getPorts();
             } catch (\Throwable $th) {
@@ -209,14 +200,12 @@ class Install extends Action
 
         if (!file_put_contents($this->path . '/docker-compose.yml', $templateForCompose->render(false))) {
             $message = 'Failed to save Docker Compose file';
-            $this->sendEvent($analytics, $message);
             Console::error($message);
             Console::exit(1);
         }
 
         if (!file_put_contents($this->path . '/.env', $templateForEnv->render(false))) {
             $message = 'Failed to save environment variables file';
-            $this->sendEvent($analytics, $message);
             Console::error($message);
             Console::exit(1);
         }
@@ -237,29 +226,12 @@ class Install extends Action
 
         if ($exit !== 0) {
             $message = 'Failed to install Appwrite dockers';
-            $this->sendEvent($analytics, $message);
             Console::error($message);
             Console::error($stderr);
             Console::exit($exit);
         } else {
             $message = 'Appwrite installed successfully';
-            $this->sendEvent($analytics, $message);
             Console::success($message);
         }
     }
-
-    private function sendEvent(Adapter $analytics, string $message): void
-    {
-        $event = new Event();
-        $event->setName(APP_VERSION_STABLE);
-        $event->setValue($message);
-        $event->setUrl('http://localhost/');
-        $event->setProps([
-            'category' => 'install/server',
-            'action' => 'install',
-        ]);
-        $event->setType('install/server');
-
-        $analytics->createEvent($event);
-    }
 }

src/Appwrite/Platform/Workers/Certificates.php

@@ -426,29 +426,31 @@ class Certificates extends Action
             $locale->setDefault('en');
         }
 
-        $body = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl');
-
         $subject = \sprintf($locale->getText("emails.certificate.subject"), $domain);
-        $body
-            ->setParam('{{domain}}', $domain)
-            ->setParam('{{error}}', $errorMessage)
-            ->setParam('{{attempt}}', $attempt)
-            ->setParam('{{subject}}', $subject)
-            ->setParam('{{hello}}', $locale->getText("emails.certificate.hello"))
+
+        $message = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-inner-base.tpl');
+        $message
             ->setParam('{{body}}', $locale->getText("emails.certificate.body"))
-            ->setParam('{{redirect}}', 'https://' . $domain)
+            ->setParam('{{hello}}', $locale->getText("emails.certificate.hello"))
             ->setParam('{{footer}}', $locale->getText("emails.certificate.footer"))
             ->setParam('{{thanks}}', $locale->getText("emails.certificate.thanks"))
-            ->setParam('{{signature}}', $locale->getText("emails.certificate.signature"))
-            ->setParam('{{project}}', 'Console')
-            ->setParam('{{direction}}', $locale->getText('settings.direction'))
-            ->setParam('{{bg-body}}', '#f7f7f7')
-            ->setParam('{{bg-content}}', '#ffffff')
-            ->setParam('{{text-content}}', '#000000');
+            ->setParam('{{signature}}', $locale->getText("emails.certificate.signature"));
+        $body = $message->render();
+
+        $emailVariables = [
+            'direction' => $locale->getText('settings.direction'),
+            'domain' => $domain,
+            'error' => '<br><pre>' . $errorMessage . '</pre>',
+            'attempt' => $attempt,
+            'project' => 'Console',
+            'redirect' => 'https://' . $domain,
+        ];
 
         $queueForMails
             ->setRecipient(App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'))
-            ->setBody($body->render())
+            ->setSubject($subject)
+            ->setBody($body)
+            ->setVariables($emailVariables)
             ->setName('Appwrite Administrator')
             ->trigger();
     }

src/Appwrite/Platform/Workers/Deletes.php

@@ -178,7 +178,8 @@ class Deletes extends Action
                 $project = $dbForConsole->getDocument('projects', $document->getAttribute('projectId'));
 
                 if ($project->isEmpty()) {
-                    Console::warning('Unable to delete schedule for function ' . $document->getAttribute('resourceId'));
+                    $dbForConsole->deleteDocument('schedules', $document->getId());
+                    Console::success('Deleted schedule for deleted project ' . $document->getAttribute('projectId'));
                     return;
                 }
 
@@ -921,17 +922,28 @@ class Deletes extends Action
         $count = 0;
         $chunk = 0;
         $limit = 50;
+        $results = [];
         $sum = $limit;
+        $cursor = null;
 
         $executionStart = \microtime(true);
 
         while ($sum === $limit) {
             $chunk++;
 
-            $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
+            $mergedQueries = \array_merge([Query::limit($limit)], $queries);
+            if ($cursor instanceof Document) {
+                $mergedQueries[] = Query::cursorAfter($cursor);
+            }
+
+            $results = $database->find($collection, $mergedQueries);
 
             $sum = count($results);
 
+            if ($sum > 0) {
+                $cursor = $results[$sum - 1];
+            }
+
             foreach ($results as $document) {
                 if (is_callable($callback)) {
                     $callback($document);

src/Appwrite/Platform/Workers/Mails.php

@@ -58,13 +58,21 @@ class Mails extends Action
         $subject = $payload['subject'];
         $variables = $payload['variables'];
         $name = $payload['name'];
-        $body = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl');
+        $body = $payload['body'];
 
+        $bodyTemplate = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl');
+        $bodyTemplate->setParam('{{body}}', $body);
         foreach ($variables as $key => $value) {
-            $body->setParam('{{' . $key . '}}', $value);
+            $bodyTemplate->setParam('{{' . $key . '}}', $value);
         }
+        $body = $bodyTemplate->render();
 
-        $body = $body->render();
+        $subjectTemplate = Template::fromString($subject);
+        foreach ($variables as $key => $value) {
+            $subjectTemplate->setParam('{{' . $key . '}}', $value);
+        }
+        // render() will return the subject in <p> tags, so use strip_tags() to remove them
+        $subject = \strip_tags($subjectTemplate->render());
 
         /** @var PHPMailer $mail */
         $mail = empty($smtp)

src/Appwrite/Utopia/Request.php

@@ -95,4 +95,46 @@ class Request extends UtopiaRequest
     {
         return self::$route != null;
     }
+
+    /**
+     * Get headers
+     *
+     * Method for getting all HTTP header parameters, including cookies.
+     *
+     * @return array<string,mixed>
+     */
+    public function getHeaders(): array
+    {
+        $headers = $this->generateHeaders();
+
+        if (empty($this->swoole->cookie)) {
+            return $headers;
+        }
+
+        $cookieHeaders = [];
+        foreach ($this->swoole->cookie as $key => $value) {
+            $cookieHeaders[] = "{$key}={$value}";
+        }
+
+        if (!empty($cookieHeaders)) {
+            $headers['cookie'] = \implode('; ', $cookieHeaders);
+        }
+
+        return $headers;
+    }
+
+    /**
+     * Get header
+     *
+     * Method for querying HTTP header parameters. If $key is not found $default value will be returned.
+     *
+     * @param  string  $key
+     * @param  string  $default
+     * @return string
+     */
+    public function getHeader(string $key, string $default = ''): string
+    {
+        $headers = $this->getHeaders();
+        return $headers[$key] ?? $default;
+    }
 }