Merge remote-tracking branch 'origin/cl-1.1.x' into db-pools-backups

# Conflicts:
#	composer.lock

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "app/console"]
 	path = app/console
 	url = https://github.com/appwrite/console
-	branch = cloud-cards
+	branch = cloud-1.1.x

app/controllers/api/projects.php

@@ -95,6 +95,7 @@ App::post('/v1/projects')
         $backups['database_db_fra1_03'] = ['from' => '10:30', 'to' => '11:15'];
         $backups['database_db_fra1_04'] = ['from' => '13:30', 'to' => '14:15'];
         $backups['database_db_fra1_05'] = ['from' => '4:30', 'to' => '5:15'];
+        $backups['database_db_fra1_06'] = ['from' => '16:30', 'to' => '17:15'];
 
         $databases = Config::getParam('pools-database', []);
 
@@ -118,7 +119,7 @@ App::post('/v1/projects')
             }
         }
 
-        if ($index = array_search('database_db_fra1_05', $databases)) {
+        if ($index = array_search('database_db_fra1_06', $databases)) {
             $database = $databases[$index];
         } else {
             $database = $databases[array_rand($databases)];

app/controllers/api/teams.php

@@ -255,18 +255,6 @@ App::delete('/v1/teams/:teamId')
             throw new Exception(Exception::TEAM_NOT_FOUND);
         }
 
-        $memberships = $dbForProject->find('memberships', [
-            Query::equal('teamId', [$teamId]),
-            Query::limit(2000), // TODO fix members limit
-        ]);
-
-        // TODO delete all members individually from the user object
-        foreach ($memberships as $membership) {
-            if (!$dbForProject->deleteDocument('memberships', $membership->getId())) {
-                throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove membership for team from DB');
-            }
-        }
-
         if (!$dbForProject->deleteDocument('teams', $teamId)) {
             throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove team from DB');
         }

app/controllers/api/users.php

@@ -414,7 +414,7 @@ App::get('/v1/users/:userId')
         $user = $dbForProject->getDocument('users', $userId);
 
         if ($user->isEmpty()) {
-            throw new Exception('User not found', 404, Exception::USER_NOT_FOUND);
+            throw new Exception(Exception::USER_NOT_FOUND);
         }
 
         $response->dynamic($user, Response::MODEL_USER);

app/controllers/web/console.php

@@ -9,6 +9,7 @@ App::get('/console/*')
     ->alias('auth/*')
     ->alias('/invite')
     ->alias('/login')
+    ->alias('/card/*')
     ->alias('/recover')
     ->alias('/register/*')
     ->groups(['web'])

app/workers/deletes.php

@@ -8,7 +8,6 @@ use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Query;
 use Appwrite\Resque\Worker;
-use Executor\Executor;
 use Utopia\Storage\Device\Local;
 use Utopia\Abuse\Abuse;
 use Utopia\Abuse\Adapters\TimeLimit;
@@ -147,7 +146,8 @@ class DeletesV1 extends Worker
                 $project = $this->getConsoleDB()->getDocument('projects', $document->getAttribute('projectId'));
 
                 if ($project->isEmpty()) {
-                    Console::warning('Unable to delete schedule for function ' . $document->getAttribute('resourceId'));
+                    $this->getConsoleDB()->deleteDocument('schedules', $document->getId());
+                    Console::success('Deleted schedule for deleted project ' . $document->getAttribute('projectId'));
                     return;
                 }
 
@@ -155,7 +155,7 @@ class DeletesV1 extends Worker
 
                 if ($function->isEmpty()) {
                     $this->getConsoleDB()->deleteDocument('schedules', $document->getId());
-                    Console::success('Deleting schedule for function ' . $document->getAttribute('resourceId'));
+                    Console::success('Deleted schedule for function ' . $document->getAttribute('resourceId'));
                 }
             }
         );
@@ -255,6 +255,7 @@ class DeletesV1 extends Worker
     protected function deleteCollection(Document $document, Document $project): void
     {
         $collectionId = $document->getId();
+        $collectionInternalId = $document->getInternalId();
         $databaseId = $document->getAttribute('databaseId');
         $databaseInternalId = $document->getAttribute('databaseInternalId');
 
@@ -263,13 +264,13 @@ class DeletesV1 extends Worker
         $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId());
 
         $this->deleteByGroup('attributes', [
-            Query::equal('databaseId', [$databaseId]),
-            Query::equal('collectionId', [$collectionId])
+            Query::equal('databaseInternalId', [$databaseInternalId]),
+            Query::equal('collectionInternalId', [$collectionInternalId])
         ], $dbForProject);
 
         $this->deleteByGroup('indexes', [
-            Query::equal('databaseId', [$databaseId]),
-            Query::equal('collectionId', [$collectionId])
+            Query::equal('databaseInternalId', [$databaseInternalId]),
+            Query::equal('collectionInternalId', [$collectionInternalId])
         ], $dbForProject);
 
         $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project);
@@ -296,12 +297,21 @@ class DeletesV1 extends Worker
      */
     protected function deleteMemberships(Document $document, Document $project): void
     {
-        $teamId = $document->getAttribute('teamId', '');
+        $dbForProject = $this->getProjectDB($project);
+        $teamInternalId = $document->getInternalId();
 
         // Delete Memberships
-        $this->deleteByGroup('memberships', [
-            Query::equal('teamId', [$teamId])
-        ], $this->getProjectDB($project));
+        $this->deleteByGroup(
+            'memberships',
+            [
+                Query::equal('teamInternalId', [$teamInternalId])
+            ],
+            $dbForProject,
+            function (Document $membership) use ($dbForProject) {
+                $userId = $membership->getAttribute('userId');
+                $dbForProject->deleteCachedDocument('users', $userId);
+            }
+        );
     }
 
     /**
@@ -329,19 +339,20 @@ class DeletesV1 extends Worker
     protected function deleteUser(Document $document, Document $project): void
     {
         $userId = $document->getId();
+        $userInternalId = $document->getInternalId();
 
         $dbForProject = $this->getProjectDB($project);
 
         // Delete all sessions of this user from the sessions table and update the sessions field of the user record
         $this->deleteByGroup('sessions', [
-            Query::equal('userId', [$userId])
+            Query::equal('userInternalId', [$userInternalId])
         ], $dbForProject);
 
         $dbForProject->deleteCachedDocument('users', $userId);
 
         // Delete Memberships and decrement team membership counts
         $this->deleteByGroup('memberships', [
-            Query::equal('userId', [$userId])
+            Query::equal('userInternalId', [$userInternalId])
         ], $dbForProject, function (Document $document) use ($dbForProject) {
             if ($document->getAttribute('confirm')) { // Count only confirmed members
                 $teamId = $document->getAttribute('teamId');
@@ -351,7 +362,7 @@ class DeletesV1 extends Worker
                         'teams',
                         $teamId,
                         // Ensure that total >= 0
-                            $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
+                        $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
                     );
                 }
             }
@@ -359,7 +370,7 @@ class DeletesV1 extends Worker
 
         // Delete tokens
         $this->deleteByGroup('tokens', [
-            Query::equal('userId', [$userId])
+            Query::equal('userInternalId', [$userInternalId])
         ], $dbForProject);
     }
 
@@ -474,13 +485,14 @@ class DeletesV1 extends Worker
         $projectId = $project->getId();
         $dbForProject = $this->getProjectDB($project);
         $functionId = $document->getId();
+        $functionInternalId = $document->getInternalId();
 
         /**
          * Delete Variables
          */
         Console::info("Deleting variables for function " . $functionId);
         $this->deleteByGroup('variables', [
-            Query::equal('functionId', [$functionId])
+            Query::equal('functionInternalId', [$functionInternalId])
         ], $dbForProject);
 
         /**
@@ -678,16 +690,26 @@ class DeletesV1 extends Worker
         $limit = 50;
         $results = [];
         $sum = $limit;
+        $cursor = null;
 
         $executionStart = \microtime(true);
 
         while ($sum === $limit) {
             $chunk++;
 
-            $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
+            $mergedQueries = \array_merge([Query::limit($limit)], $queries);
+            if ($cursor instanceof Document) {
+                $mergedQueries[] = Query::cursorAfter($cursor);
+            }
+
+            $results = $database->find($collection, $mergedQueries);
 
             $sum = count($results);
 
+            if ($sum > 0) {
+                $cursor = $results[$sum - 1];
+            }
+
             foreach ($results as $document) {
                 if (is_callable($callback)) {
                     $callback($document);

bin/upgrade

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php upgrade $@

src/Appwrite/Platform/Services/Tasks.php

@@ -24,6 +24,7 @@ use Appwrite\Platform\Tasks\VolumeSync;
 use Appwrite\Platform\Tasks\CalcUsersStats;
 use Appwrite\Platform\Tasks\CalcTierStats;
 use Appwrite\Platform\Tasks\PatchDeleteProjectCollections;
+use Appwrite\Platform\Tasks\Upgrade;
 
 class Tasks extends Service
 {
@@ -38,6 +39,7 @@ class Tasks extends Service
             ->addAction(Hamster::getName(), new Hamster())
             ->addAction(Doctor::getName(), new Doctor())
             ->addAction(Install::getName(), new Install())
+            ->addAction(Upgrade::getName(), new Upgrade())
             ->addAction(Maintenance::getName(), new Maintenance())
             ->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules())
             ->addAction(ClearCardCache::getName(), new ClearCardCache())

src/Appwrite/Platform/Tasks/Install.php

@@ -6,7 +6,8 @@ use Appwrite\Auth\Auth;
 use Appwrite\Docker\Compose;
 use Appwrite\Docker\Env;
 use Appwrite\Utopia\View;
-use Utopia\Analytics\GoogleAnalytics;
+use Utopia\Analytics\Adapter\GoogleAnalytics;
+use Utopia\Analytics\Event;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Utopia\Validator\Text;
@@ -14,6 +15,8 @@ use Utopia\Platform\Action;
 
 class Install extends Action
 {
+    protected string $path = '/usr/src/code/appwrite';
+
     public static function getName(): string
     {
         return 'install';
@@ -50,7 +53,6 @@ class Install extends Action
          * 6. Run data migration
          */
         $config = Config::getParam('variables');
-        $path = '/usr/src/code/appwrite';
         $defaultHTTPPort = '80';
         $defaultHTTPSPort = '443';
         $vars = [];
@@ -70,19 +72,28 @@ class Install extends Action
         Console::success('Starting Appwrite installation...');
 
         // Create directory with write permissions
-        if (null !== $path && !\file_exists(\dirname($path))) {
-            if (!@\mkdir(\dirname($path), 0755, true)) {
-                Console::error('Can\'t create directory ' . \dirname($path));
+        if (null !== $this->path && !\file_exists(\dirname($this->path))) {
+            if (!@\mkdir(\dirname($this->path), 0755, true)) {
+                Console::error('Can\'t create directory ' . \dirname($this->path));
                 Console::exit(1);
             }
         }
 
-        $data = @file_get_contents($path . '/docker-compose.yml');
+        $data = @file_get_contents($this->path . '/docker-compose.yml');
 
         if ($data !== false) {
+            if ($interactive == 'Y' && Console::isInteractive()) {
+                $answer = Console::confirm('Previous installation found, do you want to overwrite it? (Y/n)');
+
+                if ($answer !== 'Y') {
+                    Console::info('No action taken.');
+                    return;
+                }
+            }
+
             $time = \time();
             Console::info('Compose file found, creating backup: docker-compose.yml.' . $time . '.backup');
-            file_put_contents($path . '/docker-compose.yml.' . $time . '.backup', $data);
+            file_put_contents($this->path . '/docker-compose.yml.' . $time . '.backup', $data);
             $compose = new Compose($data);
             $appwrite = $compose->getService('appwrite');
             $oldVersion = ($appwrite) ? $appwrite->getImageVersion() : null;
@@ -116,11 +127,11 @@ class Install extends Action
                     }
                 }
 
-                $data = @file_get_contents($path . '/.env');
+                $data = @file_get_contents($this->path . '/.env');
 
                 if ($data !== false) { // Fetch all env vars from previous .env file
                     Console::info('Env file found, creating backup: .env.' . $time . '.backup');
-                    file_put_contents($path . '/.env.' . $time . '.backup', $data);
+                    file_put_contents($this->path . '/.env.' . $time . '.backup', $data);
                     $env = new Env($data);
 
                     foreach ($env->list() as $key => $value) {
@@ -198,8 +209,8 @@ class Install extends Action
             }
         }
 
-        $templateForCompose = new View(__DIR__ . '/../views/install/compose.phtml');
-        $templateForEnv = new View(__DIR__ . '/../views/install/env.phtml');
+        $templateForCompose = new View(__DIR__ . '/../../../../app/views/install/compose.phtml');
+        $templateForEnv = new View(__DIR__ . '/../../../../app/views/install/env.phtml');
 
         $templateForCompose
             ->setParam('httpPort', $httpPort)
@@ -213,16 +224,24 @@ class Install extends Action
             ->setParam('vars', $input)
         ;
 
-        if (!file_put_contents($path . '/docker-compose.yml', $templateForCompose->render(false))) {
+        if (!file_put_contents($this->path . '/docker-compose.yml', $templateForCompose->render(false))) {
             $message = 'Failed to save Docker Compose file';
-            $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
+            $event = new Event();
+            $event->setName(APP_VERSION_STABLE . ' - ' . $message)
+                ->addProp('category', 'install/server')
+                ->addProp('action', 'install');
+            $analytics->createEvent($event);
             Console::error($message);
             Console::exit(1);
         }
 
-        if (!file_put_contents($path . '/.env', $templateForEnv->render(false))) {
+        if (!file_put_contents($this->path . '/.env', $templateForEnv->render(false))) {
             $message = 'Failed to save environment variables file';
-            $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
+            $event = new Event();
+            $event->setName(APP_VERSION_STABLE . ' - ' . $message)
+                ->addProp('category', 'install/server')
+                ->addProp('action', 'install');
+            $analytics->createEvent($event);
             Console::error($message);
             Console::exit(1);
         }
@@ -237,19 +256,27 @@ class Install extends Action
             }
         }
 
-        Console::log("Running \"docker compose -f {$path}/docker-compose.yml up -d --remove-orphans --renew-anon-volumes\"");
+        Console::log("Running \"docker compose -f {$this->path}/docker-compose.yml up -d --remove-orphans --renew-anon-volumes\"");
 
-        $exit = Console::execute("${env} docker compose -f {$path}/docker-compose.yml up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr);
+        $exit = Console::execute("${env} docker compose -f {$this->path}/docker-compose.yml up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr);
 
         if ($exit !== 0) {
             $message = 'Failed to install Appwrite dockers';
-            $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
+            $event = new Event();
+            $event->setName(APP_VERSION_STABLE . ' - ' . $message)
+                ->addProp('category', 'install/server')
+                ->addProp('action', 'install');
+            $analytics->createEvent($event);
             Console::error($message);
             Console::error($stderr);
             Console::exit($exit);
         } else {
             $message = 'Appwrite installed successfully';
-            $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
+            $event = new Event();
+            $event->setName(APP_VERSION_STABLE . ' - ' . $message)
+                ->addProp('category', 'install/server')
+                ->addProp('action', 'install');
+            $analytics->createEvent($event);
             Console::success($message);
         }
     }

src/Appwrite/Platform/Tasks/Upgrade.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Utopia\CLI\Console;
+use Utopia\Validator\Text;
+
+class Upgrade extends Install
+{
+    public static function getName(): string
+    {
+        return 'upgrade';
+    }
+
+    public function __construct()
+    {
+        $this
+            ->desc('Upgrade Appwrite')
+            ->param('httpPort', '', new Text(4), 'Server HTTP port', true)
+            ->param('httpsPort', '', new Text(4), 'Server HTTPS port', true)
+            ->param('organization', 'appwrite', new Text(0), 'Docker Registry organization', true)
+            ->param('image', 'appwrite', new Text(0), 'Main appwrite docker image', true)
+            ->param('interactive', 'Y', new Text(1), 'Run an interactive session', true)
+            ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive));
+    }
+
+    public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive): void
+    {
+        // Check for previous installation
+        $data = @file_get_contents($this->path . '/docker-compose.yml');
+        if (empty($data)) {
+            Console::error('Appwrite installation not found.');
+            Console::log('The command was not run in the parent folder of your appwrite installation.');
+            Console::log('Please navigate to the parent directory of the Appwrite installation and try again.');
+            Console::log('  parent_directory <= you run the command in this directory');
+            Console::log('  └── appwrite');
+            Console::log('      └── docker-compose.yml');
+            Console::exit(1);
+        }
+        parent::action($httpPort, $httpsPort, $organization, $image, $interactive);
+    }
+}