From c6f9f2fc1e22b0a8f076c240a23804c1f0266c75 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Wed, 13 Jan 2021 17:51:02 +0100
Subject: [PATCH 01/27] feat(migration): v06 - first draft

---
 app/tasks/migrate.php                  | 200 +++----------------------
 src/Appwrite/Migration/Migration.php   |  41 +++++
 src/Appwrite/Migration/Version/V04.php |  14 ++
 src/Appwrite/Migration/Version/V05.php | 164 ++++++++++++++++++++
 src/Appwrite/Migration/Version/V06.php |  17 +++
 5 files changed, 256 insertions(+), 180 deletions(-)
 create mode 100644 src/Appwrite/Migration/Migration.php
 create mode 100644 src/Appwrite/Migration/Version/V04.php
 create mode 100644 src/Appwrite/Migration/Version/V05.php
 create mode 100644 src/Appwrite/Migration/Version/V06.php

diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php
index bba8ad416c..c25763ddef 100644
--- a/app/tasks/migrate.php
+++ b/app/tasks/migrate.php
@@ -5,187 +5,26 @@ global $cli, $register, $projectDB, $console;
 use Utopia\Config\Config;
 use Utopia\CLI\Console;
 use Appwrite\Database\Database;
-use Appwrite\Database\Document;
 use Appwrite\Database\Validator\Authorization;
 use Appwrite\Database\Adapter\MySQL as MySQLAdapter;
 use Appwrite\Database\Adapter\Redis as RedisAdapter;
-
-$callbacks = [
-    '0.4.0' => function() {
-        Console::log('I got nothing to do.');
-    },
-
-    '0.5.0' => function($project) use ($register, $projectDB) {
-        $db = $register->get('db');
-
-        Console::log('Migrating project: '.$project->getAttribute('name').' ('.$project->getId().')');
-
-        // Update all documents $uid -> $id
-
-        $limit = 30;
-        $sum = 30;
-        $offset = 0;
-
-        while ($sum >= 30) {
-            $all = $projectDB->getCollection([
-                'limit' => $limit,
-                'offset' => $offset,
-                'orderType' => 'DESC',
-            ]);
-
-            $sum = \count($all);
-            
-            Console::log('Migrating: '.$offset.' / '.$projectDB->getSum());
-            
-            foreach($all as $document) {
-                $document = fixDocument($document);
-
-                if(empty($document->getId())) {
-                    throw new Exception('Missing ID');
-                }
-
-                try {
-                    $new = $projectDB->overwriteDocument($document->getArrayCopy());
-                } catch (\Throwable $th) {
-                    var_dump($document);
-                    Console::error('Failed to update document: '.$th->getMessage());
-                    continue;
-                }
-
-                if($new->getId() !== $document->getId()) {
-                    throw new Exception('Duplication Error');
-                }
-            }
-
-            $offset = $offset + $limit;
-        }
-
-        $schema = $_SERVER['_APP_DB_SCHEMA'] ?? '';
-
-        try {
-            $statement = $db->prepare("
-
-                CREATE TABLE IF NOT EXISTS `template.database.unique` (
-                    `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
-                    `key` varchar(128) DEFAULT NULL,
-                    PRIMARY KEY (`id`),
-                    UNIQUE KEY `index1` (`key`)
-                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-
-                CREATE TABLE IF NOT EXISTS `{$schema}`.`app_{$project->getId()}.database.unique` LIKE `template.database.unique`;
-                ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` DROP COLUMN IF EXISTS `userType`;
-                ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` DROP INDEX IF EXISTS `index_1`;
-                ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` ADD INDEX IF NOT EXISTS `index_1` (`userId` ASC);
-            ");
-
-            $statement->closeCursor();
-
-            $statement->execute();
-        }
-        catch (\Exception $e) {
-            Console::error('Failed to alter table for project: '.$project->getId().' with message: '.$e->getMessage().'/');
-        }
-    },
-];
-
-function fixDocument(Document $document) {
-    $providers = Config::getParam('providers');
-
-    if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_PROJECTS){
-        foreach($providers as $key => $provider) {
-            if(!empty($document->getAttribute('usersOauth'.\ucfirst($key).'Appid'))) {
-                $document
-                    ->setAttribute('usersOauth2'.\ucfirst($key).'Appid', $document->getAttribute('usersOauth'.\ucfirst($key).'Appid', ''))
-                    ->removeAttribute('usersOauth'.\ucfirst($key).'Appid')
-                ;
-            }
-
-            if(!empty($document->getAttribute('usersOauth'.\ucfirst($key).'Secret'))) {
-                $document
-                    ->setAttribute('usersOauth2'.\ucfirst($key).'Secret', $document->getAttribute('usersOauth'.\ucfirst($key).'Secret', ''))
-                    ->removeAttribute('usersOauth'.\ucfirst($key).'Secret')
-                ;
-            }
-        }
-    }
-
-    if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_WEBHOOKS){
-        $document->setAttribute('security', ($document->getAttribute('security')) ? true : false);
-    }
-
-    if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_TASKS){
-        $document->setAttribute('security', ($document->getAttribute('security')) ? true : false);
-    }
-
-    if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_USERS) {
-        foreach($providers as $key => $provider) {
-            if(!empty($document->getAttribute('oauth'.\ucfirst($key)))) {
-                $document
-                    ->setAttribute('oauth2'.\ucfirst($key), $document->getAttribute('oauth'.\ucfirst($key), ''))
-                    ->removeAttribute('oauth'.\ucfirst($key))
-                ;
-            }
-
-            if(!empty($document->getAttribute('oauth'.\ucfirst($key).'AccessToken'))) {
-                $document
-                    ->setAttribute('oauth2'.\ucfirst($key).'AccessToken', $document->getAttribute('oauth'.\ucfirst($key).'AccessToken', ''))
-                    ->removeAttribute('oauth'.\ucfirst($key).'AccessToken')
-                ;
-            }
-        }
-    
-        if($document->getAttribute('confirm', null) !== null) {
-            $document
-                ->setAttribute('emailVerification', $document->getAttribute('confirm', $document->getAttribute('emailVerification', false)))
-                ->removeAttribute('confirm')
-            ;
-        }
-    }
-
-    if($document->getAttribute('$collection') === Database::SYSTEM_COLLECTION_PLATFORMS) {
-        if($document->getAttribute('url', null) !== null) {
-            $document
-                ->setAttribute('hostname', \parse_url($document->getAttribute('url', $document->getAttribute('hostname', '')), PHP_URL_HOST))
-                ->removeAttribute('url')
-            ;
-        }
-    }
-
-    $document
-        ->setAttribute('$id', $document->getAttribute('$uid', $document->getAttribute('$id')))
-        ->removeAttribute('$uid')
-    ;
-
-    foreach($document as &$attr) { // Handle child documents
-        if($attr instanceof Document) {
-            $attr = fixDocument($attr);
-        }
-
-        if(\is_array($attr)) {
-            foreach($attr as &$child) {
-                if($child instanceof Document) {
-                    $child = fixDocument($child);
-                }
-            }
-        }
-    }
-
-    return $document;
-}
+use Appwrite\Migration\Version;
 
 $cli
     ->task('migrate')
-    ->action(function () use ($register, $callbacks) {
+    ->action(function () use ($register) {
         Console::success('Starting Data Migration');
 
         $consoleDB = new Database();
-        $consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register));
-        $consoleDB->setNamespace('app_console'); // Main DB
-        $consoleDB->setMocks(Config::getParam('collections', []));
-        
+        $consoleDB
+            ->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register))
+            ->setNamespace('app_console') // Main DB
+            ->setMocks(Config::getParam('collections', []));
+
         $projectDB = new Database();
-        $projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register));
-        $projectDB->setMocks(Config::getParam('collections', []));
+        $projectDB
+            ->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register))
+            ->setMocks(Config::getParam('collections', []));
 
         $console = $consoleDB->getDocument('console');
 
@@ -197,16 +36,17 @@ $cli
         $projects = [$console];
         $count = 0;
 
-        while ($sum >= 30) {
-            foreach($projects as $project) {
-                
-                $projectDB->setNamespace('app_'.$project->getId());
+        $migration = new Version\V06($register->get('db')); //TODO: remove hardcoded version and move to dynamic migration
 
+        while ($sum >= 30) {
+            foreach ($projects as $project) {
                 try {
-                    $callbacks['0.5.0']($project, $projectDB);
+                    $migration
+                        ->setProject($project, $projectDB)
+                        ->execute();
                 } catch (\Throwable $th) {
                     throw $th;
-                    Console::error('Failed to update project ("'.$project->getId().'") version with error: '.$th->getMessage());
+                    Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
                 }
             }
 
@@ -214,7 +54,7 @@ $cli
                 'limit' => $limit,
                 'offset' => $offset,
                 'filters' => [
-                    '$collection='.Database::SYSTEM_COLLECTION_PROJECTS,
+                    '$collection=' . Database::SYSTEM_COLLECTION_PROJECTS,
                 ],
             ]);
 
@@ -222,8 +62,8 @@ $cli
             $offset = $offset + $limit;
             $count = $count + $sum;
 
-            Console::log('Fetched '.$count.'/'.$consoleDB->getSum().' projects...');
+            Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...');
         }
 
         Console::success('Data Migration Completed');
-    });
\ No newline at end of file
+    });
diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php
new file mode 100644
index 0000000000..f2d6a8c0a5
--- /dev/null
+++ b/src/Appwrite/Migration/Migration.php
@@ -0,0 +1,41 @@
+<?php
+
+namespace Appwrite\Migration;
+
+use Appwrite\Database\Document;
+use Appwrite\Database\Database;
+
+abstract class Migration
+{
+  protected \PDO $db;
+
+  protected int $limit = 30;
+  protected int $sum = 30;
+  protected int $offset = 0;
+  protected Document $project;
+  protected Database $projectDB;
+
+  /**
+   * Migration constructor.
+   */
+  public function __construct(\PDO $db)
+  {
+    $this->db = $db;
+  }
+
+  /**
+   * Set project for migration.
+   */
+  public function setProject(Document $project, Database $projectDB)
+  {
+    $this->project = $project;
+    $this->projectDB = $projectDB;
+    $this->projectDB->setNamespace('app_'.$project->getId());
+    return $this;
+  }
+
+  /**
+   * Executes migration for set project.
+   */
+  abstract public function execute(): void;
+}
diff --git a/src/Appwrite/Migration/Version/V04.php b/src/Appwrite/Migration/Version/V04.php
new file mode 100644
index 0000000000..f1f879d898
--- /dev/null
+++ b/src/Appwrite/Migration/Version/V04.php
@@ -0,0 +1,14 @@
+<?php
+
+namespace Appwrite\Migration\Version;
+
+use Utopia\CLI\Console;
+use Appwrite\Migration\Migration;
+
+class V04 extends Migration
+{
+  public function execute(): void
+  {
+    Console::log('I got nothing to do.');
+  }
+}
diff --git a/src/Appwrite/Migration/Version/V05.php b/src/Appwrite/Migration/Version/V05.php
new file mode 100644
index 0000000000..e8e0d7b4bc
--- /dev/null
+++ b/src/Appwrite/Migration/Version/V05.php
@@ -0,0 +1,164 @@
+<?php
+
+namespace Appwrite\Migration\Version;
+
+use Appwrite\Migration\Migration;
+use Utopia\Config\Config;
+use Utopia\CLI\Console;
+use Utopia\Exception;
+use Appwrite\Database\Database;
+use Appwrite\Database\Document;
+
+class V05 extends Migration
+{
+  public function execute(): void
+  {
+    $db = $this->db;
+    $project = $this->project;
+    $projectDB = $this->projectDB;
+    Console::log('Migrating project: ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
+
+    // Update all documents $uid -> $id
+
+    $limit = 30;
+    $sum = 30;
+    $offset = 0;
+
+    while ($sum >= 30) {
+      $all = $projectDB->getCollection([
+        'limit' => $limit,
+        'offset' => $offset,
+        'orderType' => 'DESC',
+      ]);
+
+      $sum = \count($all);
+
+      Console::log('Migrating: ' . $offset . ' / ' . $projectDB->getSum());
+
+      foreach ($all as $document) {
+        $document = $this->fixDocument($document);
+
+        if (empty($document->getId())) {
+          throw new Exception('Missing ID');
+        }
+
+        try {
+          $new = $projectDB->overwriteDocument($document->getArrayCopy());
+        } catch (\Throwable $th) {
+          var_dump($document);
+          Console::error('Failed to update document: ' . $th->getMessage());
+          continue;
+        }
+
+        if ($new->getId() !== $document->getId()) {
+          throw new Exception('Duplication Error');
+        }
+      }
+
+      $offset = $offset + $limit;
+    }
+
+    $schema = $_SERVER['_APP_DB_SCHEMA'] ?? '';
+
+    try {
+      $statement = $db->prepare("
+
+            CREATE TABLE IF NOT EXISTS `template.database.unique` (
+                `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+                `key` varchar(128) DEFAULT NULL,
+                PRIMARY KEY (`id`),
+                UNIQUE KEY `index1` (`key`)
+            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+            CREATE TABLE IF NOT EXISTS `{$schema}`.`app_{$project->getId()}.database.unique` LIKE `template.database.unique`;
+            ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` DROP COLUMN IF EXISTS `userType`;
+            ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` DROP INDEX IF EXISTS `index_1`;
+            ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` ADD INDEX IF NOT EXISTS `index_1` (`userId` ASC);
+        ");
+
+      $statement->closeCursor();
+
+      $statement->execute();
+    } catch (\Exception $e) {
+      Console::error('Failed to alter table for project: ' . $project->getId() . ' with message: ' . $e->getMessage() . '/');
+    }
+  }
+
+  private function fixDocument(Document $document)
+  {
+    $providers = Config::getParam('providers');
+
+    switch ($document->getAttribute('$collection')) {
+      case Database::SYSTEM_COLLECTION_PROJECTS:
+        foreach ($providers as $key => $provider) {
+          if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid'))) {
+            $document
+              ->setAttribute('usersOauth2' . \ucfirst($key) . 'Appid', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid', ''))
+              ->removeAttribute('usersOauth' . \ucfirst($key) . 'Appid');
+          }
+
+          if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret'))) {
+            $document
+              ->setAttribute('usersOauth2' . \ucfirst($key) . 'Secret', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', ''))
+              ->removeAttribute('usersOauth' . \ucfirst($key) . 'Secret');
+          }
+        }
+        break;
+
+      case Database::SYSTEM_COLLECTION_PROJECTS:
+      case Database::SYSTEM_COLLECTION_TASKS:
+        $document->setAttribute('security', ($document->getAttribute('security')) ? true : false);
+        break;
+
+      case Database::SYSTEM_COLLECTION_USERS:
+        foreach ($providers as $key => $provider) {
+          if (!empty($document->getAttribute('oauth' . \ucfirst($key)))) {
+            $document
+              ->setAttribute('oauth2' . \ucfirst($key), $document->getAttribute('oauth' . \ucfirst($key), ''))
+              ->removeAttribute('oauth' . \ucfirst($key));
+          }
+
+          if (!empty($document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken'))) {
+            $document
+              ->setAttribute('oauth2' . \ucfirst($key) . 'AccessToken', $document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken', ''))
+              ->removeAttribute('oauth' . \ucfirst($key) . 'AccessToken');
+          }
+        }
+
+        if ($document->getAttribute('confirm', null) !== null) {
+          $document
+            ->setAttribute('emailVerification', $document->getAttribute('confirm', $document->getAttribute('emailVerification', false)))
+            ->removeAttribute('confirm');
+        }
+        break;
+
+      case Database::SYSTEM_COLLECTION_PLATFORMS:
+        if ($document->getAttribute('url', null) !== null) {
+          $document
+            ->setAttribute('hostname', \parse_url($document->getAttribute('url', $document->getAttribute('hostname', '')), PHP_URL_HOST))
+            ->removeAttribute('url');
+        }
+        break;
+    }
+
+    $document
+      ->setAttribute('$id', $document->getAttribute('$uid', $document->getAttribute('$id')))
+      ->removeAttribute('$uid');
+
+    foreach ($document as &$attr) { // Handle child documents
+      if ($attr instanceof Document) {
+        $attr = $this->fixDocument($attr);
+      }
+
+      if (\is_array($attr)) {
+        foreach ($attr as &$child) {
+          if ($child instanceof Document) {
+            $child = $this->fixDocument($child);
+          }
+        }
+      }
+    }
+
+    return $document;
+  }
+}
diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php
new file mode 100644
index 0000000000..33ddafe814
--- /dev/null
+++ b/src/Appwrite/Migration/Version/V06.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace Appwrite\Migration\Version;
+
+use Utopia\CLI\Console;
+use Appwrite\Migration\Migration;
+
+class V06 extends Migration
+{
+  public function execute(): void
+  {
+    Console::log('I got nothing to do. Yet.');
+
+    //TODO: migrate new `filter` property
+
+  }
+}

From 8413d3daafcfdd1c6b4a1a6ed1893a85c58fa3c1 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Wed, 13 Jan 2021 17:58:01 +0100
Subject: [PATCH 02/27] fix(migration): add return type

---
 src/Appwrite/Migration/Migration.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php
index f2d6a8c0a5..8d8100ee74 100644
--- a/src/Appwrite/Migration/Migration.php
+++ b/src/Appwrite/Migration/Migration.php
@@ -26,7 +26,7 @@ abstract class Migration
   /**
    * Set project for migration.
    */
-  public function setProject(Document $project, Database $projectDB)
+  public function setProject(Document $project, Database $projectDB): Migration
   {
     $this->project = $project;
     $this->projectDB = $projectDB;

From c51caac494f52b1c30452e0555ab5f0b364cd9e8 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Thu, 14 Jan 2021 13:04:16 +0100
Subject: [PATCH 03/27] refactor(collections): enfore camel case

---
 app/config/collections.php      | 2 +-
 app/controllers/api/account.php | 6 +++---
 app/controllers/api/teams.php   | 2 +-
 app/controllers/api/users.php   | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/config/collections.php b/app/config/collections.php
index a9451d4a5b..89236c6844 100644
--- a/app/config/collections.php
+++ b/app/config/collections.php
@@ -228,7 +228,7 @@ $collections = [
             [
                 '$collection' => Database::SYSTEM_COLLECTION_RULES,
                 'label' => 'Password Update Date',
-                'key' => 'password-update',
+                'key' => 'passwordUpdate',
                 'type' => Database::SYSTEM_VAR_TYPE_NUMERIC,
                 'default' => '',
                 'required' => true,
diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php
index c28dd19e33..7e0b53866d 100644
--- a/app/controllers/api/account.php
+++ b/app/controllers/api/account.php
@@ -99,7 +99,7 @@ App::post('/v1/account')
                 'emailVerification' => false,
                 'status' => Auth::USER_STATUS_UNACTIVATED,
                 'password' => Auth::passwordHash($password),
-                'password-update' => \time(),
+                'passwordUpdate' => \time(),
                 'registration' => \time(),
                 'reset' => false,
                 'name' => $name,
@@ -511,7 +511,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
                         'emailVerification' => true,
                         'status' => Auth::USER_STATUS_ACTIVATED, // Email should already be authenticated by OAuth2 provider
                         'password' => Auth::passwordHash(Auth::passwordGenerator()),
-                        'password-update' => \time(),
+                        'passwordUpdate' => \time(),
                         'registration' => \time(),
                         'reset' => false,
                         'name' => $name,
@@ -1412,7 +1412,7 @@ App::put('/v1/account/recovery')
 
         $profile = $projectDB->updateDocument(\array_merge($profile->getArrayCopy(), [
             'password' => Auth::passwordHash($password),
-            'password-update' => \time(),
+            'passwordUpdate' => \time(),
             'emailVerification' => true,
         ]));
 
diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php
index 084bdeeb1b..5ebeb63669 100644
--- a/app/controllers/api/teams.php
+++ b/app/controllers/api/teams.php
@@ -324,7 +324,7 @@ App::post('/v1/teams/:teamId/memberships')
                     'emailVerification' => false,
                     'status' => Auth::USER_STATUS_UNACTIVATED,
                     'password' => Auth::passwordHash(Auth::passwordGenerator()),
-                    'password-update' => \time(),
+                    'passwordUpdate' => \time(),
                     'registration' => \time(),
                     'reset' => false,
                     'name' => $name,
diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php
index 76e20aad98..a0cff97232 100644
--- a/app/controllers/api/users.php
+++ b/app/controllers/api/users.php
@@ -62,7 +62,7 @@ App::post('/v1/users')
                 'emailVerification' => false,
                 'status' => Auth::USER_STATUS_UNACTIVATED,
                 'password' => Auth::passwordHash($password),
-                'password-update' => \time(),
+                'passwordUpdate' => \time(),
                 'registration' => \time(),
                 'reset' => false,
                 'name' => $name,

From 328405a774ca60779ebda793041c1560f3c15ae1 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Thu, 14 Jan 2021 13:05:49 +0100
Subject: [PATCH 04/27] fix(migration): move iteration to parent class

---
 src/Appwrite/Migration/Migration.php   |  99 ++++++++----
 src/Appwrite/Migration/Version/V04.php |   8 +-
 src/Appwrite/Migration/Version/V05.php | 210 ++++++++++---------------
 src/Appwrite/Migration/Version/V06.php |  26 ++-
 4 files changed, 183 insertions(+), 160 deletions(-)

diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php
index 8d8100ee74..0e7f95abd2 100644
--- a/src/Appwrite/Migration/Migration.php
+++ b/src/Appwrite/Migration/Migration.php
@@ -4,38 +4,83 @@ namespace Appwrite\Migration;
 
 use Appwrite\Database\Document;
 use Appwrite\Database\Database;
+use Utopia\CLI\Console;
+use Utopia\Exception;
 
 abstract class Migration
 {
-  protected \PDO $db;
+    protected \PDO $db;
 
-  protected int $limit = 30;
-  protected int $sum = 30;
-  protected int $offset = 0;
-  protected Document $project;
-  protected Database $projectDB;
+    protected int $limit = 30;
+    protected int $sum = 30;
+    protected int $offset = 0;
+    protected Document $project;
+    protected Database $projectDB;
 
-  /**
-   * Migration constructor.
-   */
-  public function __construct(\PDO $db)
-  {
-    $this->db = $db;
-  }
+    /**
+     * Migration constructor.
+     */
+    public function __construct(\PDO $db)
+    {
+        $this->db = $db;
+    }
 
-  /**
-   * Set project for migration.
-   */
-  public function setProject(Document $project, Database $projectDB): Migration
-  {
-    $this->project = $project;
-    $this->projectDB = $projectDB;
-    $this->projectDB->setNamespace('app_'.$project->getId());
-    return $this;
-  }
+    /**
+     * Set project for migration.
+     */
+    public function setProject(Document $project, Database $projectDB): Migration
+    {
+        $this->project = $project;
+        $this->projectDB = $projectDB;
+        $this->projectDB->setNamespace('app_' . $project->getId());
+        return $this;
+    }
 
-  /**
-   * Executes migration for set project.
-   */
-  abstract public function execute(): void;
+    /**
+     * Iterates through every document.
+     * 
+     * @param function(Document): Document $callback
+     */
+    public function forEachDocument(callable $callback)
+    {
+        while ($this->sum >= 30) {
+            $all = $this->projectDB->getCollection([
+                'limit' => $this->limit,
+                'offset' => $this->offset,
+                'orderType' => 'DESC',
+            ]);
+
+            $this->sum = \count($all);
+
+            Console::log('Migrating: ' . $this->offset . ' / ' . $this->projectDB->getSum());
+
+            foreach ($all as $document) {
+
+                $document = call_user_func($callback, $document);
+
+                if (empty($document->getId())) {
+                    throw new Exception('Missing ID');
+                }
+
+                try {
+                    $new = $this->projectDB->overwriteDocument($document->getArrayCopy());
+                } catch (\Throwable $th) {
+                    var_dump($document);
+                    Console::error('Failed to update document: ' . $th->getMessage());
+                    continue;
+                }
+
+                if ($new->getId() !== $document->getId()) {
+                    throw new Exception('Duplication Error');
+                }
+            }
+
+            $this->offset = $this->offset + $this->limit;
+        }
+    }
+
+    /**
+     * Executes migration for set project.
+     */
+    abstract public function execute(): void;
 }
diff --git a/src/Appwrite/Migration/Version/V04.php b/src/Appwrite/Migration/Version/V04.php
index f1f879d898..3b13d8920b 100644
--- a/src/Appwrite/Migration/Version/V04.php
+++ b/src/Appwrite/Migration/Version/V04.php
@@ -7,8 +7,8 @@ use Appwrite\Migration\Migration;
 
 class V04 extends Migration
 {
-  public function execute(): void
-  {
-    Console::log('I got nothing to do.');
-  }
+    public function execute(): void
+    {
+        Console::log('I got nothing to do.');
+    }
 }
diff --git a/src/Appwrite/Migration/Version/V05.php b/src/Appwrite/Migration/Version/V05.php
index e8e0d7b4bc..c79c059dbf 100644
--- a/src/Appwrite/Migration/Version/V05.php
+++ b/src/Appwrite/Migration/Version/V05.php
@@ -5,63 +5,25 @@ namespace Appwrite\Migration\Version;
 use Appwrite\Migration\Migration;
 use Utopia\Config\Config;
 use Utopia\CLI\Console;
-use Utopia\Exception;
 use Appwrite\Database\Database;
 use Appwrite\Database\Document;
 
 class V05 extends Migration
 {
-  public function execute(): void
-  {
-    $db = $this->db;
-    $project = $this->project;
-    $projectDB = $this->projectDB;
-    Console::log('Migrating project: ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
+    public function execute(): void
+    {
+        $db = $this->db;
+        $project = $this->project;
+        Console::log('Migrating project: ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
 
-    // Update all documents $uid -> $id
+        // Update all documents $uid -> $id
 
-    $limit = 30;
-    $sum = 30;
-    $offset = 0;
+        $this->forEachDocument([$this, 'fixDocument']);
 
-    while ($sum >= 30) {
-      $all = $projectDB->getCollection([
-        'limit' => $limit,
-        'offset' => $offset,
-        'orderType' => 'DESC',
-      ]);
-
-      $sum = \count($all);
-
-      Console::log('Migrating: ' . $offset . ' / ' . $projectDB->getSum());
-
-      foreach ($all as $document) {
-        $document = $this->fixDocument($document);
-
-        if (empty($document->getId())) {
-          throw new Exception('Missing ID');
-        }
+        $schema = $_SERVER['_APP_DB_SCHEMA'] ?? '';
 
         try {
-          $new = $projectDB->overwriteDocument($document->getArrayCopy());
-        } catch (\Throwable $th) {
-          var_dump($document);
-          Console::error('Failed to update document: ' . $th->getMessage());
-          continue;
-        }
-
-        if ($new->getId() !== $document->getId()) {
-          throw new Exception('Duplication Error');
-        }
-      }
-
-      $offset = $offset + $limit;
-    }
-
-    $schema = $_SERVER['_APP_DB_SCHEMA'] ?? '';
-
-    try {
-      $statement = $db->prepare("
+            $statement = $db->prepare("
 
             CREATE TABLE IF NOT EXISTS `template.database.unique` (
                 `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
@@ -76,89 +38,89 @@ class V05 extends Migration
             ALTER TABLE `{$schema}`.`app_{$project->getId()}.audit.audit` ADD INDEX IF NOT EXISTS `index_1` (`userId` ASC);
         ");
 
-      $statement->closeCursor();
+            $statement->closeCursor();
 
-      $statement->execute();
-    } catch (\Exception $e) {
-      Console::error('Failed to alter table for project: ' . $project->getId() . ' with message: ' . $e->getMessage() . '/');
-    }
-  }
-
-  private function fixDocument(Document $document)
-  {
-    $providers = Config::getParam('providers');
-
-    switch ($document->getAttribute('$collection')) {
-      case Database::SYSTEM_COLLECTION_PROJECTS:
-        foreach ($providers as $key => $provider) {
-          if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid'))) {
-            $document
-              ->setAttribute('usersOauth2' . \ucfirst($key) . 'Appid', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid', ''))
-              ->removeAttribute('usersOauth' . \ucfirst($key) . 'Appid');
-          }
-
-          if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret'))) {
-            $document
-              ->setAttribute('usersOauth2' . \ucfirst($key) . 'Secret', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', ''))
-              ->removeAttribute('usersOauth' . \ucfirst($key) . 'Secret');
-          }
+            $statement->execute();
+        } catch (\Exception $e) {
+            Console::error('Failed to alter table for project: ' . $project->getId() . ' with message: ' . $e->getMessage() . '/');
         }
-        break;
-
-      case Database::SYSTEM_COLLECTION_PROJECTS:
-      case Database::SYSTEM_COLLECTION_TASKS:
-        $document->setAttribute('security', ($document->getAttribute('security')) ? true : false);
-        break;
-
-      case Database::SYSTEM_COLLECTION_USERS:
-        foreach ($providers as $key => $provider) {
-          if (!empty($document->getAttribute('oauth' . \ucfirst($key)))) {
-            $document
-              ->setAttribute('oauth2' . \ucfirst($key), $document->getAttribute('oauth' . \ucfirst($key), ''))
-              ->removeAttribute('oauth' . \ucfirst($key));
-          }
-
-          if (!empty($document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken'))) {
-            $document
-              ->setAttribute('oauth2' . \ucfirst($key) . 'AccessToken', $document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken', ''))
-              ->removeAttribute('oauth' . \ucfirst($key) . 'AccessToken');
-          }
-        }
-
-        if ($document->getAttribute('confirm', null) !== null) {
-          $document
-            ->setAttribute('emailVerification', $document->getAttribute('confirm', $document->getAttribute('emailVerification', false)))
-            ->removeAttribute('confirm');
-        }
-        break;
-
-      case Database::SYSTEM_COLLECTION_PLATFORMS:
-        if ($document->getAttribute('url', null) !== null) {
-          $document
-            ->setAttribute('hostname', \parse_url($document->getAttribute('url', $document->getAttribute('hostname', '')), PHP_URL_HOST))
-            ->removeAttribute('url');
-        }
-        break;
     }
 
-    $document
-      ->setAttribute('$id', $document->getAttribute('$uid', $document->getAttribute('$id')))
-      ->removeAttribute('$uid');
+    protected function fixDocument(Document $document)
+    {
+        $providers = Config::getParam('providers');
 
-    foreach ($document as &$attr) { // Handle child documents
-      if ($attr instanceof Document) {
-        $attr = $this->fixDocument($attr);
-      }
+        switch ($document->getAttribute('$collection')) {
+            case Database::SYSTEM_COLLECTION_PROJECTS:
+                foreach ($providers as $key => $provider) {
+                    if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid'))) {
+                        $document
+                            ->setAttribute('usersOauth2' . \ucfirst($key) . 'Appid', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Appid', ''))
+                            ->removeAttribute('usersOauth' . \ucfirst($key) . 'Appid');
+                    }
 
-      if (\is_array($attr)) {
-        foreach ($attr as &$child) {
-          if ($child instanceof Document) {
-            $child = $this->fixDocument($child);
-          }
+                    if (!empty($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret'))) {
+                        $document
+                            ->setAttribute('usersOauth2' . \ucfirst($key) . 'Secret', $document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', ''))
+                            ->removeAttribute('usersOauth' . \ucfirst($key) . 'Secret');
+                    }
+                }
+                break;
+
+            case Database::SYSTEM_COLLECTION_PROJECTS:
+            case Database::SYSTEM_COLLECTION_TASKS:
+                $document->setAttribute('security', ($document->getAttribute('security')) ? true : false);
+                break;
+
+            case Database::SYSTEM_COLLECTION_USERS:
+                foreach ($providers as $key => $provider) {
+                    if (!empty($document->getAttribute('oauth' . \ucfirst($key)))) {
+                        $document
+                            ->setAttribute('oauth2' . \ucfirst($key), $document->getAttribute('oauth' . \ucfirst($key), ''))
+                            ->removeAttribute('oauth' . \ucfirst($key));
+                    }
+
+                    if (!empty($document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken'))) {
+                        $document
+                            ->setAttribute('oauth2' . \ucfirst($key) . 'AccessToken', $document->getAttribute('oauth' . \ucfirst($key) . 'AccessToken', ''))
+                            ->removeAttribute('oauth' . \ucfirst($key) . 'AccessToken');
+                    }
+                }
+
+                if ($document->getAttribute('confirm', null) !== null) {
+                    $document
+                        ->setAttribute('emailVerification', $document->getAttribute('confirm', $document->getAttribute('emailVerification', false)))
+                        ->removeAttribute('confirm');
+                }
+                break;
+
+            case Database::SYSTEM_COLLECTION_PLATFORMS:
+                if ($document->getAttribute('url', null) !== null) {
+                    $document
+                        ->setAttribute('hostname', \parse_url($document->getAttribute('url', $document->getAttribute('hostname', '')), PHP_URL_HOST))
+                        ->removeAttribute('url');
+                }
+                break;
         }
-      }
-    }
 
-    return $document;
-  }
+        $document
+            ->setAttribute('$id', $document->getAttribute('$uid', $document->getAttribute('$id')))
+            ->removeAttribute('$uid');
+
+        foreach ($document as &$attr) { // Handle child documents
+            if ($attr instanceof Document) {
+                $attr = $this->fixDocument($attr);
+            }
+
+            if (\is_array($attr)) {
+                foreach ($attr as &$child) {
+                    if ($child instanceof Document) {
+                        $child = $this->fixDocument($child);
+                    }
+                }
+            }
+        }
+
+        return $document;
+    }
 }
diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php
index 33ddafe814..d28d3b652b 100644
--- a/src/Appwrite/Migration/Version/V06.php
+++ b/src/Appwrite/Migration/Version/V06.php
@@ -3,15 +3,31 @@
 namespace Appwrite\Migration\Version;
 
 use Utopia\CLI\Console;
+use Appwrite\Database\Database;
+use Appwrite\Database\Document;
 use Appwrite\Migration\Migration;
 
 class V06 extends Migration
 {
-  public function execute(): void
-  {
-    Console::log('I got nothing to do. Yet.');
+    public function execute(): void
+    {
+        Console::log('I got nothing to do. Yet.');
 
-    //TODO: migrate new `filter` property
+        //TODO: migrate new `filter` property
+        $this->forEachDocument([$this, 'fixDocument']);
+    }
 
-  }
+    protected function fixDocument(Document $document)
+    {
+        switch ($document->getAttribute('$collection')) {
+            case Database::SYSTEM_COLLECTION_USERS:
+                if ($document->getAttribute('password-update', null)) {
+                    $document
+                        ->setAttribute('passwordUpdate', $document->getAttribute('password-update', $document->getAttribute('passwordUpdate', '')))
+                        ->removeAttribute('password-update');
+                }
+                break;
+        }
+        return $document;
+    }
 }

From bc71a1868bd2a0e93828bb3edb2801dbe42df2f7 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Thu, 14 Jan 2021 13:50:26 +0100
Subject: [PATCH 05/27] fix(migration): phpdoc callable

---
 src/Appwrite/Migration/Migration.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php
index 0e7f95abd2..51ec1a6e02 100644
--- a/src/Appwrite/Migration/Migration.php
+++ b/src/Appwrite/Migration/Migration.php
@@ -39,7 +39,7 @@ abstract class Migration
     /**
      * Iterates through every document.
      * 
-     * @param function(Document): Document $callback
+     * @param callable(Document):Document $callback
      */
     public function forEachDocument(callable $callback)
     {

From d41ee76d3621908d7480b47a748bd5b8fa6953dc Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Thu, 14 Jan 2021 14:12:55 +0100
Subject: [PATCH 06/27] fix(migration): prepare filter migration

---
 src/Appwrite/Migration/Version/V06.php | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php
index d28d3b652b..b9a253701c 100644
--- a/src/Appwrite/Migration/Version/V06.php
+++ b/src/Appwrite/Migration/Version/V06.php
@@ -3,6 +3,7 @@
 namespace Appwrite\Migration\Version;
 
 use Utopia\CLI\Console;
+use Utopia\Config\Config;
 use Appwrite\Database\Database;
 use Appwrite\Database\Document;
 use Appwrite\Migration\Migration;
@@ -13,12 +14,13 @@ class V06 extends Migration
     {
         Console::log('I got nothing to do. Yet.');
 
-        //TODO: migrate new `filter` property
         $this->forEachDocument([$this, 'fixDocument']);
     }
 
     protected function fixDocument(Document $document)
     {
+        $providers = Config::getParam('providers');
+
         switch ($document->getAttribute('$collection')) {
             case Database::SYSTEM_COLLECTION_USERS:
                 if ($document->getAttribute('password-update', null)) {
@@ -26,6 +28,26 @@ class V06 extends Migration
                         ->setAttribute('passwordUpdate', $document->getAttribute('password-update', $document->getAttribute('passwordUpdate', '')))
                         ->removeAttribute('password-update');
                 }
+                if($document->getAttribute('prefs', null)) {
+                    //TODO: take care of filter ['json']
+                }
+                break;
+            case Database::SYSTEM_COLLECTION_WEBHOOKS:
+                if($document->getAttribute('httpPass', null)) {
+                    //TODO: take care of filter ['encrypt']
+                }
+                break;
+            case Database::SYSTEM_COLLECTION_TASKS:
+                if($document->getAttribute('httpPass', null)) {
+                    //TODO: take care of filter ['encrypt']
+                }
+                break;
+            case Database::SYSTEM_COLLECTION_PROJECTS:
+                foreach ($providers as $key => $provider) {
+                    if ($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret')) {
+                        //TODO: take care of filter ['encrypt]
+                    }
+                }
                 break;
         }
         return $document;

From 28d4552ee214bf584415debd4daf783161096cf6 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Thu, 14 Jan 2021 14:16:17 +0100
Subject: [PATCH 07/27] fix(migration): missing null value in provider

---
 src/Appwrite/Migration/Version/V06.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php
index b9a253701c..46798b315f 100644
--- a/src/Appwrite/Migration/Version/V06.php
+++ b/src/Appwrite/Migration/Version/V06.php
@@ -44,7 +44,7 @@ class V06 extends Migration
                 break;
             case Database::SYSTEM_COLLECTION_PROJECTS:
                 foreach ($providers as $key => $provider) {
-                    if ($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret')) {
+                    if ($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', null)) {
                         //TODO: take care of filter ['encrypt]
                     }
                 }

From a6a7f896c1cd5076e82cfd8b5062ee3bd4fd8b49 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Thu, 14 Jan 2021 15:03:34 +0100
Subject: [PATCH 08/27] fix(migration): major project iteration bug

---
 app/tasks/migrate.php                  |  8 +++++---
 src/Appwrite/Migration/Version/V06.php | 13 +++++++------
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php
index c25763ddef..7b9b61cc06 100644
--- a/app/tasks/migrate.php
+++ b/app/tasks/migrate.php
@@ -38,7 +38,7 @@ $cli
 
         $migration = new Version\V06($register->get('db')); //TODO: remove hardcoded version and move to dynamic migration
 
-        while ($sum >= 30) {
+        while ($sum > 0) {
             foreach ($projects as $project) {
                 try {
                     $migration
@@ -61,8 +61,10 @@ $cli
             $sum = \count($projects);
             $offset = $offset + $limit;
             $count = $count + $sum;
-
-            Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...');
+            
+            if ($sum > 0) {
+                Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...');
+            }
         }
 
         Console::success('Data Migration Completed');
diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php
index 46798b315f..f533135446 100644
--- a/src/Appwrite/Migration/Version/V06.php
+++ b/src/Appwrite/Migration/Version/V06.php
@@ -12,15 +12,14 @@ class V06 extends Migration
 {
     public function execute(): void
     {
-        Console::log('I got nothing to do. Yet.');
+        $project = $this->project;
+        Console::log('Migrating project: ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
 
         $this->forEachDocument([$this, 'fixDocument']);
     }
 
     protected function fixDocument(Document $document)
     {
-        $providers = Config::getParam('providers');
-
         switch ($document->getAttribute('$collection')) {
             case Database::SYSTEM_COLLECTION_USERS:
                 if ($document->getAttribute('password-update', null)) {
@@ -28,21 +27,23 @@ class V06 extends Migration
                         ->setAttribute('passwordUpdate', $document->getAttribute('password-update', $document->getAttribute('passwordUpdate', '')))
                         ->removeAttribute('password-update');
                 }
-                if($document->getAttribute('prefs', null)) {
+                if ($document->getAttribute('prefs', null)) {
                     //TODO: take care of filter ['json']
                 }
                 break;
             case Database::SYSTEM_COLLECTION_WEBHOOKS:
-                if($document->getAttribute('httpPass', null)) {
+                if ($document->getAttribute('httpPass', null)) {
                     //TODO: take care of filter ['encrypt']
                 }
                 break;
             case Database::SYSTEM_COLLECTION_TASKS:
-                if($document->getAttribute('httpPass', null)) {
+                if ($document->getAttribute('httpPass', null)) {
                     //TODO: take care of filter ['encrypt']
                 }
                 break;
             case Database::SYSTEM_COLLECTION_PROJECTS:
+                $providers = Config::getParam('providers');
+
                 foreach ($providers as $key => $provider) {
                     if ($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', null)) {
                         //TODO: take care of filter ['encrypt]

From d7276aa502a9c5b73bc56b1d7160946e2f7857ba Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Thu, 14 Jan 2021 15:09:48 +0100
Subject: [PATCH 09/27] fix(migration): change loop dependent integers scope

---
 src/Appwrite/Migration/Migration.php | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php
index 51ec1a6e02..b0e6f4f991 100644
--- a/src/Appwrite/Migration/Migration.php
+++ b/src/Appwrite/Migration/Migration.php
@@ -12,8 +12,7 @@ abstract class Migration
     protected \PDO $db;
 
     protected int $limit = 30;
-    protected int $sum = 30;
-    protected int $offset = 0;
+
     protected Document $project;
     protected Database $projectDB;
 
@@ -43,16 +42,19 @@ abstract class Migration
      */
     public function forEachDocument(callable $callback)
     {
-        while ($this->sum >= 30) {
+        $sum = 30;
+        $offset = 0;
+
+        while ($sum >= 30) {
             $all = $this->projectDB->getCollection([
                 'limit' => $this->limit,
-                'offset' => $this->offset,
+                'offset' => $offset,
                 'orderType' => 'DESC',
             ]);
 
-            $this->sum = \count($all);
+            $sum = \count($all);
 
-            Console::log('Migrating: ' . $this->offset . ' / ' . $this->projectDB->getSum());
+            Console::log('Migrating: ' . $offset . ' / ' . $this->projectDB->getSum());
 
             foreach ($all as $document) {
 
@@ -75,7 +77,7 @@ abstract class Migration
                 }
             }
 
-            $this->offset = $this->offset + $this->limit;
+            $offset += $this->limit;
         }
     }
 

From 3c5d32e5fc8c8a2ae28508c487679c3112a3aa28 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Thu, 14 Jan 2021 15:47:13 +0100
Subject: [PATCH 10/27] fix(migration): migrate encrypt filter

---
 src/Appwrite/Migration/Version/V06.php | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php
index f533135446..715508f787 100644
--- a/src/Appwrite/Migration/Version/V06.php
+++ b/src/Appwrite/Migration/Version/V06.php
@@ -2,11 +2,13 @@
 
 namespace Appwrite\Migration\Version;
 
+use Utopia\App;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Appwrite\Database\Database;
 use Appwrite\Database\Document;
 use Appwrite\Migration\Migration;
+use Appwrite\OpenSSL\OpenSSL;
 
 class V06 extends Migration
 {
@@ -22,6 +24,8 @@ class V06 extends Migration
     {
         switch ($document->getAttribute('$collection')) {
             case Database::SYSTEM_COLLECTION_USERS:
+                var_dump($this->applyFilterEncrypt($document->getAttribute("email")));
+
                 if ($document->getAttribute('password-update', null)) {
                     $document
                         ->setAttribute('passwordUpdate', $document->getAttribute('password-update', $document->getAttribute('passwordUpdate', '')))
@@ -53,4 +57,19 @@ class V06 extends Migration
         }
         return $document;
     }
+
+    private function applyFilterEncrypt(String $value)
+    {
+        $key = App::getEnv('_APP_OPENSSL_KEY_V1');
+        $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
+        $tag = null;
+
+        return json_encode([
+            'data' => OpenSSL::encrypt($value, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag),
+            'method' => OpenSSL::CIPHER_AES_128_GCM,
+            'iv' => bin2hex($iv),
+            'tag' => bin2hex($tag),
+            'version' => '1',
+        ]);
+    }
 }

From 970eac94ad54cab6fa38e520107578b31ab4fe30 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Thu, 14 Jan 2021 16:05:27 +0100
Subject: [PATCH 11/27] fix(migration): migrate encrypt filter

---
 src/Appwrite/Migration/Version/V06.php | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php
index 715508f787..d73887b379 100644
--- a/src/Appwrite/Migration/Version/V06.php
+++ b/src/Appwrite/Migration/Version/V06.php
@@ -24,8 +24,6 @@ class V06 extends Migration
     {
         switch ($document->getAttribute('$collection')) {
             case Database::SYSTEM_COLLECTION_USERS:
-                var_dump($this->applyFilterEncrypt($document->getAttribute("email")));
-
                 if ($document->getAttribute('password-update', null)) {
                     $document
                         ->setAttribute('passwordUpdate', $document->getAttribute('password-update', $document->getAttribute('passwordUpdate', '')))
@@ -36,13 +34,9 @@ class V06 extends Migration
                 }
                 break;
             case Database::SYSTEM_COLLECTION_WEBHOOKS:
-                if ($document->getAttribute('httpPass', null)) {
-                    //TODO: take care of filter ['encrypt']
-                }
-                break;
             case Database::SYSTEM_COLLECTION_TASKS:
                 if ($document->getAttribute('httpPass', null)) {
-                    //TODO: take care of filter ['encrypt']
+                    $document->setAttribute('httpPass', $this->applyFilterEncrypt($document->getAttribute('httpPass')));
                 }
                 break;
             case Database::SYSTEM_COLLECTION_PROJECTS:
@@ -50,7 +44,7 @@ class V06 extends Migration
 
                 foreach ($providers as $key => $provider) {
                     if ($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', null)) {
-                        //TODO: take care of filter ['encrypt]
+                        $document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', $this->applyFilterEncrypt($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret')));
                     }
                 }
                 break;

From 229c649f9a078cc10c9ef41be23003d33bd52185 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Thu, 14 Jan 2021 16:24:01 +0100
Subject: [PATCH 12/27] revert(migration) encryption

---
 src/Appwrite/Migration/Version/V06.php | 36 --------------------------
 1 file changed, 36 deletions(-)

diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php
index d73887b379..253800fc94 100644
--- a/src/Appwrite/Migration/Version/V06.php
+++ b/src/Appwrite/Migration/Version/V06.php
@@ -2,13 +2,10 @@
 
 namespace Appwrite\Migration\Version;
 
-use Utopia\App;
 use Utopia\CLI\Console;
-use Utopia\Config\Config;
 use Appwrite\Database\Database;
 use Appwrite\Database\Document;
 use Appwrite\Migration\Migration;
-use Appwrite\OpenSSL\OpenSSL;
 
 class V06 extends Migration
 {
@@ -29,41 +26,8 @@ class V06 extends Migration
                         ->setAttribute('passwordUpdate', $document->getAttribute('password-update', $document->getAttribute('passwordUpdate', '')))
                         ->removeAttribute('password-update');
                 }
-                if ($document->getAttribute('prefs', null)) {
-                    //TODO: take care of filter ['json']
-                }
-                break;
-            case Database::SYSTEM_COLLECTION_WEBHOOKS:
-            case Database::SYSTEM_COLLECTION_TASKS:
-                if ($document->getAttribute('httpPass', null)) {
-                    $document->setAttribute('httpPass', $this->applyFilterEncrypt($document->getAttribute('httpPass')));
-                }
-                break;
-            case Database::SYSTEM_COLLECTION_PROJECTS:
-                $providers = Config::getParam('providers');
-
-                foreach ($providers as $key => $provider) {
-                    if ($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', null)) {
-                        $document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret', $this->applyFilterEncrypt($document->getAttribute('usersOauth' . \ucfirst($key) . 'Secret')));
-                    }
-                }
                 break;
         }
         return $document;
     }
-
-    private function applyFilterEncrypt(String $value)
-    {
-        $key = App::getEnv('_APP_OPENSSL_KEY_V1');
-        $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
-        $tag = null;
-
-        return json_encode([
-            'data' => OpenSSL::encrypt($value, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag),
-            'method' => OpenSSL::CIPHER_AES_128_GCM,
-            'iv' => bin2hex($iv),
-            'tag' => bin2hex($tag),
-            'version' => '1',
-        ]);
-    }
 }

From 538510d1717cdbcb7d76fe97d0b5fbd34d138140 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Fri, 15 Jan 2021 12:56:03 +0100
Subject: [PATCH 13/27] feat(collections): encrypt api keys

---
 app/config/collections.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/config/collections.php b/app/config/collections.php
index 89236c6844..9170b07573 100644
--- a/app/config/collections.php
+++ b/app/config/collections.php
@@ -827,6 +827,7 @@ $collections = [
                 'type' => Database::SYSTEM_VAR_TYPE_TEXT,
                 'default' => '',
                 'required' => false,
+                'filter' => ['encrypt'],
             ],
         ],
     ],

From bda5e13ac0a079f9c668fe1e710039d1f349d5f1 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Fri, 15 Jan 2021 12:56:39 +0100
Subject: [PATCH 14/27] feat(Database): introduce enabling/disabling filters

---
 src/Appwrite/Database/Database.php | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/src/Appwrite/Database/Database.php b/src/Appwrite/Database/Database.php
index 8610aeac16..c4e1370e1b 100644
--- a/src/Appwrite/Database/Database.php
+++ b/src/Appwrite/Database/Database.php
@@ -57,6 +57,11 @@ class Database
      */
     static protected $filters = [];
 
+    /**
+     * @var bool
+     */
+    static protected $statusFilters = true;
+
     /**
      * @var array
      */
@@ -448,6 +453,26 @@ class Database
         ];
     }
 
+    /**
+     * Disable Attribute decoding
+     *
+     * @return void
+     */
+    public static function disableFilters(): void
+    {
+        self::$statusFilters = false;
+    }
+
+    /**
+     * Enable Attribute decoding
+     *
+     * @return void
+     */
+    public static function enableFilters(): void
+    {
+        self::$statusFilters = true;
+    }
+
     public function encode(Document $document):Document
     {
         $collection = $this->getDocument($document->getCollection(), true , false);
@@ -550,6 +575,10 @@ class Database
      */
     static protected function decodeAttribute(string $name, $value)
     {
+        if (!self::$statusFilters) {
+            return $value;
+        }
+        
         if (!isset(self::$filters[$name])) {
             return $value;
             throw new Exception('Filter not found');

From bfccaa13629dcb6d960437734e489548d307301b Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Fri, 15 Jan 2021 12:57:32 +0100
Subject: [PATCH 15/27] feat(migration): add api key migration

---
 src/Appwrite/Migration/Version/V06.php | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php
index 253800fc94..34620f93c7 100644
--- a/src/Appwrite/Migration/Version/V06.php
+++ b/src/Appwrite/Migration/Version/V06.php
@@ -2,10 +2,13 @@
 
 namespace Appwrite\Migration\Version;
 
+
+use Utopia\App;
 use Utopia\CLI\Console;
 use Appwrite\Database\Database;
 use Appwrite\Database\Document;
 use Appwrite\Migration\Migration;
+use Appwrite\OpenSSL\OpenSSL;
 
 class V06 extends Migration
 {
@@ -14,7 +17,9 @@ class V06 extends Migration
         $project = $this->project;
         Console::log('Migrating project: ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
 
+        $this->projectDB->disableFilters();
         $this->forEachDocument([$this, 'fixDocument']);
+        $this->projectDB->enableFilters();
     }
 
     protected function fixDocument(Document $document)
@@ -27,6 +32,21 @@ class V06 extends Migration
                         ->removeAttribute('password-update');
                 }
                 break;
+            case Database::SYSTEM_COLLECTION_KEYS:
+                if ($document->getAttribute('secret', null)) {
+                    $key = App::getEnv('_APP_OPENSSL_KEY_V1');
+                    $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
+                    $tag = null;
+
+                    $document->setAttribute('secret', json_encode([
+                        'data' => OpenSSL::encrypt($document->getAttribute('secret'), OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag),
+                        'method' => OpenSSL::CIPHER_AES_128_GCM,
+                        'iv' => bin2hex($iv),
+                        'tag' => bin2hex($tag),
+                        'version' => '1',
+                    ]));
+                }
+                break;
         }
         return $document;
     }

From 8c1371db4d0db2d6dc907ee76d3ed7af89a91ced Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Fri, 15 Jan 2021 16:35:51 +0100
Subject: [PATCH 16/27] chore(changelog): add migration script changes

---
 CHANGES.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CHANGES.md b/CHANGES.md
index 1418fb4532..f682c7dd2f 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -39,6 +39,7 @@
 - New OAuth adapter for PayPal sandbox  (@armino-dev - [#420](https://github.com/appwrite/appwrite/issues/410))
 - Introducing new permssion types: role:guest, role:member, role:app
 - Disabled rate-limits on server side integrations
+- Refactored migration script 
 
 ### User Interface
 
@@ -105,6 +106,7 @@
 - Fixed OAuth redirect when using the self-hosted instance default success URL ([#454](https://github.com/appwrite/appwrite/issues/454))
 - Fixed bug denying authentication with Github OAuth provider
 - Fixed a bug making read permission overwrite write permission in some cases
+- Fixed consistent property names in databases by enforcing camel case
 
 ## Security
 
@@ -113,6 +115,7 @@
 - Now using your `_APP_SYSTEM_EMAIL_ADDRESS` as the email address for issuing and renewing SSL certificates
 - Block iframe access to Appwrite console using the `X-Frame-Options` header.
 - Fixed `roles` param input validator
+- API Keys are now stored encrypted 
 
 # Version 0.6.2 (PRE-RELEASE)
 

From 5da6b31edb7db94f32ee7244c9c929acd5b13991 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Fri, 15 Jan 2021 16:36:42 +0100
Subject: [PATCH 17/27] style(migration): remove space on concatenation

---
 app/tasks/migrate.php | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php
index 7b9b61cc06..f9e92b24b7 100644
--- a/app/tasks/migrate.php
+++ b/app/tasks/migrate.php
@@ -46,7 +46,7 @@ $cli
                         ->execute();
                 } catch (\Throwable $th) {
                     throw $th;
-                    Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
+                    Console::error('Failed to update project ("'.$project->getId().'") version with error: '.$th->getMessage());
                 }
             }
 
@@ -54,7 +54,7 @@ $cli
                 'limit' => $limit,
                 'offset' => $offset,
                 'filters' => [
-                    '$collection=' . Database::SYSTEM_COLLECTION_PROJECTS,
+                    '$collection='.Database::SYSTEM_COLLECTION_PROJECTS,
                 ],
             ]);
 
@@ -63,7 +63,7 @@ $cli
             $count = $count + $sum;
             
             if ($sum > 0) {
-                Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...');
+                Console::log('Fetched '.$count.'/'.$consoleDB->getSum().' projects...');
             }
         }
 

From 100e1f37e9ba4fec2858b1c17b3c75650bde139f Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Mon, 18 Jan 2021 13:27:04 +0100
Subject: [PATCH 18/27] fix(migration): security property

---
 src/Appwrite/Migration/Version/V05.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Appwrite/Migration/Version/V05.php b/src/Appwrite/Migration/Version/V05.php
index c79c059dbf..bcabe6e0a1 100644
--- a/src/Appwrite/Migration/Version/V05.php
+++ b/src/Appwrite/Migration/Version/V05.php
@@ -65,11 +65,11 @@ class V05 extends Migration
                             ->removeAttribute('usersOauth' . \ucfirst($key) . 'Secret');
                     }
                 }
+                $document->setAttribute('security', $document->getAttribute('security') ? true : false);
                 break;
 
-            case Database::SYSTEM_COLLECTION_PROJECTS:
             case Database::SYSTEM_COLLECTION_TASKS:
-                $document->setAttribute('security', ($document->getAttribute('security')) ? true : false);
+                $document->setAttribute('security', $document->getAttribute('security') ? true : false);
                 break;
 
             case Database::SYSTEM_COLLECTION_USERS:

From 43220e95c50c5cd1ea36bb3f8d78e62ed01d700d Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Mon, 18 Jan 2021 13:27:28 +0100
Subject: [PATCH 19/27] test(migration): adds unit tests

---
 tests/unit/Migration/V05Test.php | 102 +++++++++++++++++++++++++++++++
 tests/unit/Migration/V06Test.php |  56 +++++++++++++++++
 2 files changed, 158 insertions(+)
 create mode 100644 tests/unit/Migration/V05Test.php
 create mode 100644 tests/unit/Migration/V06Test.php

diff --git a/tests/unit/Migration/V05Test.php b/tests/unit/Migration/V05Test.php
new file mode 100644
index 0000000000..5206443c43
--- /dev/null
+++ b/tests/unit/Migration/V05Test.php
@@ -0,0 +1,102 @@
+<?php
+
+namespace Appwrite\Tests;
+
+use ReflectionClass;
+use Appwrite\Migration\Version\V05;
+use Appwrite\Database\Database;
+use Appwrite\Database\Document;
+use PHPUnit\Framework\TestCase;
+use Utopia\Config\Config;
+
+class V05Test extends TestCase
+{
+    /**
+     * @var PDO
+     */
+    protected \PDO $pdo;
+
+    public function setUp(): void
+    {
+        $this->pdo = new \PDO('sqlite::memory:');
+    }
+
+    public function testMigration()
+    {
+        Config::load('providers', __DIR__ . '/../../../app/config/providers.php');
+
+        $v05 = new V05($this->pdo);
+
+        $reflector = new ReflectionClass('Appwrite\Migration\Version\V05');
+        $method = $reflector->getMethod('fixDocument');
+        $method->setAccessible(true);
+
+        $document =  $method->invokeArgs($v05, [
+            new Document([
+                '$uid' => 'unique',
+                '$collection' => Database::SYSTEM_COLLECTION_PROJECTS,
+                'usersOauthGithubAppid' => 123,
+                'usersOauthGithubSecret' => 456
+            ])
+        ]);
+
+        $this->assertEquals($document->getAttribute('$uid', null), null);
+        $this->assertEquals($document->getAttribute('$id', null), 'unique');
+
+        $this->assertEquals($document->getAttribute('usersOauthGithubAppid', null), null);
+        $this->assertEquals($document->getAttribute('usersOauth2GithubAppid', null), 123);
+
+        $this->assertEquals($document->getAttribute('usersOauthGithubSecret', null), null);
+        $this->assertEquals($document->getAttribute('usersOauth2GithubSecret', null), 456);
+
+        $this->assertEquals($document->getAttribute('security', true), false);
+
+        $document =  $method->invokeArgs($v05, [
+            new Document([
+                '$uid' => 'unique',
+                '$collection' => Database::SYSTEM_COLLECTION_TASKS
+            ])
+        ]);
+
+        $this->assertEquals($document->getAttribute('$uid', null), null);
+        $this->assertEquals($document->getAttribute('$id', null), 'unique');
+
+        $this->assertEquals($document->getAttribute('security', true), false);
+
+        $document =  $method->invokeArgs($v05, [
+            new Document([
+                '$uid' => 'unique',
+                '$collection' => Database::SYSTEM_COLLECTION_USERS,
+                'oauthGithub' => 'id',
+                'oauthGithubAccessToken' => 'token',
+                'confirm' => false
+            ])
+        ]);
+
+        $this->assertEquals($document->getAttribute('$uid', null), null);
+        $this->assertEquals($document->getAttribute('$id', null), 'unique');
+
+        $this->assertEquals($document->getAttribute('confirm', null), null);
+        $this->assertEquals($document->getAttribute('emailVerification', true), false);
+
+        $this->assertEquals($document->getAttribute('oauthGithub', null), null);
+        $this->assertEquals($document->getAttribute('oauth2Github', null), 'id');
+
+        $this->assertEquals($document->getAttribute('oauthGithubAccessToken', null), null);
+        $this->assertEquals($document->getAttribute('oauth2GithubAccessToken', null), 'token');
+
+        $document =  $method->invokeArgs($v05, [
+            new Document([
+                '$uid' => 'unique',
+                '$collection' => Database::SYSTEM_COLLECTION_PLATFORMS,
+                'url' => 'https://appwrite.io'
+            ])
+        ]);
+
+        $this->assertEquals($document->getAttribute('$uid', null), null);
+        $this->assertEquals($document->getAttribute('$id', null), 'unique');
+
+        $this->assertEquals($document->getAttribute('url', null), null);
+        $this->assertEquals($document->getAttribute('hostname', null), 'appwrite.io');
+    }
+}
diff --git a/tests/unit/Migration/V06Test.php b/tests/unit/Migration/V06Test.php
new file mode 100644
index 0000000000..9056e87fc0
--- /dev/null
+++ b/tests/unit/Migration/V06Test.php
@@ -0,0 +1,56 @@
+<?php
+
+namespace Appwrite\Tests;
+
+use ReflectionClass;
+use Appwrite\Migration\Version\V06;
+use Appwrite\Database\Database;
+use Appwrite\Database\Document;
+use PHPUnit\Framework\TestCase;
+
+class V06Test extends TestCase
+{
+    /**
+     * @var PDO
+     */
+    protected \PDO $pdo;
+
+    public function setUp(): void
+    {
+        $this->pdo = new \PDO('sqlite::memory:');
+    }
+
+    public function testMigration()
+    {
+        $v06 = new V06($this->pdo);
+
+        $reflector = new ReflectionClass('Appwrite\Migration\Version\V06');
+        $method = $reflector->getMethod('fixDocument');
+        $method->setAccessible(true);
+
+        $document =  $method->invokeArgs($v06, [
+            new Document([
+                '$id' => uniqid(),
+                '$collection' => Database::SYSTEM_COLLECTION_USERS,
+                'password-update' => 123
+            ])
+        ]);
+
+        $this->assertEquals($document->getAttribute('password-update', null), null);
+        $this->assertEquals($document->getAttribute('passwordUpdate', null), 123);
+
+        $document =  $method->invokeArgs($v06, [
+            new Document([
+                '$id' => uniqid(),
+                '$collection' => Database::SYSTEM_COLLECTION_KEYS,
+                'secret' => 123
+            ])
+        ]);
+        $encrypted = json_decode($document->getAttribute('secret', null));
+        $this->assertObjectHasAttribute('data', $encrypted);
+        $this->assertObjectHasAttribute('method', $encrypted);
+        $this->assertObjectHasAttribute('iv', $encrypted);
+        $this->assertObjectHasAttribute('tag', $encrypted);
+        $this->assertObjectHasAttribute('version', $encrypted);
+    }
+}

From b59c9abe21d89b57117c747706b3ffd8b9a1febe Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Mon, 18 Jan 2021 14:32:52 +0100
Subject: [PATCH 20/27] test(migration): rename class names

---
 tests/unit/Migration/V05Test.php | 2 +-
 tests/unit/Migration/V06Test.php | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/unit/Migration/V05Test.php b/tests/unit/Migration/V05Test.php
index 5206443c43..407e22f7cb 100644
--- a/tests/unit/Migration/V05Test.php
+++ b/tests/unit/Migration/V05Test.php
@@ -9,7 +9,7 @@ use Appwrite\Database\Document;
 use PHPUnit\Framework\TestCase;
 use Utopia\Config\Config;
 
-class V05Test extends TestCase
+class MigrationV05Test extends TestCase
 {
     /**
      * @var PDO
diff --git a/tests/unit/Migration/V06Test.php b/tests/unit/Migration/V06Test.php
index 9056e87fc0..97b776e2eb 100644
--- a/tests/unit/Migration/V06Test.php
+++ b/tests/unit/Migration/V06Test.php
@@ -8,7 +8,7 @@ use Appwrite\Database\Database;
 use Appwrite\Database\Document;
 use PHPUnit\Framework\TestCase;
 
-class V06Test extends TestCase
+class MigrationV05Test extends TestCase
 {
     /**
      * @var PDO

From 7d911bd976ca8dba7b9bcfab46580cbb7fe97c74 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Mon, 18 Jan 2021 14:33:36 +0100
Subject: [PATCH 21/27] test(migration): rename class names

---
 tests/unit/Migration/V06Test.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/Migration/V06Test.php b/tests/unit/Migration/V06Test.php
index 97b776e2eb..e43888ffa1 100644
--- a/tests/unit/Migration/V06Test.php
+++ b/tests/unit/Migration/V06Test.php
@@ -8,7 +8,7 @@ use Appwrite\Database\Database;
 use Appwrite\Database\Document;
 use PHPUnit\Framework\TestCase;
 
-class MigrationV05Test extends TestCase
+class MigrationV06Test extends TestCase
 {
     /**
      * @var PDO

From 64cae392d1de94195c0e20c56b262f54e09d53c4 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Mon, 18 Jan 2021 14:35:32 +0100
Subject: [PATCH 22/27] test(migration): rename class names

---
 tests/unit/Migration/{V05Test.php => MigrationV05Test.php} | 0
 tests/unit/Migration/{V06Test.php => MigrationV06Test.php} | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/unit/Migration/{V05Test.php => MigrationV05Test.php} (100%)
 rename tests/unit/Migration/{V06Test.php => MigrationV06Test.php} (100%)

diff --git a/tests/unit/Migration/V05Test.php b/tests/unit/Migration/MigrationV05Test.php
similarity index 100%
rename from tests/unit/Migration/V05Test.php
rename to tests/unit/Migration/MigrationV05Test.php
diff --git a/tests/unit/Migration/V06Test.php b/tests/unit/Migration/MigrationV06Test.php
similarity index 100%
rename from tests/unit/Migration/V06Test.php
rename to tests/unit/Migration/MigrationV06Test.php

From d4bf4027c150336d48a6b1f34d161b23c02fb416 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Mon, 18 Jan 2021 15:43:55 +0100
Subject: [PATCH 23/27] chore(migration): fixes phpdocs

---
 src/Appwrite/Migration/Migration.php | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php
index b0e6f4f991..a74c40cd81 100644
--- a/src/Appwrite/Migration/Migration.php
+++ b/src/Appwrite/Migration/Migration.php
@@ -6,26 +6,45 @@ use Appwrite\Database\Document;
 use Appwrite\Database\Database;
 use Utopia\CLI\Console;
 use Utopia\Exception;
+use PDO;
 
 abstract class Migration
 {
-    protected \PDO $db;
+    /**
+     * @var PDO
+     */
+    protected PDO $db;
 
+    /**
+     * @var int
+     */
     protected int $limit = 30;
 
+    /**
+     * @var Document
+     */
     protected Document $project;
+
+    /**
+     * @var Database
+     */
     protected Database $projectDB;
 
     /**
      * Migration constructor.
+     * 
+     * @param PDO $pdo
      */
-    public function __construct(\PDO $db)
+    public function __construct(PDO $db)
     {
         $this->db = $db;
     }
 
     /**
      * Set project for migration.
+     * 
+     * @param Document $project
+     * @param Database $projectDB
      */
     public function setProject(Document $project, Database $projectDB): Migration
     {
@@ -38,7 +57,7 @@ abstract class Migration
     /**
      * Iterates through every document.
      * 
-     * @param callable(Document):Document $callback
+     * @param callable $callback
      */
     public function forEachDocument(callable $callback)
     {

From ac3e908c04cf049ee463612134f9ce5f35040f8f Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Tue, 19 Jan 2021 09:35:51 +0100
Subject: [PATCH 24/27] fix(migration): prevent encrypting already encrypted
 values

---
 src/Appwrite/Migration/Version/V06.php    | 7 +++++++
 tests/unit/Migration/MigrationV06Test.php | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/src/Appwrite/Migration/Version/V06.php b/src/Appwrite/Migration/Version/V06.php
index 34620f93c7..686330620c 100644
--- a/src/Appwrite/Migration/Version/V06.php
+++ b/src/Appwrite/Migration/Version/V06.php
@@ -34,6 +34,13 @@ class V06 extends Migration
                 break;
             case Database::SYSTEM_COLLECTION_KEYS:
                 if ($document->getAttribute('secret', null)) {
+                    $json = \json_decode($document->getAttribute('secret'));
+                    if ($json->{'data'} || $json->{'method'} || $json->{'iv'} || $json->{'tag'} || $json->{'version'})
+                    {
+                        Console::log('Secret already encrypted. Skipped: ' . $document->getId());
+                        break;
+                    }
+
                     $key = App::getEnv('_APP_OPENSSL_KEY_V1');
                     $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
                     $tag = null;
diff --git a/tests/unit/Migration/MigrationV06Test.php b/tests/unit/Migration/MigrationV06Test.php
index e43888ffa1..bfe841980c 100644
--- a/tests/unit/Migration/MigrationV06Test.php
+++ b/tests/unit/Migration/MigrationV06Test.php
@@ -52,5 +52,8 @@ class MigrationV06Test extends TestCase
         $this->assertObjectHasAttribute('iv', $encrypted);
         $this->assertObjectHasAttribute('tag', $encrypted);
         $this->assertObjectHasAttribute('version', $encrypted);
+        
+        $document =  $method->invokeArgs($v06, [$document]);
+        $this->assertEquals($document->getAttribute('secret', null), json_encode($encrypted));
     }
 }

From fbcfa008566e677d17098b3b2491507483d09271 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Tue, 19 Jan 2021 09:54:19 +0100
Subject: [PATCH 25/27] test(migration): run fixing method twice

---
 tests/unit/Migration/MigrationTest.php    | 39 ++++++++++++
 tests/unit/Migration/MigrationV05Test.php | 74 +++++++++--------------
 tests/unit/Migration/MigrationV06Test.php | 39 +++++-------
 3 files changed, 82 insertions(+), 70 deletions(-)
 create mode 100644 tests/unit/Migration/MigrationTest.php

diff --git a/tests/unit/Migration/MigrationTest.php b/tests/unit/Migration/MigrationTest.php
new file mode 100644
index 0000000000..5a85ee23a2
--- /dev/null
+++ b/tests/unit/Migration/MigrationTest.php
@@ -0,0 +1,39 @@
+<?php
+
+namespace Appwrite\Tests;
+
+use Appwrite\Database\Document;
+use Appwrite\Migration\Migration;
+use PHPUnit\Framework\TestCase;
+use ReflectionMethod;
+
+abstract class MigrationTest extends TestCase
+{
+    /**
+     * @var PDO
+     */
+    protected \PDO $pdo;
+
+    /**
+     * @var Migration
+     */
+    protected Migration $migration;
+
+    /**
+     * @var ReflectionMethod
+     */
+    protected ReflectionMethod $method;
+
+    /**
+     * Runs every document fix twice, to prevent corrupted data on multiple migrations.
+     * 
+     * @param Document $document
+     */
+    protected function fixDocument(Document $document)
+    {
+        return $this->method->invokeArgs($this->migration, [
+            $this->method->invokeArgs($this->migration, [$document])
+        ]);
+    }
+
+}
diff --git a/tests/unit/Migration/MigrationV05Test.php b/tests/unit/Migration/MigrationV05Test.php
index 407e22f7cb..09e28dd054 100644
--- a/tests/unit/Migration/MigrationV05Test.php
+++ b/tests/unit/Migration/MigrationV05Test.php
@@ -6,39 +6,29 @@ use ReflectionClass;
 use Appwrite\Migration\Version\V05;
 use Appwrite\Database\Database;
 use Appwrite\Database\Document;
-use PHPUnit\Framework\TestCase;
 use Utopia\Config\Config;
 
-class MigrationV05Test extends TestCase
+class MigrationV05Test extends MigrationTest
 {
-    /**
-     * @var PDO
-     */
-    protected \PDO $pdo;
-
     public function setUp(): void
     {
+        Config::load('providers', __DIR__ . '/../../../app/config/providers.php');
+
         $this->pdo = new \PDO('sqlite::memory:');
+        $this->migration = new V05($this->pdo);
+        $reflector = new ReflectionClass('Appwrite\Migration\Version\V05');
+        $this->method = $reflector->getMethod('fixDocument');
+        $this->method->setAccessible(true);
     }
 
     public function testMigration()
     {
-        Config::load('providers', __DIR__ . '/../../../app/config/providers.php');
-
-        $v05 = new V05($this->pdo);
-
-        $reflector = new ReflectionClass('Appwrite\Migration\Version\V05');
-        $method = $reflector->getMethod('fixDocument');
-        $method->setAccessible(true);
-
-        $document =  $method->invokeArgs($v05, [
-            new Document([
-                '$uid' => 'unique',
-                '$collection' => Database::SYSTEM_COLLECTION_PROJECTS,
-                'usersOauthGithubAppid' => 123,
-                'usersOauthGithubSecret' => 456
-            ])
-        ]);
+        $document = $this->fixDocument(new Document([
+            '$uid' => 'unique',
+            '$collection' => Database::SYSTEM_COLLECTION_PROJECTS,
+            'usersOauthGithubAppid' => 123,
+            'usersOauthGithubSecret' => 456
+        ]));
 
         $this->assertEquals($document->getAttribute('$uid', null), null);
         $this->assertEquals($document->getAttribute('$id', null), 'unique');
@@ -51,27 +41,23 @@ class MigrationV05Test extends TestCase
 
         $this->assertEquals($document->getAttribute('security', true), false);
 
-        $document =  $method->invokeArgs($v05, [
-            new Document([
-                '$uid' => 'unique',
-                '$collection' => Database::SYSTEM_COLLECTION_TASKS
-            ])
-        ]);
+        $document = $this->fixDocument(new Document([
+            '$uid' => 'unique',
+            '$collection' => Database::SYSTEM_COLLECTION_TASKS
+        ]));
 
         $this->assertEquals($document->getAttribute('$uid', null), null);
         $this->assertEquals($document->getAttribute('$id', null), 'unique');
 
         $this->assertEquals($document->getAttribute('security', true), false);
 
-        $document =  $method->invokeArgs($v05, [
-            new Document([
-                '$uid' => 'unique',
-                '$collection' => Database::SYSTEM_COLLECTION_USERS,
-                'oauthGithub' => 'id',
-                'oauthGithubAccessToken' => 'token',
-                'confirm' => false
-            ])
-        ]);
+        $document = $this->fixDocument(new Document([
+            '$uid' => 'unique',
+            '$collection' => Database::SYSTEM_COLLECTION_USERS,
+            'oauthGithub' => 'id',
+            'oauthGithubAccessToken' => 'token',
+            'confirm' => false
+        ]));
 
         $this->assertEquals($document->getAttribute('$uid', null), null);
         $this->assertEquals($document->getAttribute('$id', null), 'unique');
@@ -85,13 +71,11 @@ class MigrationV05Test extends TestCase
         $this->assertEquals($document->getAttribute('oauthGithubAccessToken', null), null);
         $this->assertEquals($document->getAttribute('oauth2GithubAccessToken', null), 'token');
 
-        $document =  $method->invokeArgs($v05, [
-            new Document([
-                '$uid' => 'unique',
-                '$collection' => Database::SYSTEM_COLLECTION_PLATFORMS,
-                'url' => 'https://appwrite.io'
-            ])
-        ]);
+        $document = $this->fixDocument(new Document([
+            '$uid' => 'unique',
+            '$collection' => Database::SYSTEM_COLLECTION_PLATFORMS,
+            'url' => 'https://appwrite.io'
+        ]));
 
         $this->assertEquals($document->getAttribute('$uid', null), null);
         $this->assertEquals($document->getAttribute('$id', null), 'unique');
diff --git a/tests/unit/Migration/MigrationV06Test.php b/tests/unit/Migration/MigrationV06Test.php
index bfe841980c..19e892c435 100644
--- a/tests/unit/Migration/MigrationV06Test.php
+++ b/tests/unit/Migration/MigrationV06Test.php
@@ -6,54 +6,43 @@ use ReflectionClass;
 use Appwrite\Migration\Version\V06;
 use Appwrite\Database\Database;
 use Appwrite\Database\Document;
-use PHPUnit\Framework\TestCase;
 
-class MigrationV06Test extends TestCase
+class MigrationV06Test extends MigrationTest
 {
-    /**
-     * @var PDO
-     */
-    protected \PDO $pdo;
-
     public function setUp(): void
     {
         $this->pdo = new \PDO('sqlite::memory:');
+
+        $this->migration = new V06($this->pdo);
+        $reflector = new ReflectionClass('Appwrite\Migration\Version\V06');
+        $this->method = $reflector->getMethod('fixDocument');
+        $this->method->setAccessible(true);
     }
 
     public function testMigration()
     {
-        $v06 = new V06($this->pdo);
-
-        $reflector = new ReflectionClass('Appwrite\Migration\Version\V06');
-        $method = $reflector->getMethod('fixDocument');
-        $method->setAccessible(true);
-
-        $document =  $method->invokeArgs($v06, [
-            new Document([
-                '$id' => uniqid(),
-                '$collection' => Database::SYSTEM_COLLECTION_USERS,
-                'password-update' => 123
-            ])
-        ]);
+        $document = $this->fixDocument(new Document([
+            '$id' => uniqid(),
+            '$collection' => Database::SYSTEM_COLLECTION_USERS,
+            'password-update' => 123
+        ]));
 
         $this->assertEquals($document->getAttribute('password-update', null), null);
         $this->assertEquals($document->getAttribute('passwordUpdate', null), 123);
 
-        $document =  $method->invokeArgs($v06, [
+        $document = $this->fixDocument(
             new Document([
                 '$id' => uniqid(),
                 '$collection' => Database::SYSTEM_COLLECTION_KEYS,
                 'secret' => 123
             ])
-        ]);
+        );
+
         $encrypted = json_decode($document->getAttribute('secret', null));
         $this->assertObjectHasAttribute('data', $encrypted);
         $this->assertObjectHasAttribute('method', $encrypted);
         $this->assertObjectHasAttribute('iv', $encrypted);
         $this->assertObjectHasAttribute('tag', $encrypted);
         $this->assertObjectHasAttribute('version', $encrypted);
-        
-        $document =  $method->invokeArgs($v06, [$document]);
-        $this->assertEquals($document->getAttribute('secret', null), json_encode($encrypted));
     }
 }

From 9fc021c7eda1a1d28a045040b9d5ea25498f4286 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Thu, 21 Jan 2021 10:57:15 +0100
Subject: [PATCH 26/27] fix(migration): adapt to review

---
 src/Appwrite/Migration/Migration.php | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php
index a74c40cd81..a2b9a016ca 100644
--- a/src/Appwrite/Migration/Migration.php
+++ b/src/Appwrite/Migration/Migration.php
@@ -45,6 +45,8 @@ abstract class Migration
      * 
      * @param Document $project
      * @param Database $projectDB
+     * 
+     * @return Migration 
      */
     public function setProject(Document $project, Database $projectDB): Migration
     {
@@ -59,12 +61,13 @@ abstract class Migration
      * 
      * @param callable $callback
      */
-    public function forEachDocument(callable $callback)
+    public function forEachDocument(callable $callback): void
     {
-        $sum = 30;
+        
+        $sum = $this->limit;
         $offset = 0;
 
-        while ($sum >= 30) {
+        while ($sum >= $this->limit) {
             $all = $this->projectDB->getCollection([
                 'limit' => $this->limit,
                 'offset' => $offset,
@@ -86,7 +89,6 @@ abstract class Migration
                 try {
                     $new = $this->projectDB->overwriteDocument($document->getArrayCopy());
                 } catch (\Throwable $th) {
-                    var_dump($document);
                     Console::error('Failed to update document: ' . $th->getMessage());
                     continue;
                 }

From 968d733881fd7c1d386b4c293605ea4d7826a4d0 Mon Sep 17 00:00:00 2001
From: Torsten Dittmann <torsten.dittmann@googlemail.com>
Date: Mon, 25 Jan 2021 09:26:52 +0100
Subject: [PATCH 27/27] introduce swoole coroutine to migration

---
 app/http.php                         |  1 +
 src/Appwrite/Migration/Migration.php | 48 +++++++++++++++++-----------
 2 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/app/http.php b/app/http.php
index 74fb950825..9e45bc3b52 100644
--- a/app/http.php
+++ b/app/http.php
@@ -18,6 +18,7 @@ use Utopia\CLI\Console;
 ini_set('memory_limit','512M');
 ini_set('display_errors', 1);
 ini_set('display_startup_errors', 1);
+ini_set('default_socket_timeout', -1);
 error_reporting(E_ALL);
 
 $http = new Server("0.0.0.0", App::getEnv('PORT', 80));
diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php
index a2b9a016ca..fabf968b06 100644
--- a/src/Appwrite/Migration/Migration.php
+++ b/src/Appwrite/Migration/Migration.php
@@ -4,9 +4,10 @@ namespace Appwrite\Migration;
 
 use Appwrite\Database\Document;
 use Appwrite\Database\Database;
+use PDO;
+use Swoole\Runtime;
 use Utopia\CLI\Console;
 use Utopia\Exception;
-use PDO;
 
 abstract class Migration
 {
@@ -18,7 +19,7 @@ abstract class Migration
     /**
      * @var int
      */
-    protected int $limit = 30;
+    protected int $limit = 50;
 
     /**
      * @var Document
@@ -63,7 +64,6 @@ abstract class Migration
      */
     public function forEachDocument(callable $callback): void
     {
-        
         $sum = $this->limit;
         $offset = 0;
 
@@ -75,28 +75,38 @@ abstract class Migration
             ]);
 
             $sum = \count($all);
+            Runtime::setHookFlags(SWOOLE_HOOK_ALL);
 
             Console::log('Migrating: ' . $offset . ' / ' . $this->projectDB->getSum());
+            \Co\run(function () use ($all, $callback) {
+                Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
 
-            foreach ($all as $document) {
+                foreach ($all as $document) {
+                    go(function () use ($document, $callback) {
 
-                $document = call_user_func($callback, $document);
+                        $old = $document->getArrayCopy();
+                        $new = call_user_func($callback, $document);
 
-                if (empty($document->getId())) {
-                    throw new Exception('Missing ID');
+                        if (empty($new->getId())) {
+                            throw new Exception('Missing ID');
+                        }
+                        if (!array_diff($new->getArrayCopy(), $old)) {
+                            return;
+                        }
+
+                        try {
+                            $new = $this->projectDB->overwriteDocument($document->getArrayCopy());
+                        } catch (\Throwable $th) {
+                            Console::error('Failed to update document: ' . $th->getMessage());
+                            return;
+
+                            if ($document && $new->getId() !== $document->getId()) {
+                                throw new Exception('Duplication Error');
+                            }
+                        }
+                    });
                 }
-
-                try {
-                    $new = $this->projectDB->overwriteDocument($document->getArrayCopy());
-                } catch (\Throwable $th) {
-                    Console::error('Failed to update document: ' . $th->getMessage());
-                    continue;
-                }
-
-                if ($new->getId() !== $document->getId()) {
-                    throw new Exception('Duplication Error');
-                }
-            }
+            });
 
             $offset += $this->limit;
         }