From 53deb68985af5a705be0965857154e035d653aa1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matej=20Ba=C4=8Do?= <matejbaco2000@gmail.com>
Date: Fri, 16 Jun 2023 11:07:47 +0200
Subject: [PATCH] Improve performance of listRepositories

---
 app/controllers/api/vcs.php | 54 ++++++++++++++++++++++++++++---------
 app/http.php                |  3 +++
 app/workers/builds.php      |  2 ++
 3 files changed, 47 insertions(+), 12 deletions(-)

diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php
index 66f722a149..8309caf343 100644
--- a/app/controllers/api/vcs.php
+++ b/app/controllers/api/vcs.php
@@ -1,5 +1,6 @@
 <?php
 
+use Swoole\Coroutine as Co;
 use Utopia\App;
 use Appwrite\Event\Build;
 use Appwrite\Event\Delete;
@@ -30,6 +31,8 @@ use Utopia\Detector\Adapter\Ruby;
 use Utopia\Detector\Adapter\Swift;
 use Utopia\Detector\Detector;
 
+use function Swoole\Coroutine\batch;
+
 App::get('/v1/vcs/github/installations')
     ->desc('Install GitHub App')
     ->groups(['api', 'vcs'])
@@ -148,6 +151,8 @@ App::get('/v1/vcs/github/installations/:installationId/repositories')
     ->inject('project')
     ->inject('dbForConsole')
     ->action(function (string $vcsInstallationId, string $search, GitHub $github, Response $response, Document $project, Database $dbForConsole) {
+        $start = \microtime(true);
+
         if (empty($search)) {
             $search = "";
         }
@@ -166,30 +171,53 @@ App::get('/v1/vcs/github/installations/:installationId/repositories')
         $github->initialiseVariables($installationId, $privateKey, $githubAppId);
 
         $page = 1;
-        $per_page = 100; // max limit of GitHub API
-        $repos = []; // Array to store all repositories
+        $perPage = 100;
 
-        do {
-            $repositories = $github->listRepositoriesForGitHubApp($page, $per_page);
-            $repos = array_merge($repos, $repositories);
-            $page++;
-        } while (\count($repositories) === $per_page);
+        $loadPage = function ($page) use ($github, $perPage) {
+            $repos = $github->listRepositoriesForGitHubApp($page, $perPage);
+            return $repos;
+        };
+
+        $reposPages = batch([
+            function () use ($loadPage) {
+                return $loadPage(1);
+            },
+            function () use ($loadPage) {
+                return $loadPage(2);
+            },
+            function () use ($loadPage) {
+                return $loadPage(3);
+            }
+        ]);
+
+        $page += 3;
+        $repos = [];
+        foreach ($reposPages as $reposPage) {
+            $repos = \array_merge($repos, $reposPage);
+        }
+
+        // All 3 pages were full, we paginate more
+        if(\count($repos) === 3 * $perPage) {
+            do {
+                $reposPage = $loadPage($page);
+                $repos = array_merge($repos, $reposPage);
+                $page++;
+            } while (\count($reposPage) === $perPage);
+        }
 
         // Filter repositories based on search parameter
         if (!empty($search)) {
             $repos = array_filter($repos, function ($repo) use ($search) {
-                $repoName = strtolower($repo['name']);
-                $searchTerm = strtolower($search);
-                return strpos($repoName, $searchTerm) !== false;
+                return \str_contains(\strtolower($repo['name']), \strtolower($search));
             });
         }
         // Sort repositories by last modified date in descending order
         usort($repos, function ($repo1, $repo2) {
-            return strtotime($repo2['pushed_at']) - strtotime($repo1['pushed_at']);
+            return \strtotime($repo2['pushed_at']) - \strtotime($repo1['pushed_at']);
         });
 
         // Limit the maximum results to 5
-        $repos = array_slice($repos, 0, 5);
+        $repos = \array_slice($repos, 0, 5);
 
         $repos = \array_map(function ($repo) {
             $repo['id'] = \strval($repo['id']);
@@ -197,6 +225,8 @@ App::get('/v1/vcs/github/installations/:installationId/repositories')
             return new Document($repo);
         }, $repos);
 
+        \var_dump(\microtime(true) - $start);
+
         $response->dynamic(new Document([
             'repositories' => $repos,
             'total' => \count($repos),
diff --git a/app/http.php b/app/http.php
index aa5ab3ec2c..f8b6f592d1 100644
--- a/app/http.php
+++ b/app/http.php
@@ -20,10 +20,13 @@ use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Swoole\Files;
 use Appwrite\Utopia\Request;
+use Swoole\Runtime;
 use Utopia\Logger\Log;
 use Utopia\Logger\Log\User;
 use Utopia\Pools\Group;
 
+Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
+
 $http = new Server("0.0.0.0", App::getEnv('PORT', 80));
 
 $payloadSize = 6 * (1024 * 1024); // 6MB
diff --git a/app/workers/builds.php b/app/workers/builds.php
index 8d6271b0a1..a3fde0bc3b 100644
--- a/app/workers/builds.php
+++ b/app/workers/builds.php
@@ -278,6 +278,7 @@ class BuildsV1 extends Worker
             Co\run(function () use ($project, $deployment, &$response, $source, $function, $runtime, $vars, $command, &$build, $dbForProject, $allEvents) {
                 Co::join([
                     Co\go(function () use ($project, $deployment, &$response, &$build, $dbForProject, $allEvents) {
+                        \var_dump("Start1");
                         $this->executor->getLogs(
                             projectId: $project->getId(),
                             deploymentId: $deployment->getId(),
@@ -307,6 +308,7 @@ class BuildsV1 extends Worker
                         );
                     }),
                     Co\go(function () use (&$response, $project, $deployment, $source, $function, $runtime, $vars, $command) {
+                        \var_dump("Start2");
                         $response = $this->executor->createRuntime(
                             projectId: $project->getId(),
                             deploymentId: $deployment->getId(),