diff --git a/app/workers.php b/app/workers.php index 7ae36c1922..7dd48eaff8 100644 --- a/app/workers.php +++ b/app/workers.php @@ -1,13 +1,7 @@ set('cache', function () { // Register cache connection return $redis; }); -/** - * Get internal project database - * @param string $projectId - * @return Database - */ -function getInternalDB(string $projectId): Database -{ - global $register; - - $attempts = 0; - $max = 10; - $sleep = 2; - - do { - try { - $attempts++; - $cache = new Cache(new RedisCache($register->get('cache'))); - $dbForInternal = new Database(new MariaDB($register->get('db')), $cache); - $dbForInternal->setNamespace("project_{$projectId}_internal"); // Main DB - if (!$dbForInternal->exists()) { - throw new Exception("Table does not exist: {$dbForInternal->getNamespace()}"); - } - break; // leave loop if successful - } catch(\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('Failed to connect to database: '. $e->getMessage()); - } - sleep($sleep); - } - } while ($attempts < $max); - - return $dbForInternal; -} - -/** - * Get external project database - * @param string $projectId - * @return Database - */ -function getExternalDB(string $projectId): Database -{ - global $register; - - $attempts = 0; - $max = 10; - $sleep = 2; - - do { - try { - $attempts++; - $cache = new Cache(new RedisCache($register->get('cache'))); - $dbForExternal = new Database(new MariaDB($register->get('db')), $cache); - $dbForExternal->setNamespace("project_{$projectId}_external"); // Main DB - if (!$dbForExternal->exists()) { - throw new Exception("Table does not exist: {$dbForExternal->getNamespace()}"); - } - break; // leave loop if successful - } catch(\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('Failed to connect to database: '. $e->getMessage()); - } - sleep($sleep); - } - } while ($attempts < $max); - - return $dbForExternal; -} - -/** - * Get console database - * @return Database - */ -function getConsoleDB(): Database -{ - global $register; - - $attempts = 0; - $max = 5; - $sleep = 5; - - do { - try { - $attempts++; - $cache = new Cache(new RedisCache($register->get('cache'))); - $dbForConsole = new Database(new MariaDB($register->get('db')), $cache); - $dbForConsole->setNamespace('project_console_internal'); // Main DB - if (!$dbForConsole->exists()) { - throw new Exception("Table does not exist: {$dbForConsole->getNamespace()}"); - } - break; // leave loop if successful - } catch(\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('Failed to connect to database: '. $e->getMessage()); - } - sleep($sleep); - } - } while ($attempts < $max); - - return $dbForConsole; -} diff --git a/app/workers/audits.php b/app/workers/audits.php index 2b603092fe..663de3ce4f 100644 --- a/app/workers/audits.php +++ b/app/workers/audits.php @@ -2,11 +2,7 @@ use Appwrite\Resque\Worker; use Utopia\Audit\Audit; -use Utopia\Cache\Adapter\Redis; -use Utopia\Cache\Cache; use Utopia\CLI\Console; -use Utopia\Database\Adapter\MariaDB; -use Utopia\Database\Database; require_once __DIR__.'/../workers.php'; @@ -31,7 +27,7 @@ class AuditsV1 extends Worker $ip = $this->args['ip']; $data = $this->args['data']; - $dbForInternal = getInternalDB($projectId); + $dbForInternal = $this->getInternalDB($projectId); $audit = new Audit($dbForInternal); $audit->log($userId, $event, $resource, $userAgent, $ip, '', $data); diff --git a/app/workers/database.php b/app/workers/database.php index bdf33a4b37..544ffe1060 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -4,7 +4,7 @@ use Appwrite\Resque\Worker; use Utopia\CLI\Console; use Utopia\Database\Document; -require_once __DIR__.'/../init.php'; +require_once __DIR__.'/../workers.php'; Console::title('Database V1 Worker'); Console::success(APP_NAME.' database worker v1 has started'."\n"); @@ -61,7 +61,7 @@ class DatabaseV1 extends Worker */ protected function createAttribute($attribute, $projectId): void { - $dbForExternal = getExternalDB($projectId); + $dbForExternal = $this->getExternalDB($projectId); $collectionId = $attribute->getCollection(); $id = $attribute->getAttribute('$id', ''); @@ -85,7 +85,7 @@ class DatabaseV1 extends Worker */ protected function deleteAttribute($attribute, $projectId): void { - $dbForExternal = getExternalDB($projectId); + $dbForExternal = $this->getExternalDB($projectId); $collectionId = $attribute->getCollection(); $id = $attribute->getAttribute('$id'); @@ -99,7 +99,7 @@ class DatabaseV1 extends Worker */ protected function createIndex($index, $projectId): void { - $dbForExternal = getExternalDB($projectId); + $dbForExternal = $this->getExternalDB($projectId); $collectionId = $index->getCollection(); $id = $index->getAttribute('$id', ''); @@ -120,7 +120,7 @@ class DatabaseV1 extends Worker */ protected function deleteIndex($index, $projectId): void { - $dbForExternal = getExternalDB($projectId); + $dbForExternal = $this->getExternalDB($projectId); $collectionId = $index->getCollection(); $id = $index->getAttribute('$id'); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 8ecce1d483..e98723a5ea 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -93,7 +93,7 @@ class DeletesV1 extends Worker // Delete Memberships $this->deleteByGroup('memberships', [ new Query('teamId', Query::TYPE_EQUAL, [$teamId]) - ], getInternalDB($projectId)); + ], $this->getInternalDB($projectId)); } /** @@ -103,8 +103,8 @@ class DeletesV1 extends Worker { $projectId = $document->getId(); // Delete all DBs - getExternalDB($projectId)->delete(); - getInternalDB($projectId)->delete(); + $this->getExternalDB($projectId)->delete(); + $this->getInternalDB($projectId)->delete(); // Delete all storage directories $uploads = new Local(APP_STORAGE_UPLOADS.'/app-'.$document->getId()); @@ -126,13 +126,13 @@ class DeletesV1 extends Worker // Delete Memberships and decrement team membership counts $this->deleteByGroup('memberships', [ new Query('userId', Query::TYPE_EQUAL, [$userId]) - ], getInternalDB($projectId), function(Document $document) use ($projectId, $userId) { + ], $this->getInternalDB($projectId), function(Document $document) use ($projectId, $userId) { if ($document->getAttribute('confirm')) { // Count only confirmed members $teamId = $document->getAttribute('teamId'); - $team = getInternalDB($projectId)->getDocument('teams', $teamId); + $team = $this->getInternalDB($projectId)->getDocument('teams', $teamId); if(!$team->isEmpty()) { - $team = getInternalDB($projectId)->updateDocument('teams', $teamId, new Document(\array_merge($team->getArrayCopy(), [ + $team = $this->getInternalDB($projectId)->updateDocument('teams', $teamId, new Document(\array_merge($team->getArrayCopy(), [ 'sum' => \max($team->getAttribute('sum', 0) - 1, 0), // Ensure that sum >= 0 ]))); } @@ -146,7 +146,7 @@ class DeletesV1 extends Worker protected function deleteExecutionLogs($timestamp) { $this->deleteForProjectIds(function($projectId) use ($timestamp) { - if (!($dbForInternal = getInternalDB($projectId))) { + if (!($dbForInternal = $this->getInternalDB($projectId))) { throw new Exception('Failed to get projectDB for project '.$projectId); } @@ -167,7 +167,7 @@ class DeletesV1 extends Worker } $this->deleteForProjectIds(function($projectId) use ($timestamp){ - $timeLimit = new TimeLimit("", 0, 1, getInternalDB($projectId)); + $timeLimit = new TimeLimit("", 0, 1, $this->getInternalDB($projectId)); $abuse = new Abuse($timeLimit); $status = $abuse->cleanup($timestamp); @@ -186,7 +186,7 @@ class DeletesV1 extends Worker throw new Exception('Failed to delete audit logs. No timestamp provided'); } $this->deleteForProjectIds(function($projectId) use ($timestamp){ - $audit = new Audit(getInternalDB($projectId)); + $audit = new Audit($this->getInternalDB($projectId)); $status = $audit->cleanup($timestamp); if (!$status) { throw new Exception('Failed to delete Audit logs for project'.$projectId); @@ -200,7 +200,7 @@ class DeletesV1 extends Worker */ protected function deleteFunction(Document $document, $projectId) { - $dbForInternal = getInternalDB($projectId); + $dbForInternal = $this->getInternalDB($projectId); $device = new Local(APP_STORAGE_FUNCTIONS.'/app-'.$projectId); // Delete Tags @@ -269,7 +269,7 @@ class DeletesV1 extends Worker $chunk++; Authorization::disable(); - $projects = getConsoleDB()->find('projects', [], $limit); + $projects = $this->getConsoleDB()->find('projects', [], $limit); Authorization::reset(); $projectIds = array_map (function ($project) { diff --git a/app/workers/functions.php b/app/workers/functions.php index 8db7d64753..b6263cc5f8 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -6,11 +6,8 @@ use Appwrite\Utopia\Response\Model\Execution; use Cron\CronExpression; use Swoole\Runtime; use Utopia\App; -use Utopia\Cache\Adapter\Redis; -use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; -use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; @@ -139,9 +136,6 @@ class FunctionsV1 extends Worker { global $register; - $db = $register->get('db'); - $cache = $register->get('cache'); - $projectId = $this->args['projectId'] ?? ''; $functionId = $this->args['functionId'] ?? ''; $webhooks = $this->args['webhooks'] ?? []; @@ -154,9 +148,7 @@ class FunctionsV1 extends Worker $userId = $this->args['userId'] ?? ''; $jwt = $this->args['jwt'] ?? ''; - $cache = new Cache(new Redis($cache)); - $database = new Database(new MariaDB($db), $cache); - $database->setNamespace('project_'.$projectId.'_internal'); + $database = $this->getInternalDB($projectId); switch ($trigger) { case 'event': diff --git a/app/workers/tasks.php b/app/workers/tasks.php index cbc8d9573e..b3469393ef 100644 --- a/app/workers/tasks.php +++ b/app/workers/tasks.php @@ -39,7 +39,7 @@ class TasksV1 extends Worker $logLimit = 5; $alert = ''; - $dbForConsole = getConsoleDB(); + $dbForConsole = $this->getConsoleDB(); /* * 1. Get Original Task diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index db8dc91ce7..93c666723e 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -2,6 +2,13 @@ namespace Appwrite\Resque; +use Exception; +use Utopia\Cache\Cache; +use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\CLI\Console; +use Utopia\Database\Database; +use Utopia\Database\Adapter\MariaDB; + abstract class Worker { public $args = []; @@ -26,4 +33,107 @@ abstract class Worker { $this->shutdown(); } + /** + * Get internal project database + * @param string $projectId + * @return Database + */ + protected function getInternalDB(string $projectId): Database + { + global $register; + + $attempts = 0; + $max = 10; + $sleep = 2; + + do { + try { + $attempts++; + $cache = new Cache(new RedisCache($register->get('cache'))); + $dbForInternal = new Database(new MariaDB($register->get('db')), $cache); + $dbForInternal->setNamespace("project_{$projectId}_internal"); // Main DB + if (!$dbForInternal->exists()) { + throw new Exception("Table does not exist: {$dbForInternal->getNamespace()}"); + } + break; // leave loop if successful + } catch(\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed to connect to database: '. $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); + + return $dbForInternal; + } + + /** + * Get external project database + * @param string $projectId + * @return Database + */ + protected function getExternalDB(string $projectId): Database + { + global $register; + + $attempts = 0; + $max = 10; + $sleep = 2; + + do { + try { + $attempts++; + $cache = new Cache(new RedisCache($register->get('cache'))); + $dbForExternal = new Database(new MariaDB($register->get('db')), $cache); + $dbForExternal->setNamespace("project_{$projectId}_external"); // Main DB + if (!$dbForExternal->exists()) { + throw new Exception("Table does not exist: {$dbForExternal->getNamespace()}"); + } + break; // leave loop if successful + } catch(\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed to connect to database: '. $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); + + return $dbForExternal; + } + + /** + * Get console database + * @return Database + */ + protected function getConsoleDB(): Database + { + global $register; + + $attempts = 0; + $max = 5; + $sleep = 5; + + do { + try { + $attempts++; + $cache = new Cache(new RedisCache($register->get('cache'))); + $dbForConsole = new Database(new MariaDB($register->get('db')), $cache); + $dbForConsole->setNamespace('project_console_internal'); // Main DB + if (!$dbForConsole->exists()) { + throw new Exception("Table does not exist: {$dbForConsole->getNamespace()}"); + } + break; // leave loop if successful + } catch(\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed to connect to database: '. $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); + + return $dbForConsole; + } } \ No newline at end of file