Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/db-pools-backups' into db-pools-backups

Browse Source
This commit is contained in:
fogelito
2023-08-15 12:49:22 +03:00
parent 4fdd76dc95 fbae9f859f
commit 8a965b2ce3
12 changed files with 142 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitmodules
+1 -1
View File
@@ -1,4 +1,4 @@
[submodule "app/console"]
path = app/console
url = https://github.com/appwrite/console
branch = cloud-cards
branch = cloud-1.1.x
app/console
+1 -1
Submodule app/console updated: b981302dee...59d6ebe70e
app/controllers/api/projects.php
+2 -1
View File
@@ -95,6 +95,7 @@ App::post('/v1/projects')
$backups['database_db_fra1_03'] = ['from' => '10:30', 'to' => '11:15'];
$backups['database_db_fra1_04'] = ['from' => '13:30', 'to' => '14:15'];
$backups['database_db_fra1_05'] = ['from' => '4:30', 'to' => '5:15'];
$backups['database_db_fra1_06'] = ['from' => '16:30', 'to' => '17:15'];
$databases = Config::getParam('pools-database', []);
@@ -118,7 +119,7 @@ App::post('/v1/projects')
}
}
if ($index = array_search('database_db_fra1_05', $databases)) {
if ($index = array_search('database_db_fra1_06', $databases)) {
$database = $databases[$index];
} else {
$database = $databases[array_rand($databases)];
app/controllers/api/teams.php
-12
View File
@@ -255,18 +255,6 @@ App::delete('/v1/teams/:teamId')
throw new Exception(Exception::TEAM_NOT_FOUND);
}
$memberships = $dbForProject->find('memberships', [
Query::equal('teamId', [$teamId]),
Query::limit(2000), // TODO fix members limit
]);
// TODO delete all members individually from the user object
foreach ($memberships as $membership) {
if (!$dbForProject->deleteDocument('memberships', $membership->getId())) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove membership for team from DB');
}
}
if (!$dbForProject->deleteDocument('teams', $teamId)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove team from DB');
}
app/controllers/api/users.php
+1 -1
View File
@@ -414,7 +414,7 @@ App::get('/v1/users/:userId')
$user = $dbForProject->getDocument('users', $userId);
if ($user->isEmpty()) {
throw new Exception('User not found', 404, Exception::USER_NOT_FOUND);
throw new Exception(Exception::USER_NOT_FOUND);
}
$response->dynamic($user, Response::MODEL_USER);
app/controllers/web/console.php
+1
View File
@@ -9,6 +9,7 @@ App::get('/console/*')
->alias('auth/*')
->alias('/invite')
->alias('/login')
->alias('/card/*')
->alias('/recover')
->alias('/register/*')
->groups(['web'])
app/workers/deletes.php
+39 -17
View File
@@ -8,7 +8,6 @@ use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Appwrite\Resque\Worker;
use Executor\Executor;
use Utopia\Storage\Device\Local;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit;
@@ -147,7 +146,8 @@ class DeletesV1 extends Worker
$project = $this->getConsoleDB()->getDocument('projects', $document->getAttribute('projectId'));
if ($project->isEmpty()) {
Console::warning('Unable to delete schedule for function ' . $document->getAttribute('resourceId'));
$this->getConsoleDB()->deleteDocument('schedules', $document->getId());
Console::success('Deleted schedule for deleted project ' . $document->getAttribute('projectId'));
return;
}
@@ -155,7 +155,7 @@ class DeletesV1 extends Worker
if ($function->isEmpty()) {
$this->getConsoleDB()->deleteDocument('schedules', $document->getId());
Console::success('Deleting schedule for function ' . $document->getAttribute('resourceId'));
Console::success('Deleted schedule for function ' . $document->getAttribute('resourceId'));
}
}
);
@@ -255,6 +255,7 @@ class DeletesV1 extends Worker
protected function deleteCollection(Document $document, Document $project): void
{
$collectionId = $document->getId();
$collectionInternalId = $document->getInternalId();
$databaseId = $document->getAttribute('databaseId');
$databaseInternalId = $document->getAttribute('databaseInternalId');
@@ -263,13 +264,13 @@ class DeletesV1 extends Worker
$dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId());
$this->deleteByGroup('attributes', [
Query::equal('databaseId', [$databaseId]),
Query::equal('collectionId', [$collectionId])
Query::equal('databaseInternalId', [$databaseInternalId]),
Query::equal('collectionInternalId', [$collectionInternalId])
], $dbForProject);
$this->deleteByGroup('indexes', [
Query::equal('databaseId', [$databaseId]),
Query::equal('collectionId', [$collectionId])
Query::equal('databaseInternalId', [$databaseInternalId]),
Query::equal('collectionInternalId', [$collectionInternalId])
], $dbForProject);
$this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project);
@@ -296,12 +297,21 @@ class DeletesV1 extends Worker
*/
protected function deleteMemberships(Document $document, Document $project): void
{
$teamId = $document->getAttribute('teamId', '');
$dbForProject = $this->getProjectDB($project);
$teamInternalId = $document->getInternalId();
// Delete Memberships
$this->deleteByGroup('memberships', [
Query::equal('teamId', [$teamId])
], $this->getProjectDB($project));
$this->deleteByGroup(
'memberships',
[
Query::equal('teamInternalId', [$teamInternalId])
],
$dbForProject,
function (Document $membership) use ($dbForProject) {
$userId = $membership->getAttribute('userId');
$dbForProject->deleteCachedDocument('users', $userId);
}
);
}
/**
@@ -329,19 +339,20 @@ class DeletesV1 extends Worker
protected function deleteUser(Document $document, Document $project): void
{
$userId = $document->getId();
$userInternalId = $document->getInternalId();
$dbForProject = $this->getProjectDB($project);
// Delete all sessions of this user from the sessions table and update the sessions field of the user record
$this->deleteByGroup('sessions', [
Query::equal('userId', [$userId])
Query::equal('userInternalId', [$userInternalId])
], $dbForProject);
$dbForProject->deleteCachedDocument('users', $userId);
// Delete Memberships and decrement team membership counts
$this->deleteByGroup('memberships', [
Query::equal('userId', [$userId])
Query::equal('userInternalId', [$userInternalId])
], $dbForProject, function (Document $document) use ($dbForProject) {
if ($document->getAttribute('confirm')) { // Count only confirmed members
$teamId = $document->getAttribute('teamId');
@@ -351,7 +362,7 @@ class DeletesV1 extends Worker
'teams',
$teamId,
// Ensure that total >= 0
$team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
$team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
);
}
}
@@ -359,7 +370,7 @@ class DeletesV1 extends Worker
// Delete tokens
$this->deleteByGroup('tokens', [
Query::equal('userId', [$userId])
Query::equal('userInternalId', [$userInternalId])
], $dbForProject);
}
@@ -474,13 +485,14 @@ class DeletesV1 extends Worker
$projectId = $project->getId();
$dbForProject = $this->getProjectDB($project);
$functionId = $document->getId();
$functionInternalId = $document->getInternalId();
/**
* Delete Variables
*/
Console::info("Deleting variables for function " . $functionId);
$this->deleteByGroup('variables', [
Query::equal('functionId', [$functionId])
Query::equal('functionInternalId', [$functionInternalId])
], $dbForProject);
/**
@@ -678,16 +690,26 @@ class DeletesV1 extends Worker
$limit = 50;
$results = [];
$sum = $limit;
$cursor = null;
$executionStart = \microtime(true);
while ($sum === $limit) {
$chunk++;
$results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
$mergedQueries = \array_merge([Query::limit($limit)], $queries);
if ($cursor instanceof Document) {
$mergedQueries[] = Query::cursorAfter($cursor);
}
$results = $database->find($collection, $mergedQueries);
$sum = count($results);
if ($sum > 0) {
$cursor = $results[$sum - 1];
}
foreach ($results as $document) {
if (is_callable($callback)) {
$callback($document);
bin/upgrade
Executable
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php upgrade $@
src/Appwrite/Platform/Services/Tasks.php
+2
View File
@@ -24,6 +24,7 @@ use Appwrite\Platform\Tasks\VolumeSync;
use Appwrite\Platform\Tasks\CalcUsersStats;
use Appwrite\Platform\Tasks\CalcTierStats;
use Appwrite\Platform\Tasks\PatchDeleteProjectCollections;
use Appwrite\Platform\Tasks\Upgrade;
class Tasks extends Service
{
@@ -38,6 +39,7 @@ class Tasks extends Service
->addAction(Hamster::getName(), new Hamster())
->addAction(Doctor::getName(), new Doctor())
->addAction(Install::getName(), new Install())
->addAction(Upgrade::getName(), new Upgrade())
->addAction(Maintenance::getName(), new Maintenance())
->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules())
->addAction(ClearCardCache::getName(), new ClearCardCache())
src/Appwrite/Platform/Tasks/Backup.php
+4 -1
View File
@@ -16,7 +16,7 @@ use Utopia\Validator\Text;
class Backup extends Action
{
public const BACKUPS_PATH = '/backups';
public const BACKUP_INTERVAL_SECONDS = 60 * 60 * 4; // 4 hours;
public const BACKUP_INTERVAL_SECONDS = 60 * 60 * 1; // 4 hours;
public const COMPRESS_ALGORITHM = 'lz4';
public const CONFIG_PATH = '/etc/my.cnf';
protected ?DSN $dsn = null;
@@ -56,6 +56,9 @@ class Backup extends Action
Console::exit();
}
console::info('Trying to connect to ' . $this->dsn->getHost() . ':' . $this->dsn->getPort() .
'user:' . $this->dsn->getUser() . 'password:' . $this->dsn->getPassword());
try {
$dsn = new DSN(App::getEnv('_APP_CONNECTIONS_BACKUPS_STORAGE', ''));
$this->s3 = new DOSpaces('/' . $database . '/full', $dsn->getUser(), $dsn->getPassword(), $dsn->getPath(), $dsn->getParam('region'));
src/Appwrite/Platform/Tasks/Install.php
+46 -19
View File
@@ -6,7 +6,8 @@ use Appwrite\Auth\Auth;
use Appwrite\Docker\Compose;
use Appwrite\Docker\Env;
use Appwrite\Utopia\View;
use Utopia\Analytics\GoogleAnalytics;
use Utopia\Analytics\Adapter\GoogleAnalytics;
use Utopia\Analytics\Event;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Validator\Text;
@@ -14,6 +15,8 @@ use Utopia\Platform\Action;
class Install extends Action
{
protected string $path = '/usr/src/code/appwrite';
public static function getName(): string
{
return 'install';
@@ -50,7 +53,6 @@ class Install extends Action
* 6. Run data migration
*/
$config = Config::getParam('variables');
$path = '/usr/src/code/appwrite';
$defaultHTTPPort = '80';
$defaultHTTPSPort = '443';
$vars = [];
@@ -70,19 +72,28 @@ class Install extends Action
Console::success('Starting Appwrite installation...');
// Create directory with write permissions
if (null !== $path && !\file_exists(\dirname($path))) {
if (!@\mkdir(\dirname($path), 0755, true)) {
Console::error('Can\'t create directory ' . \dirname($path));
if (null !== $this->path && !\file_exists(\dirname($this->path))) {
if (!@\mkdir(\dirname($this->path), 0755, true)) {
Console::error('Can\'t create directory ' . \dirname($this->path));
Console::exit(1);
}
}
$data = @file_get_contents($path . '/docker-compose.yml');
$data = @file_get_contents($this->path . '/docker-compose.yml');
if ($data !== false) {
if ($interactive == 'Y' && Console::isInteractive()) {
$answer = Console::confirm('Previous installation found, do you want to overwrite it? (Y/n)');
if ($answer !== 'Y') {
Console::info('No action taken.');
return;
}
}
$time = \time();
Console::info('Compose file found, creating backup: docker-compose.yml.' . $time . '.backup');
file_put_contents($path . '/docker-compose.yml.' . $time . '.backup', $data);
file_put_contents($this->path . '/docker-compose.yml.' . $time . '.backup', $data);
$compose = new Compose($data);
$appwrite = $compose->getService('appwrite');
$oldVersion = ($appwrite) ? $appwrite->getImageVersion() : null;
@@ -116,11 +127,11 @@ class Install extends Action
}
}
$data = @file_get_contents($path . '/.env');
$data = @file_get_contents($this->path . '/.env');
if ($data !== false) { // Fetch all env vars from previous .env file
Console::info('Env file found, creating backup: .env.' . $time . '.backup');
file_put_contents($path . '/.env.' . $time . '.backup', $data);
file_put_contents($this->path . '/.env.' . $time . '.backup', $data);
$env = new Env($data);
foreach ($env->list() as $key => $value) {
@@ -198,8 +209,8 @@ class Install extends Action
}
}
$templateForCompose = new View(__DIR__ . '/../views/install/compose.phtml');
$templateForEnv = new View(__DIR__ . '/../views/install/env.phtml');
$templateForCompose = new View(__DIR__ . '/../../../../app/views/install/compose.phtml');
$templateForEnv = new View(__DIR__ . '/../../../../app/views/install/env.phtml');
$templateForCompose
->setParam('httpPort', $httpPort)
@@ -213,16 +224,24 @@ class Install extends Action
->setParam('vars', $input)
;
if (!file_put_contents($path . '/docker-compose.yml', $templateForCompose->render(false))) {
if (!file_put_contents($this->path . '/docker-compose.yml', $templateForCompose->render(false))) {
$message = 'Failed to save Docker Compose file';
$analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
$event = new Event();
$event->setName(APP_VERSION_STABLE . ' - ' . $message)
->addProp('category', 'install/server')
->addProp('action', 'install');
$analytics->createEvent($event);
Console::error($message);
Console::exit(1);
}
if (!file_put_contents($path . '/.env', $templateForEnv->render(false))) {
if (!file_put_contents($this->path . '/.env', $templateForEnv->render(false))) {
$message = 'Failed to save environment variables file';
$analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
$event = new Event();
$event->setName(APP_VERSION_STABLE . ' - ' . $message)
->addProp('category', 'install/server')
->addProp('action', 'install');
$analytics->createEvent($event);
Console::error($message);
Console::exit(1);
}
@@ -237,19 +256,27 @@ class Install extends Action
}
}
Console::log("Running \"docker compose -f {$path}/docker-compose.yml up -d --remove-orphans --renew-anon-volumes\"");
Console::log("Running \"docker compose -f {$this->path}/docker-compose.yml up -d --remove-orphans --renew-anon-volumes\"");
$exit = Console::execute("${env} docker compose -f {$path}/docker-compose.yml up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr);
$exit = Console::execute("${env} docker compose -f {$this->path}/docker-compose.yml up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr);
if ($exit !== 0) {
$message = 'Failed to install Appwrite dockers';
$analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
$event = new Event();
$event->setName(APP_VERSION_STABLE . ' - ' . $message)
->addProp('category', 'install/server')
->addProp('action', 'install');
$analytics->createEvent($event);
Console::error($message);
Console::error($stderr);
Console::exit($exit);
} else {
$message = 'Appwrite installed successfully';
$analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
$event = new Event();
$event->setName(APP_VERSION_STABLE . ' - ' . $message)
->addProp('category', 'install/server')
->addProp('action', 'install');
$analytics->createEvent($event);
Console::success($message);
}
}
src/Appwrite/Platform/Tasks/Upgrade.php
+42
View File
@@ -0,0 +1,42 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\CLI\Console;
use Utopia\Validator\Text;
class Upgrade extends Install
{
public static function getName(): string
{
return 'upgrade';
}
public function __construct()
{
$this
->desc('Upgrade Appwrite')
->param('httpPort', '', new Text(4), 'Server HTTP port', true)
->param('httpsPort', '', new Text(4), 'Server HTTPS port', true)
->param('organization', 'appwrite', new Text(0), 'Docker Registry organization', true)
->param('image', 'appwrite', new Text(0), 'Main appwrite docker image', true)
->param('interactive', 'Y', new Text(1), 'Run an interactive session', true)
->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive));
}
public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive): void
{
// Check for previous installation
$data = @file_get_contents($this->path . '/docker-compose.yml');
if (empty($data)) {
Console::error('Appwrite installation not found.');
Console::log('The command was not run in the parent folder of your appwrite installation.');
Console::log('Please navigate to the parent directory of the Appwrite installation and try again.');
Console::log(' parent_directory <= you run the command in this directory');
Console::log(' └── appwrite');
Console::log(' └── docker-compose.yml');
Console::exit(1);
}
parent::action($httpPort, $httpsPort, $organization, $image, $interactive);
}
}