Merge branch 'feat-database-indexing' into feat-quest-filters

This commit is contained in:
Matej Baco committed 2021-12-31 16:48:52 +01:00
commit b703943b26
37 files changed
+683 -116

No files matched your search

+84 -13
View File
@@ -42,6 +42,7 @@ class V11 extends Migration
$this->options = array_map(fn ($option) => $option === 'yes' ? true : false, $this->options);
if (!is_null($cache)) {
$this->cache->flushAll();
$cacheAdapter = new Cache(new RedisCache($this->cache));
$this->dbProject = new Database(new MariaDB($this->db), $cacheAdapter); // namespace is set on execution
$this->dbConsole = new Database(new MariaDB($this->db), $cacheAdapter);
@@ -63,6 +64,7 @@ class V11 extends Migration
$oldProject = $this->project;
$this->dbProject->setNamespace('_project_' . $oldProject->getId());
$this->dbConsole->setNamespace('_project_console');
Console::info('');
Console::info('------------------------------------');
@@ -73,7 +75,11 @@ class V11 extends Migration
* Create internal/external structure for projects and skip the console project.
*/
if ($oldProject->getId() !== 'console') {
$project = $this->dbConsole->getDocument('projects', $oldProject->getId());
try {
$project = $this->dbConsole->getDocument(collection: 'projects', id: $oldProject->getId());
} catch (\Throwable $th) {
Console::error($th->getTraceAsString());
}
/**
* Migrate Project Document.
@@ -88,10 +94,10 @@ class V11 extends Migration
/**
* Create internal tables
*/
if (!$this->dbProject->exists('appwrite')) {
$this->dbProject->create('appwrite');
try {
Console::log('Created internal tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')');
}
$this->dbProject->createMetadata();
} catch (\Throwable $th) { }
/**
* Create Audit tables
@@ -195,12 +201,8 @@ class V11 extends Migration
$this->dbProject->createDocument($new->getCollection(), $new);
}
} catch (\Throwable $th) {
Console::error('Failed to update document: ' . $th->getMessage());
Console::error("Failed to migrate document ({$new->getId()}) from collection ({$new->getCollection()}): " . $th->getMessage());
continue;
if ($document && $new->getId() !== $document->getId()) {
throw new Exception('Duplication Error');
}
}
}
@@ -270,6 +272,7 @@ class V11 extends Migration
'dateCreated' => time(),
'dateUpdated' => time(),
'name' => $name,
'enabled' => true,
'search' => implode(' ', [$id, $name]),
]));
} else {
@@ -284,7 +287,7 @@ class V11 extends Migration
foreach ($attributes as $attribute) {
try {
$this->dbProject->createAttribute(
collection: $attribute['$collection'],
collection: 'collection_' . $attribute['$collection'],
id: $attribute['$id'],
type: $attribute['type'],
size: $attribute['size'],
@@ -296,7 +299,6 @@ class V11 extends Migration
formatOptions: $attribute['formatOptions'] ?? [],
filters: $attribute['filters']
);
$this->dbProject->createDocument('attributes', new Document([
'$id' => $attribute['$collection'] . '_' . $attribute['$id'],
'key' => $attribute['$id'],
@@ -391,7 +393,13 @@ class V11 extends Migration
}, $document);
$document = new Document($document->getArrayCopy());
$document = $this->migratePermissions($document);
$this->dbProject->createDocument('collection_' . $collection, $document);
try {
$this->dbProject->createDocument('collection_' . $collection, $document);
} catch (\Throwable $th) {
Console::error("Failed to migrate document ({$document->getId()}): " . $th->getMessage());
continue;
}
}
$offset += $this->limit;
}
@@ -426,7 +434,30 @@ class V11 extends Migration
}
switch ($document->getAttribute('$collection')) {
case OldDatabase::SYSTEM_COLLECTION_PLATFORMS:
case OldDatabase::SYSTEM_COLLECTION_PROJECTS:
$newProviders = [];
$providers = Config::getParam('providers', []);
/*
* Add enabled OAuth2 providers to default data rules
*/
foreach ($providers as $index => $provider) {
$appId = $document->getAttribute('usersOauth2'.\ucfirst($index).'Appid');
$appSecret = $document->getAttribute('usersOauth2'.\ucfirst($index).'Secret');
if (!is_null($appId) || !is_null($appId)) {
$newProviders[$appId] = $appSecret;
}
$document
->removeAttribute('usersOauth2'.\ucfirst($index).'Appid')
->removeAttribute('usersOauth2'.\ucfirst($index).'Secret');
}
$document->setAttribute('providers', $newProviders);
break;
case OldDatabase::SYSTEM_COLLECTION_PLATFORMS:
$projectId = $this->getProjectIdFromReadPermissions($document);
/**
@@ -478,6 +509,29 @@ class V11 extends Migration
break;
case OldDatabase::SYSTEM_COLLECTION_KEYS:
$projectId = $this->getProjectIdFromReadPermissions($document);
/**
* Set Project ID
*/
if ($document->getAttribute('projectId') === null) {
$document->setAttribute('projectId', $projectId);
}
/**
* Set scopes if empty
*/
if (empty($document->getAttribute('scopes', []))) {
$document->setAttribute('scopes', []);
}
/**
* Reset Permissions
*/
$document->setAttribute('$read', ['role:all']);
$document->setAttribute('$write', ['role:all']);
break;
case OldDatabase::SYSTEM_COLLECTION_WEBHOOKS:
$projectId = $this->getProjectIdFromReadPermissions($document);
@@ -533,6 +587,19 @@ class V11 extends Migration
$write = $document->getWrite();
$document->setAttribute('$write', str_replace('user:{self}', "user:{$document->getId()}", $write));
break;
case OldDatabase::SYSTEM_COLLECTION_TEAMS:
/**
* Replace team:{self} with team:TEAM_ID
*/
$read = $document->getWrite();
$write = $document->getWrite();
$document->setAttribute('$read', str_replace('team:{self}', "team:{$document->getId()}", $read));
$document->setAttribute('$write', str_replace('team:{self}', "team:{$document->getId()}", $write));
break;
case OldDatabase::SYSTEM_COLLECTION_FILES:
/**
@@ -634,6 +701,10 @@ class V11 extends Migration
$size = $type === Database::VAR_STRING ? 65_535 : 0; // Max size of text in MariaDB
if ($required) {
$default = null;
}
$attributes[$key] = [
'$collection' => $collectionId,
'$id' => $id,
+111 -6
View File
@@ -9,15 +9,66 @@ use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Adapter\MariaDB;
use Exception;
abstract class Worker
{
/**
* Callbacks that will be executed when an error occurs
*
* @var array
*/
static protected array $errorCallbacks = [];
/**
* Associative array holding all information passed into the worker
*
* @return array
*/
public array $args = [];
abstract public function init(): void;
/**
* Function for identifying the worker needs to be set to unique name
*
* @return string
* @throws Exception
*/
public function getName(): string
{
throw new Exception("Please implement getName method in worker");
}
abstract public function run(): void;
/**
* Function executed before running first task.
* Can include any preparations, such as connecting to external services or loading files
*
* @return void
* @throws \Exception|\Throwable
*/
public function init() {
throw new Exception("Please implement getName method in worker");
}
abstract public function shutdown(): void;
/**
* Function executed when new task requests is received.
* You can access $args here, it will contain event information
*
* @return void
* @throws \Exception|\Throwable
*/
public function run() {
throw new Exception("Please implement getName method in worker");
}
/**
* Function executed just before shutting down the worker.
* You can do cleanup here, such as disconnecting from services or removing temp files
*
* @return void
* @throws \Exception|\Throwable
*/
public function shutdown() {
throw new Exception("Please implement getName method in worker");
}
const MAX_ATTEMPTS = 10;
const SLEEP_TIME = 2;
@@ -25,19 +76,73 @@ abstract class Worker
const DATABASE_PROJECT = 'project';
const DATABASE_CONSOLE = 'console';
/**
* A wrapper around 'init' function with non-worker-specific code
*
* @return void
* @throws \Exception|\Throwable
*/
public function setUp(): void
{
$this->init();
try {
$this->init();
} catch(\Throwable $error) {
foreach (self::$errorCallbacks as $errorCallback) {
$errorCallback($error, "init", $this->getName());
}
throw $error;
}
}
/**
* A wrapper around 'run' function with non-worker-specific code
*
* @return void
* @throws \Exception|\Throwable
*/
public function perform(): void
{
$this->run();
try {
$this->run();
} catch(\Throwable $error) {
foreach (self::$errorCallbacks as $errorCallback) {
$errorCallback($error, "run", $this->getName(), $this->args);
}
throw $error;
}
}
/**
* A wrapper around 'shutdown' function with non-worker-specific code
*
* @return void
* @throws \Exception|\Throwable
*/
public function tearDown(): void
{
$this->shutdown();
try {
$this->shutdown();
} catch(\Throwable $error) {
foreach (self::$errorCallbacks as $errorCallback) {
$errorCallback($error, "shutdown", $this->getName());
}
throw $error;
}
}
/**
* Register callback. Will be executed when error occurs.
* @param callable $callback
* @param Throwable $error
* @return self
*/
public static function error(callable $callback): void
{
\array_push(self::$errorCallbacks, $callback);
}
/**
* Get internal project database
@@ -4,6 +4,7 @@ namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use Utopia\Database\Document;
class User extends Model
{
@@ -61,6 +62,24 @@ class User extends Model
;
}
/**
* Get Collection
*
* @return string
*/
public function filter(Document $document): Document
{
$prefs = $document->getAttribute('prefs');
if($prefs instanceof Document) {
$prefs = $prefs->getArrayCopy();
}
if(is_array($prefs) && empty($prefs)) {
$document->setAttribute('prefs', new \stdClass);
}
return $document;
}
/**
* Get Name
*