From d716afac231069c0b6cb9c080b7877718ac8d11f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 28 Jan 2025 05:03:47 +0000 Subject: [PATCH] refactor init config --- app/cli.php | 207 +------- app/init/app.php | 1167 +++++++++++++++++++++++++++++++++++++++++++ app/init/cli.php | 224 +++++++++ app/init/http.php | 732 +++++++++++++++++++++++++++ app/init/worker.php | 364 ++++++++++++++ app/worker.php | 349 +------------ 6 files changed, 2489 insertions(+), 554 deletions(-) create mode 100644 app/init/app.php create mode 100644 app/init/cli.php create mode 100644 app/init/http.php create mode 100644 app/init/worker.php diff --git a/app/cli.php b/app/cli.php index 47f4525f0b..985e6bc806 100644 --- a/app/cli.php +++ b/app/cli.php @@ -1,215 +1,10 @@ getAll(supported: false)); - -// require controllers after overwriting runtimes -require_once __DIR__ . '/controllers/general.php'; - -Authorization::disable(); - -CLI::setResource('register', fn () => $register); - -CLI::setResource('cache', function ($pools) { - $list = Config::getParam('pools-cache', []); - $adapters = []; - - foreach ($list as $value) { - $adapters[] = $pools - ->get($value) - ->pop() - ->getResource() - ; - } - - return new Cache(new Sharding($adapters)); -}, ['pools']); - -CLI::setResource('pools', function (Registry $register) { - return $register->get('pools'); -}, ['register']); - -CLI::setResource('dbForPlatform', function ($pools, $cache) { - $sleep = 3; - $maxAttempts = 5; - $attempts = 0; - $ready = false; - - do { - $attempts++; - try { - // Prepare database connection - $dbAdapter = $pools - ->get('console') - ->pop() - ->getResource(); - - $dbForPlatform = new Database($dbAdapter, $cache); - - $dbForPlatform - ->setNamespace('_console') - ->setMetadata('host', \gethostname()) - ->setMetadata('project', 'console'); - - // Ensure tables exist - $collections = Config::getParam('collections', [])['console']; - $last = \array_key_last($collections); - - if (!($dbForPlatform->exists($dbForPlatform->getDatabase(), $last))) { /** TODO cache ready variable using registry */ - throw new Exception('Tables not ready yet.'); - } - - $ready = true; - } catch (\Throwable $err) { - Console::warning($err->getMessage()); - $pools->get('console')->reclaim(); - sleep($sleep); - } - } while ($attempts < $maxAttempts && !$ready); - - if (!$ready) { - throw new Exception("Console is not ready yet. Please try again later."); - } - - return $dbForPlatform; -}, ['pools', 'cache']); - -CLI::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { - $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - - return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases) { - if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; - } - - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - if (isset($databases[$dsn->getHost()])) { - $database = $databases[$dsn->getHost()]; - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - $database - ->setSharedTables(true) - ->setTenant($project->getInternalId()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } - - return $database; - } - - $dbAdapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); - $databases[$dsn->getHost()] = $database; - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - $database - ->setSharedTables(true) - ->setTenant($project->getInternalId()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } - - $database - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()); - - return $database; - }; -}, ['pools', 'dbForPlatform', 'cache']); - -CLI::setResource('queue', function (Group $pools) { - return $pools->get('queue')->pop()->getResource(); -}, ['pools']); -CLI::setResource('queueForFunctions', function (Connection $queue) { - return new Func($queue); -}, ['queue']); -CLI::setResource('queueForDeletes', function (Connection $queue) { - return new Delete($queue); -}, ['queue']); -CLI::setResource('queueForCertificates', function (Connection $queue) { - return new Certificate($queue); -}, ['queue']); -CLI::setResource('logError', function (Registry $register) { - return function (Throwable $error, string $namespace, string $action) use ($register) { - $logger = $register->get('logger'); - - if ($logger) { - $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); - - $log = new Log(); - $log->setNamespace($namespace); - $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); - $log->setVersion($version); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($error->getMessage()); - - $log->addTag('code', $error->getCode()); - $log->addTag('verboseType', get_class($error)); - - $log->addExtra('file', $error->getFile()); - $log->addExtra('line', $error->getLine()); - $log->addExtra('trace', $error->getTraceAsString()); - - $log->setAction($action); - - $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; - $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - - try { - $responseCode = $logger->addLog($log); - Console::info('Error log pushed with status code: ' . $responseCode); - } catch (Throwable $th) { - Console::error('Error pushing log: ' . $th->getMessage()); - } - } - - Console::warning("Failed: {$error->getMessage()}"); - Console::warning($error->getTraceAsString()); - }; -}, ['register']); $platform = new Appwrite(); $platform->init(Service::TYPE_TASK); diff --git a/app/init/app.php b/app/init/app.php new file mode 100644 index 0000000000..8b5d37369f --- /dev/null +++ b/app/init/app.php @@ -0,0 +1,1167 @@ + [ + 'method' => 'GET', + 'user_agent' => \sprintf( + APP_USERAGENT, + System::getEnv('_APP_VERSION', 'UNKNOWN'), + System::getEnv('_APP_EMAIL_SECURITY', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY)) + ), + 'timeout' => 2, + ], +]); + +$register = new Registry(); + +App::setMode(System::getEnv('_APP_ENV', App::MODE_TYPE_PRODUCTION)); + +/* + * ENV vars + */ +Config::load('events', __DIR__ . '/config/events.php'); +Config::load('auth', __DIR__ . '/config/auth.php'); +Config::load('apis', __DIR__ . '/config/apis.php'); // List of APIs +Config::load('errors', __DIR__ . '/config/errors.php'); +Config::load('oAuthProviders', __DIR__ . '/config/oAuthProviders.php'); +Config::load('platforms', __DIR__ . '/config/platforms.php'); +Config::load('collections', __DIR__ . '/config/collections.php'); +Config::load('runtimes', __DIR__ . '/config/runtimes.php'); +Config::load('runtimes-v2', __DIR__ . '/config/runtimes-v2.php'); +Config::load('usage', __DIR__ . '/config/usage.php'); +Config::load('roles', __DIR__ . '/config/roles.php'); // User roles and scopes +Config::load('scopes', __DIR__ . '/config/scopes.php'); // User roles and scopes +Config::load('services', __DIR__ . '/config/services.php'); // List of services +Config::load('variables', __DIR__ . '/config/variables.php'); // List of env variables +Config::load('regions', __DIR__ . '/config/regions.php'); // List of available regions +Config::load('avatar-browsers', __DIR__ . '/config/avatars/browsers.php'); +Config::load('avatar-credit-cards', __DIR__ . '/config/avatars/credit-cards.php'); +Config::load('avatar-flags', __DIR__ . '/config/avatars/flags.php'); +Config::load('locale-codes', __DIR__ . '/config/locale/codes.php'); +Config::load('locale-currencies', __DIR__ . '/config/locale/currencies.php'); +Config::load('locale-eu', __DIR__ . '/config/locale/eu.php'); +Config::load('locale-languages', __DIR__ . '/config/locale/languages.php'); +Config::load('locale-phones', __DIR__ . '/config/locale/phones.php'); +Config::load('locale-countries', __DIR__ . '/config/locale/countries.php'); +Config::load('locale-continents', __DIR__ . '/config/locale/continents.php'); +Config::load('locale-templates', __DIR__ . '/config/locale/templates.php'); +Config::load('storage-logos', __DIR__ . '/config/storage/logos.php'); +Config::load('storage-mimes', __DIR__ . '/config/storage/mimes.php'); +Config::load('storage-inputs', __DIR__ . '/config/storage/inputs.php'); +Config::load('storage-outputs', __DIR__ . '/config/storage/outputs.php'); +Config::load('runtime-specifications', __DIR__ . '/config/runtimes/specifications.php'); +Config::load('function-templates', __DIR__ . '/config/function-templates.php'); + +/** + * New DB Filters + */ +Database::addFilter( + 'casting', + function (mixed $value) { + return json_encode(['value' => $value], JSON_PRESERVE_ZERO_FRACTION); + }, + function (mixed $value) { + if (is_null($value)) { + return; + } + + return json_decode($value, true)['value']; + } +); + +Database::addFilter( + 'enum', + function (mixed $value, Document $attribute) { + if ($attribute->isSet('elements')) { + $attribute->removeAttribute('elements'); + } + + return $value; + }, + function (mixed $value, Document $attribute) { + $formatOptions = \json_decode($attribute->getAttribute('formatOptions', '[]'), true); + if (isset($formatOptions['elements'])) { + $attribute->setAttribute('elements', $formatOptions['elements']); + } + + return $value; + } +); + +Database::addFilter( + 'range', + function (mixed $value, Document $attribute) { + if ($attribute->isSet('min')) { + $attribute->removeAttribute('min'); + } + if ($attribute->isSet('max')) { + $attribute->removeAttribute('max'); + } + + return $value; + }, + function (mixed $value, Document $attribute) { + $formatOptions = json_decode($attribute->getAttribute('formatOptions', '[]'), true); + if (isset($formatOptions['min']) || isset($formatOptions['max'])) { + $attribute + ->setAttribute('min', $formatOptions['min']) + ->setAttribute('max', $formatOptions['max']); + } + + return $value; + } +); + +Database::addFilter( + 'subQueryAttributes', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + $attributes = $database->find('attributes', [ + Query::equal('collectionInternalId', [$document->getInternalId()]), + Query::equal('databaseInternalId', [$document->getAttribute('databaseInternalId')]), + Query::limit($database->getLimitForAttributes()), + ]); + + foreach ($attributes as $attribute) { + if ($attribute->getAttribute('type') === Database::VAR_RELATIONSHIP) { + $options = $attribute->getAttribute('options'); + foreach ($options as $key => $value) { + $attribute->setAttribute($key, $value); + } + $attribute->removeAttribute('options'); + } + } + + return $attributes; + } +); + +Database::addFilter( + 'subQueryIndexes', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return $database + ->find('indexes', [ + Query::equal('collectionInternalId', [$document->getInternalId()]), + Query::equal('databaseInternalId', [$document->getAttribute('databaseInternalId')]), + Query::limit($database->getLimitForIndexes()), + ]); + } +); + +Database::addFilter( + 'subQueryPlatforms', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return $database + ->find('platforms', [ + Query::equal('projectInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ]); + } +); + +Database::addFilter( + 'subQueryKeys', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return $database + ->find('keys', [ + Query::equal('projectInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ]); + } +); + +Database::addFilter( + 'subQueryWebhooks', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return $database + ->find('webhooks', [ + Query::equal('projectInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ]); + } +); + +Database::addFilter( + 'subQuerySessions', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return Authorization::skip(fn () => $database->find('sessions', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ])); + } +); + +Database::addFilter( + 'subQueryTokens', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return Authorization::skip(fn () => $database + ->find('tokens', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ])); + } +); + +Database::addFilter( + 'subQueryChallenges', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return Authorization::skip(fn () => $database + ->find('challenges', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ])); + } +); + +Database::addFilter( + 'subQueryAuthenticators', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return Authorization::skip(fn () => $database + ->find('authenticators', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ])); + } +); + +Database::addFilter( + 'subQueryMemberships', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return Authorization::skip(fn () => $database + ->find('memberships', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY), + ])); + } +); + +Database::addFilter( + 'subQueryVariables', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return $database + ->find('variables', [ + Query::equal('resourceInternalId', [$document->getInternalId()]), + Query::equal('resourceType', ['function']), + Query::limit(APP_LIMIT_SUBQUERY), + ]); + } +); + +Database::addFilter( + 'encrypt', + function (mixed $value) { + $key = System::getEnv('_APP_OPENSSL_KEY_V1'); + $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); + $tag = null; + + return json_encode([ + 'data' => OpenSSL::encrypt($value, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag), + 'method' => OpenSSL::CIPHER_AES_128_GCM, + 'iv' => \bin2hex($iv), + 'tag' => \bin2hex($tag ?? ''), + 'version' => '1', + ]); + }, + function (mixed $value) { + if (is_null($value)) { + return; + } + $value = json_decode($value, true); + $key = System::getEnv('_APP_OPENSSL_KEY_V' . $value['version']); + + return OpenSSL::decrypt($value['data'], $value['method'], $key, 0, hex2bin($value['iv']), hex2bin($value['tag'])); + } +); + +Database::addFilter( + 'subQueryProjectVariables', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return $database + ->find('variables', [ + Query::equal('resourceType', ['project']), + Query::limit(APP_LIMIT_SUBQUERY) + ]); + } +); + +Database::addFilter( + 'userSearch', + function (mixed $value, Document $user) { + $searchValues = [ + $user->getId(), + $user->getAttribute('email', ''), + $user->getAttribute('name', ''), + $user->getAttribute('phone', '') + ]; + + foreach ($user->getAttribute('labels', []) as $label) { + $searchValues[] = 'label:' . $label; + } + + $search = implode(' ', \array_filter($searchValues)); + + return $search; + }, + function (mixed $value) { + return $value; + } +); + +Database::addFilter( + 'subQueryTargets', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + return Authorization::skip(fn () => $database + ->find('targets', [ + Query::equal('userInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBQUERY) + ])); + } +); + +Database::addFilter( + 'subQueryTopicTargets', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + $targetIds = Authorization::skip(fn () => \array_map( + fn ($document) => $document->getAttribute('targetInternalId'), + $database->find('subscribers', [ + Query::equal('topicInternalId', [$document->getInternalId()]), + Query::limit(APP_LIMIT_SUBSCRIBERS_SUBQUERY) + ]) + )); + if (\count($targetIds) > 0) { + return $database->skipValidation(fn () => $database->find('targets', [ + Query::equal('$internalId', $targetIds) + ])); + } + return []; + } +); + +Database::addFilter( + 'providerSearch', + function (mixed $value, Document $provider) { + $searchValues = [ + $provider->getId(), + $provider->getAttribute('name', ''), + $provider->getAttribute('provider', ''), + $provider->getAttribute('type', '') + ]; + + $search = \implode(' ', \array_filter($searchValues)); + + return $search; + }, + function (mixed $value) { + return $value; + } +); + +Database::addFilter( + 'topicSearch', + function (mixed $value, Document $topic) { + $searchValues = [ + $topic->getId(), + $topic->getAttribute('name', ''), + $topic->getAttribute('description', ''), + ]; + + $search = \implode(' ', \array_filter($searchValues)); + + return $search; + }, + function (mixed $value) { + return $value; + } +); + +Database::addFilter( + 'messageSearch', + function (mixed $value, Document $message) { + $searchValues = [ + $message->getId(), + $message->getAttribute('description', ''), + $message->getAttribute('status', ''), + ]; + + $data = \json_decode($message->getAttribute('data', []), true); + $providerType = $message->getAttribute('providerType', ''); + + if ($providerType === MESSAGE_TYPE_EMAIL) { + $searchValues = \array_merge($searchValues, [$data['subject'], MESSAGE_TYPE_EMAIL]); + } elseif ($providerType === MESSAGE_TYPE_SMS) { + $searchValues = \array_merge($searchValues, [$data['content'], MESSAGE_TYPE_SMS]); + } else { + $searchValues = \array_merge($searchValues, [$data['title'], MESSAGE_TYPE_PUSH]); + } + + $search = \implode(' ', \array_filter($searchValues)); + + return $search; + }, + function (mixed $value) { + return $value; + } +); + +/** + * DB Formats + */ +Structure::addFormat(APP_DATABASE_ATTRIBUTE_EMAIL, function () { + return new Email(); +}, Database::VAR_STRING); + +Structure::addFormat(APP_DATABASE_ATTRIBUTE_DATETIME, function () { + return new DatetimeValidator(); +}, Database::VAR_DATETIME); + +Structure::addFormat(APP_DATABASE_ATTRIBUTE_ENUM, function ($attribute) { + $elements = $attribute['formatOptions']['elements']; + return new WhiteList($elements, true); +}, Database::VAR_STRING); + +Structure::addFormat(APP_DATABASE_ATTRIBUTE_IP, function () { + return new IP(); +}, Database::VAR_STRING); + +Structure::addFormat(APP_DATABASE_ATTRIBUTE_URL, function () { + return new URL(); +}, Database::VAR_STRING); + +Structure::addFormat(APP_DATABASE_ATTRIBUTE_INT_RANGE, function ($attribute) { + $min = $attribute['formatOptions']['min'] ?? -INF; + $max = $attribute['formatOptions']['max'] ?? INF; + return new Range($min, $max, Range::TYPE_INTEGER); +}, Database::VAR_INTEGER); + +Structure::addFormat(APP_DATABASE_ATTRIBUTE_FLOAT_RANGE, function ($attribute) { + $min = $attribute['formatOptions']['min'] ?? -INF; + $max = $attribute['formatOptions']['max'] ?? INF; + return new Range($min, $max, Range::TYPE_FLOAT); +}, Database::VAR_FLOAT); + +/* + * Registry + */ +$register->set('logger', function () { + // Register error logger + $providerName = System::getEnv('_APP_LOGGING_PROVIDER', ''); + $providerConfig = System::getEnv('_APP_LOGGING_CONFIG', ''); + + try { + $loggingProvider = new DSN($providerConfig ?? ''); + + $providerName = $loggingProvider->getScheme(); + $providerConfig = match ($providerName) { + 'sentry' => ['key' => $loggingProvider->getPassword(), 'projectId' => $loggingProvider->getUser() ?? '', 'host' => 'https://' . $loggingProvider->getHost()], + 'logowl' => ['ticket' => $loggingProvider->getUser() ?? '', 'host' => $loggingProvider->getHost()], + default => ['key' => $loggingProvider->getHost()], + }; + } catch (Throwable $th) { + // Fallback for older Appwrite versions up to 1.5.x that use _APP_LOGGING_PROVIDER and _APP_LOGGING_CONFIG environment variables + Console::warning('Using deprecated logging configuration. Please update your configuration to use DSN format.' . $th->getMessage()); + $configChunks = \explode(";", $providerConfig); + + $providerConfig = match ($providerName) { + 'sentry' => [ 'key' => $configChunks[0], 'projectId' => $configChunks[1] ?? '', 'host' => '',], + 'logowl' => ['ticket' => $configChunks[0] ?? '', 'host' => ''], + default => ['key' => $providerConfig], + }; + } + + if (empty($providerName) || empty($providerConfig)) { + return; + } + + if (!Logger::hasProvider($providerName)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Logging provider not supported. Logging is disabled"); + } + + try { + $adapter = match ($providerName) { + 'sentry' => new Sentry($providerConfig['projectId'], $providerConfig['key'], $providerConfig['host']), + 'logowl' => new LogOwl($providerConfig['ticket'], $providerConfig['host']), + 'raygun' => new Raygun($providerConfig['key']), + 'appsignal' => new AppSignal($providerConfig['key']), + default => null + }; + } catch (Throwable $th) { + $adapter = null; + } + + if ($adapter === null) { + Console::error("Logging provider not supported. Logging is disabled"); + return; + } + + return new Logger($adapter); +}); + +$register->set('pools', function () { + $group = new Group(); + + $fallbackForDB = 'db_main=' . AppwriteURL::unparse([ + 'scheme' => 'mariadb', + 'host' => System::getEnv('_APP_DB_HOST', 'mariadb'), + 'port' => System::getEnv('_APP_DB_PORT', '3306'), + 'user' => System::getEnv('_APP_DB_USER', ''), + 'pass' => System::getEnv('_APP_DB_PASS', ''), + 'path' => System::getEnv('_APP_DB_SCHEMA', ''), + ]); + $fallbackForRedis = 'redis_main=' . AppwriteURL::unparse([ + 'scheme' => 'redis', + 'host' => System::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => System::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => System::getEnv('_APP_REDIS_USER', ''), + 'pass' => System::getEnv('_APP_REDIS_PASS', ''), + ]); + + $connections = [ + 'console' => [ + 'type' => 'database', + 'dsns' => $fallbackForDB, + 'multiple' => false, + 'schemes' => ['mariadb', 'mysql'], + ], + 'database' => [ + 'type' => 'database', + 'dsns' => $fallbackForDB, + 'multiple' => true, + 'schemes' => ['mariadb', 'mysql'], + ], + 'queue' => [ + 'type' => 'queue', + 'dsns' => $fallbackForRedis, + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'pubsub' => [ + 'type' => 'pubsub', + 'dsns' => $fallbackForRedis, + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'cache' => [ + 'type' => 'cache', + 'dsns' => $fallbackForRedis, + 'multiple' => true, + 'schemes' => ['redis'], + ], + ]; + + $maxConnections = System::getEnv('_APP_CONNECTIONS_MAX', 151); + $instanceConnections = $maxConnections / System::getEnv('_APP_POOL_CLIENTS', 14); + + $multiprocessing = System::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; + + if ($multiprocessing) { + $workerCount = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); + } else { + $workerCount = 1; + } + + if ($workerCount > $instanceConnections) { + throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); + } + + $poolSize = (int)($instanceConnections / $workerCount); + + foreach ($connections as $key => $connection) { + $type = $connection['type'] ?? ''; + $multiple = $connection['multiple'] ?? false; + $schemes = $connection['schemes'] ?? []; + $config = []; + $dsns = explode(',', $connection['dsns'] ?? ''); + foreach ($dsns as &$dsn) { + $dsn = explode('=', $dsn); + $name = ($multiple) ? $key . '_' . $dsn[0] : $key; + $dsn = $dsn[1] ?? ''; + $config[] = $name; + if (empty($dsn)) { + //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); + continue; + } + + $dsn = new DSN($dsn); + $dsnHost = $dsn->getHost(); + $dsnPort = $dsn->getPort(); + $dsnUser = $dsn->getUser(); + $dsnPass = $dsn->getPassword(); + $dsnScheme = $dsn->getScheme(); + $dsnDatabase = $dsn->getPath(); + + if (!in_array($dsnScheme, $schemes)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); + } + + /** + * Get Resource + * + * Creation could be reused across connection types like database, cache, queue, etc. + * + * Resource assignment to an adapter will happen below. + */ + $resource = match ($dsnScheme) { + 'mysql', + 'mariadb' => function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + }); + }, + 'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) { + $redis = new \Redis(); + @$redis->pconnect($dsnHost, (int)$dsnPort); + if ($dsnPass) { + $redis->auth($dsnPass); + } + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); + + return $redis; + }, + default => throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Invalid scheme'), + }; + + $pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) { + // Get Adapter + switch ($type) { + case 'database': + $adapter = match ($dsn->getScheme()) { + 'mariadb' => new MariaDB($resource()), + 'mysql' => new MySQL($resource()), + default => null + }; + + $adapter->setDatabase($dsn->getPath()); + break; + case 'pubsub': + $adapter = match ($dsn->getScheme()) { + 'redis' => new PubSub($resource()), + default => null + }; + break; + case 'queue': + $adapter = match ($dsn->getScheme()) { + 'redis' => new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()), + default => null + }; + break; + case 'cache': + $adapter = match ($dsn->getScheme()) { + 'redis' => new RedisCache($resource()), + default => null + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); + } + + return $adapter; + }); + + $group->add($pool); + } + + Config::setParam('pools-' . $key, $config); + } + + return $group; +}); + +$register->set('db', function () { + // This is usually for our workers or CLI commands scope + $dbHost = System::getEnv('_APP_DB_HOST', ''); + $dbPort = System::getEnv('_APP_DB_PORT', ''); + $dbUser = System::getEnv('_APP_DB_USER', ''); + $dbPass = System::getEnv('_APP_DB_PASS', ''); + $dbScheme = System::getEnv('_APP_DB_SCHEMA', ''); + + return new PDO( + "mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", + $dbUser, + $dbPass, + SQL::getPDOAttributes() + ); +}); + +$register->set('smtp', function () { + $mail = new PHPMailer(true); + + $mail->isSMTP(); + + $username = System::getEnv('_APP_SMTP_USERNAME'); + $password = System::getEnv('_APP_SMTP_PASSWORD'); + + $mail->XMailer = 'Appwrite Mailer'; + $mail->Host = System::getEnv('_APP_SMTP_HOST', 'smtp'); + $mail->Port = System::getEnv('_APP_SMTP_PORT', 25); + $mail->SMTPAuth = !empty($username) && !empty($password); + $mail->Username = $username; + $mail->Password = $password; + $mail->SMTPSecure = System::getEnv('_APP_SMTP_SECURE', ''); + $mail->SMTPAutoTLS = false; + $mail->CharSet = 'UTF-8'; + + $from = \urldecode(System::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server')); + $email = System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM); + + $mail->setFrom($email, $from); + $mail->addReplyTo($email, $from); + + $mail->isHTML(true); + + return $mail; +}); +$register->set('geodb', function () { + return new Reader(__DIR__ . '/assets/dbip/dbip-country-lite-2024-09.mmdb'); +}); +$register->set('passwordsDictionary', function () { + $content = \file_get_contents(__DIR__ . '/assets/security/10k-common-passwords'); + $content = explode("\n", $content); + $content = array_flip($content); + return $content; +}); +$register->set('promiseAdapter', function () { + return new Swoole(); +}); +$register->set('hooks', function () { + return new Hooks(); +}); +/* + * Localization + */ +Locale::$exceptions = false; + +$locales = Config::getParam('locale-codes', []); + +foreach ($locales as $locale) { + $code = $locale['code']; + + $path = __DIR__ . '/config/locale/translations/' . $code . '.json'; + + if (!\file_exists($path)) { + $path = __DIR__ . '/config/locale/translations/' . \substr($code, 0, 2) . '.json'; // if `ar-ae` doesn't exist, look for `ar` + if (!\file_exists($path)) { + $path = __DIR__ . '/config/locale/translations/en.json'; // if none translation exists, use default from `en.json` + } + } + + Locale::setLanguageFromJSON($code, $path); +} + +function getDevice(string $root, string $connection = ''): Device +{ + $connection = !empty($connection) ? $connection : System::getEnv('_APP_CONNECTIONS_STORAGE', ''); + + if (!empty($connection)) { + $acl = 'private'; + $device = Storage::DEVICE_LOCAL; + $accessKey = ''; + $accessSecret = ''; + $bucket = ''; + $region = ''; + + try { + $dsn = new DSN($connection); + $device = $dsn->getScheme(); + $accessKey = $dsn->getUser() ?? ''; + $accessSecret = $dsn->getPassword() ?? ''; + $bucket = $dsn->getPath() ?? ''; + $region = $dsn->getParam('region'); + } catch (\Throwable $e) { + Console::warning($e->getMessage() . 'Invalid DSN. Defaulting to Local device.'); + } + + switch ($device) { + case Storage::DEVICE_S3: + return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case STORAGE::DEVICE_DO_SPACES: + $device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); + $device->setHttpVersion(S3::HTTP_VERSION_1_1); + return $device; + case Storage::DEVICE_BACKBLAZE: + return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case Storage::DEVICE_LINODE: + return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case Storage::DEVICE_WASABI: + return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl); + case Storage::DEVICE_LOCAL: + default: + return new Local($root); + } + } else { + switch (strtolower(System::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) ?? '')) { + case Storage::DEVICE_LOCAL: + default: + return new Local($root); + case Storage::DEVICE_S3: + $s3AccessKey = System::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); + $s3SecretKey = System::getEnv('_APP_STORAGE_S3_SECRET', ''); + $s3Region = System::getEnv('_APP_STORAGE_S3_REGION', ''); + $s3Bucket = System::getEnv('_APP_STORAGE_S3_BUCKET', ''); + $s3Acl = 'private'; + return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); + case Storage::DEVICE_DO_SPACES: + $doSpacesAccessKey = System::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', ''); + $doSpacesSecretKey = System::getEnv('_APP_STORAGE_DO_SPACES_SECRET', ''); + $doSpacesRegion = System::getEnv('_APP_STORAGE_DO_SPACES_REGION', ''); + $doSpacesBucket = System::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', ''); + $doSpacesAcl = 'private'; + $device = new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); + $device->setHttpVersion(S3::HTTP_VERSION_1_1); + return $device; + case Storage::DEVICE_BACKBLAZE: + $backblazeAccessKey = System::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); + $backblazeSecretKey = System::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', ''); + $backblazeRegion = System::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); + $backblazeBucket = System::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); + $backblazeAcl = 'private'; + return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); + case Storage::DEVICE_LINODE: + $linodeAccessKey = System::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); + $linodeSecretKey = System::getEnv('_APP_STORAGE_LINODE_SECRET', ''); + $linodeRegion = System::getEnv('_APP_STORAGE_LINODE_REGION', ''); + $linodeBucket = System::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); + $linodeAcl = 'private'; + return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); + case Storage::DEVICE_WASABI: + $wasabiAccessKey = System::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); + $wasabiSecretKey = System::getEnv('_APP_STORAGE_WASABI_SECRET', ''); + $wasabiRegion = System::getEnv('_APP_STORAGE_WASABI_REGION', ''); + $wasabiBucket = System::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); + $wasabiAcl = 'private'; + return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); + } + } +} diff --git a/app/init/cli.php b/app/init/cli.php new file mode 100644 index 0000000000..169c73d1e3 --- /dev/null +++ b/app/init/cli.php @@ -0,0 +1,224 @@ +getAll(supported: false)); + +// require controllers after overwriting runtimes +require_once __DIR__ . '/controllers/general.php'; + +Authorization::disable(); + +CLI::setResource('register', fn () => $register); + +CLI::setResource('cache', function ($pools) { + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +}, ['pools']); + +CLI::setResource('pools', function (Registry $register) { + return $register->get('pools'); +}, ['register']); + +CLI::setResource('dbForPlatform', function ($pools, $cache) { + $sleep = 3; + $maxAttempts = 5; + $attempts = 0; + $ready = false; + + do { + $attempts++; + try { + // Prepare database connection + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource(); + + $dbForPlatform = new Database($dbAdapter, $cache); + + $dbForPlatform + ->setNamespace('_console') + ->setMetadata('host', \gethostname()) + ->setMetadata('project', 'console'); + + // Ensure tables exist + $collections = Config::getParam('collections', [])['console']; + $last = \array_key_last($collections); + + if (!($dbForPlatform->exists($dbForPlatform->getDatabase(), $last))) { /** TODO cache ready variable using registry */ + throw new Exception('Tables not ready yet.'); + } + + $ready = true; + } catch (\Throwable $err) { + Console::warning($err->getMessage()); + $pools->get('console')->reclaim(); + sleep($sleep); + } + } while ($attempts < $maxAttempts && !$ready); + + if (!$ready) { + throw new Exception("Console is not ready yet. Please try again later."); + } + + return $dbForPlatform; +}, ['pools', 'cache']); + +CLI::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { + $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools + + return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForPlatform; + } + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + if (isset($databases[$dsn->getHost()])) { + $database = $databases[$dsn->getHost()]; + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + if (\in_array($dsn->getHost(), $sharedTables)) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + + return $database; + } + + $dbAdapter = $pools + ->get($dsn->getHost()) + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); + $databases[$dsn->getHost()] = $database; + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + if (\in_array($dsn->getHost(), $sharedTables)) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + + $database + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()); + + return $database; + }; +}, ['pools', 'dbForPlatform', 'cache']); + +CLI::setResource('queue', function (Group $pools) { + return $pools->get('queue')->pop()->getResource(); +}, ['pools']); +CLI::setResource('queueForFunctions', function (Connection $queue) { + return new Func($queue); +}, ['queue']); +CLI::setResource('queueForDeletes', function (Connection $queue) { + return new Delete($queue); +}, ['queue']); +CLI::setResource('queueForCertificates', function (Connection $queue) { + return new Certificate($queue); +}, ['queue']); +CLI::setResource('logError', function (Registry $register) { + return function (Throwable $error, string $namespace, string $action) use ($register) { + $logger = $register->get('logger'); + + if ($logger) { + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); + + $log = new Log(); + $log->setNamespace($namespace); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + + $log->addTag('code', $error->getCode()); + $log->addTag('verboseType', get_class($error)); + + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + + $log->setAction($action); + + $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + try { + $responseCode = $logger->addLog($log); + Console::info('Error log pushed with status code: ' . $responseCode); + } catch (Throwable $th) { + Console::error('Error pushing log: ' . $th->getMessage()); + } + } + + Console::warning("Failed: {$error->getMessage()}"); + Console::warning($error->getTraceAsString()); + }; +}, ['register']); diff --git a/app/init/http.php b/app/init/http.php new file mode 100644 index 0000000000..49c7076be7 --- /dev/null +++ b/app/init/http.php @@ -0,0 +1,732 @@ + new Log()); +App::setResource('logger', function ($register) { + return $register->get('logger'); +}, ['register']); + +App::setResource('hooks', function ($register) { + return $register->get('hooks'); +}, ['register']); + +App::setResource('register', fn () => $register); +App::setResource('locale', fn () => new Locale(System::getEnv('_APP_LOCALE', 'en'))); + +App::setResource('localeCodes', function () { + return array_map(fn ($locale) => $locale['code'], Config::getParam('locale-codes', [])); +}); + +// Queues +App::setResource('queue', function (Group $pools) { + return $pools->get('queue')->pop()->getResource(); +}, ['pools']); +App::setResource('queueForMessaging', function (Connection $queue) { + return new Messaging($queue); +}, ['queue']); +App::setResource('queueForMails', function (Connection $queue) { + return new Mail($queue); +}, ['queue']); +App::setResource('queueForBuilds', function (Connection $queue) { + return new Build($queue); +}, ['queue']); +App::setResource('queueForDatabase', function (Connection $queue) { + return new EventDatabase($queue); +}, ['queue']); +App::setResource('queueForDeletes', function (Connection $queue) { + return new Delete($queue); +}, ['queue']); +App::setResource('queueForEvents', function (Connection $queue) { + return new Event($queue); +}, ['queue']); +App::setResource('queueForWebhooks', function (Connection $queue) { + return new Webhook($queue); +}, ['queue']); +App::setResource('queueForRealtime', function () { + return new Realtime(); +}, []); +App::setResource('queueForAudits', function (Connection $queue) { + return new Audit($queue); +}, ['queue']); +App::setResource('queueForFunctions', function (Connection $queue) { + return new Func($queue); +}, ['queue']); +App::setResource('queueForUsage', function (Connection $queue) { + return new Usage($queue); +}, ['queue']); +App::setResource('queueForCertificates', function (Connection $queue) { + return new Certificate($queue); +}, ['queue']); +App::setResource('queueForMigrations', function (Connection $queue) { + return new Migration($queue); +}, ['queue']); +App::setResource('clients', function ($request, $console, $project) { + $console->setAttribute('platforms', [ // Always allow current host + '$collection' => ID::custom('platforms'), + 'name' => 'Current Host', + 'type' => Origin::CLIENT_TYPE_WEB, + 'hostname' => $request->getHostname(), + ], Document::SET_TYPE_APPEND); + + $hostnames = explode(',', System::getEnv('_APP_CONSOLE_HOSTNAMES', '')); + $validator = new Hostname(); + foreach ($hostnames as $hostname) { + $hostname = trim($hostname); + if (!$validator->isValid($hostname)) { + continue; + } + $console->setAttribute('platforms', [ + '$collection' => ID::custom('platforms'), + 'type' => Origin::CLIENT_TYPE_WEB, + 'name' => $hostname, + 'hostname' => $hostname, + ], Document::SET_TYPE_APPEND); + } + + /** + * Get All verified client URLs for both console and current projects + * + Filter for duplicated entries + */ + $clientsConsole = \array_map( + fn ($node) => $node['hostname'], + \array_filter( + $console->getAttribute('platforms', []), + fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB) && !empty($node['hostname'])) + ) + ); + + $clients = $clientsConsole; + $platforms = $project->getAttribute('platforms', []); + + foreach ($platforms as $node) { + if ( + isset($node['type']) && + ($node['type'] === Origin::CLIENT_TYPE_WEB || + $node['type'] === Origin::CLIENT_TYPE_FLUTTER_WEB) && + !empty($node['hostname']) + ) { + $clients[] = $node['hostname']; + } + } + + return \array_unique($clients); +}, ['request', 'console', 'project']); + +App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForPlatform) { + /** @var Appwrite\Utopia\Request $request */ + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Document $project */ + /** @var Utopia\Database\Database $dbForProject */ + /** @var Utopia\Database\Database $dbForPlatform */ + /** @var string $mode */ + + Authorization::setDefaultStatus(true); + + Auth::setCookieName('a_session_' . $project->getId()); + + if (APP_MODE_ADMIN === $mode) { + Auth::setCookieName('a_session_' . $console->getId()); + } + + $session = Auth::decodeSession( + $request->getCookie( + Auth::$cookieName, // Get sessions + $request->getCookie(Auth::$cookieName . '_legacy', '') + ) + ); + + // Get session from header for SSR clients + if (empty($session['id']) && empty($session['secret'])) { + $sessionHeader = $request->getHeader('x-appwrite-session', ''); + + if (!empty($sessionHeader)) { + $session = Auth::decodeSession($sessionHeader); + } + } + + // Get fallback session from old clients (no SameSite support) or clients who block 3rd-party cookies + if ($response) { + $response->addHeader('X-Debug-Fallback', 'false'); + } + + if (empty($session['id']) && empty($session['secret'])) { + if ($response) { + $response->addHeader('X-Debug-Fallback', 'true'); + } + $fallback = $request->getHeader('x-fallback-cookies', ''); + $fallback = \json_decode($fallback, true); + $session = Auth::decodeSession(((isset($fallback[Auth::$cookieName])) ? $fallback[Auth::$cookieName] : '')); + } + + Auth::$unique = $session['id'] ?? ''; + Auth::$secret = $session['secret'] ?? ''; + + if (APP_MODE_ADMIN !== $mode) { + if ($project->isEmpty()) { + $user = new Document([]); + } else { + if ($project->getId() === 'console') { + $user = $dbForPlatform->getDocument('users', Auth::$unique); + } else { + $user = $dbForProject->getDocument('users', Auth::$unique); + } + } + } else { + $user = $dbForPlatform->getDocument('users', Auth::$unique); + } + + if ( + $user->isEmpty() // Check a document has been found in the DB + || !Auth::sessionVerify($user->getAttribute('sessions', []), Auth::$secret) + ) { // Validate user has valid login token + $user = new Document([]); + } + + // if (APP_MODE_ADMIN === $mode) { + // if ($user->find('teamInternalId', $project->getAttribute('teamInternalId'), 'memberships')) { + // Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users. + // } else { + // $user = new Document([]); + // } + // } + + $authJWT = $request->getHeader('x-appwrite-jwt', ''); + + if (!empty($authJWT) && !$project->isEmpty()) { // JWT authentication + $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0); + + try { + $payload = $jwt->decode($authJWT); + } catch (JWTException $error) { + throw new Exception(Exception::USER_JWT_INVALID, 'Failed to verify JWT. ' . $error->getMessage()); + } + + $jwtUserId = $payload['userId'] ?? ''; + if (!empty($jwtUserId)) { + $user = $dbForProject->getDocument('users', $jwtUserId); + } + + $jwtSessionId = $payload['sessionId'] ?? ''; + if (!empty($jwtSessionId)) { + if (empty($user->find('$id', $jwtSessionId, 'sessions'))) { // Match JWT to active token + $user = new Document([]); + } + } + } + + $dbForProject->setMetadata('user', $user->getId()); + $dbForPlatform->setMetadata('user', $user->getId()); + + return $user; +}, ['mode', 'project', 'console', 'request', 'response', 'dbForProject', 'dbForPlatform']); + +App::setResource('project', function ($dbForPlatform, $request, $console) { + /** @var Appwrite\Utopia\Request $request */ + /** @var Utopia\Database\Database $dbForPlatform */ + /** @var Utopia\Database\Document $console */ + + $projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', '')); + + if (empty($projectId) || $projectId === 'console') { + return $console; + } + + $project = Authorization::skip(fn () => $dbForPlatform->getDocument('projects', $projectId)); + + return $project; +}, ['dbForPlatform', 'request', 'console']); + +App::setResource('session', function (Document $user) { + if ($user->isEmpty()) { + return; + } + + $sessions = $user->getAttribute('sessions', []); + $sessionId = Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret); + + if (!$sessionId) { + return; + } + + foreach ($sessions as $session) {/** @var Document $session */ + if ($sessionId === $session->getId()) { + return $session; + } + } + + return; +}, ['user']); + +App::setResource('console', function () { + return new Document([ + '$id' => ID::custom('console'), + '$internalId' => ID::custom('console'), + 'name' => 'Appwrite', + '$collection' => ID::custom('projects'), + 'description' => 'Appwrite core engine', + 'logo' => '', + 'teamId' => null, + 'webhooks' => [], + 'keys' => [], + 'platforms' => [ + [ + '$collection' => ID::custom('platforms'), + 'name' => 'Localhost', + 'type' => Origin::CLIENT_TYPE_WEB, + 'hostname' => 'localhost', + ], // Current host is added on app init + ], + 'legalName' => '', + 'legalCountry' => '', + 'legalState' => '', + 'legalCity' => '', + 'legalAddress' => '', + 'legalTaxId' => '', + 'auths' => [ + 'mockNumbers' => [], + 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', + 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user + 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds + 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' + ], + 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], + 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], + 'oAuthProviders' => [ + 'githubEnabled' => true, + 'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''), + 'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '') + ], + ]); +}, []); + +App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForPlatform; + } + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + $dbAdapter = $pools + ->get($dsn->getHost()) + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); + + $database + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()) + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) + ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); + + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + if (\in_array($dsn->getHost(), $sharedTables)) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + + return $database; +}, ['pools', 'dbForPlatform', 'cache', 'project']); + +App::setResource('dbForPlatform', function (Group $pools, Cache $cache) { + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); + + $database + ->setNamespace('_console') + ->setMetadata('host', \gethostname()) + ->setMetadata('project', 'console') + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) + ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); + + return $database; +}, ['pools', 'cache']); + +App::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { + $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools + + return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForPlatform; + } + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + $configure = (function (Database $database) use ($project, $dsn) { + $database + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()) + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS) + ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); + + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + if (\in_array($dsn->getHost(), $sharedTables)) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + }); + + if (isset($databases[$dsn->getHost()])) { + $database = $databases[$dsn->getHost()]; + $configure($database); + return $database; + } + + $dbAdapter = $pools + ->get($dsn->getHost()) + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); + $databases[$dsn->getHost()] = $database; + $configure($database); + + return $database; + }; +}, ['pools', 'dbForPlatform', 'cache']); + +App::setResource('cache', function (Group $pools) { + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +}, ['pools']); + +App::setResource('redis', function () { + $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); + $port = System::getEnv('_APP_REDIS_PORT', 6379); + $pass = System::getEnv('_APP_REDIS_PASS', ''); + + $redis = new \Redis(); + @$redis->pconnect($host, (int)$port); + if ($pass) { + $redis->auth($pass); + } + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); + + return $redis; +}); + +App::setResource('timelimit', function (\Redis $redis) { + return function (string $key, int $limit, int $time) use ($redis) { + return new TimeLimitRedis($key, $limit, $time, $redis); + }; +}, ['redis']); + +App::setResource('deviceForLocal', function () { + return new Local(); +}); + +App::setResource('deviceForFiles', function ($project) { + return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); +}, ['project']); + +App::setResource('deviceForFunctions', function ($project) { + return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); +}, ['project']); + +App::setResource('deviceForBuilds', function ($project) { + return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); +}, ['project']); + + +App::setResource('mode', function ($request) { + /** @var Appwrite\Utopia\Request $request */ + + /** + * Defines the mode for the request: + * - 'default' => Requests for Client and Server Side + * - 'admin' => Request from the Console on non-console projects + */ + return $request->getParam('mode', $request->getHeader('x-appwrite-mode', APP_MODE_DEFAULT)); +}, ['request']); + +App::setResource('geodb', function ($register) { + /** @var Utopia\Registry\Registry $register */ + return $register->get('geodb'); +}, ['register']); + +App::setResource('passwordsDictionary', function ($register) { + /** @var Utopia\Registry\Registry $register */ + return $register->get('passwordsDictionary'); +}, ['register']); + + +App::setResource('servers', function () { + $platforms = Config::getParam('platforms'); + $server = $platforms[APP_PLATFORM_SERVER]; + + $languages = array_map(function ($language) { + return strtolower($language['name']); + }, $server['sdks']); + + return $languages; +}); + +App::setResource('promiseAdapter', function ($register) { + return $register->get('promiseAdapter'); +}, ['register']); + +App::setResource('schema', function ($utopia, $dbForProject) { + + $complexity = function (int $complexity, array $args) { + $queries = Query::parseQueries($args['queries'] ?? []); + $query = Query::getByType($queries, [Query::TYPE_LIMIT])[0] ?? null; + $limit = $query ? $query->getValue() : APP_LIMIT_LIST_DEFAULT; + + return $complexity * $limit; + }; + + $attributes = function (int $limit, int $offset) use ($dbForProject) { + $attrs = Authorization::skip(fn () => $dbForProject->find('attributes', [ + Query::limit($limit), + Query::offset($offset), + ])); + + return \array_map(function ($attr) { + return $attr->getArrayCopy(); + }, $attrs); + }; + + $urls = [ + 'list' => function (string $databaseId, string $collectionId, array $args) { + return "/v1/databases/$databaseId/collections/$collectionId/documents"; + }, + 'create' => function (string $databaseId, string $collectionId, array $args) { + return "/v1/databases/$databaseId/collections/$collectionId/documents"; + }, + 'read' => function (string $databaseId, string $collectionId, array $args) { + return "/v1/databases/$databaseId/collections/$collectionId/documents/{$args['documentId']}"; + }, + 'update' => function (string $databaseId, string $collectionId, array $args) { + return "/v1/databases/$databaseId/collections/$collectionId/documents/{$args['documentId']}"; + }, + 'delete' => function (string $databaseId, string $collectionId, array $args) { + return "/v1/databases/$databaseId/collections/$collectionId/documents/{$args['documentId']}"; + }, + ]; + + $params = [ + 'list' => function (string $databaseId, string $collectionId, array $args) { + return [ 'queries' => $args['queries']]; + }, + 'create' => function (string $databaseId, string $collectionId, array $args) { + $id = $args['id'] ?? 'unique()'; + $permissions = $args['permissions'] ?? null; + + unset($args['id']); + unset($args['permissions']); + + // Order must be the same as the route params + return [ + 'databaseId' => $databaseId, + 'documentId' => $id, + 'collectionId' => $collectionId, + 'data' => $args, + 'permissions' => $permissions, + ]; + }, + 'update' => function (string $databaseId, string $collectionId, array $args) { + $documentId = $args['id']; + $permissions = $args['permissions'] ?? null; + + unset($args['id']); + unset($args['permissions']); + + // Order must be the same as the route params + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'documentId' => $documentId, + 'data' => $args, + 'permissions' => $permissions, + ]; + }, + ]; + + return Schema::build( + $utopia, + $complexity, + $attributes, + $urls, + $params, + ); +}, ['utopia', 'dbForProject']); + +App::setResource('contributors', function () { + $path = 'app/config/contributors.json'; + $list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : []; + return $list; +}); + +App::setResource('employees', function () { + $path = 'app/config/employees.json'; + $list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : []; + return $list; +}); + +App::setResource('heroes', function () { + $path = 'app/config/heroes.json'; + $list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : []; + return $list; +}); + +App::setResource('gitHub', function (Cache $cache) { + return new VcsGitHub($cache); +}, ['cache']); + +App::setResource('requestTimestamp', function ($request) { + //TODO: Move this to the Request class itself + $timestampHeader = $request->getHeader('x-appwrite-timestamp'); + $requestTimestamp = null; + if (!empty($timestampHeader)) { + try { + $requestTimestamp = new \DateTime($timestampHeader); + } catch (\Throwable $e) { + throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Invalid X-Appwrite-Timestamp header value'); + } + } + return $requestTimestamp; +}, ['request']); + +App::setResource('plan', function (array $plan = []) { + return []; +}); + +App::setResource('team', function (Document $project, Database $dbForPlatform, App $utopia, Request $request) { + $teamInternalId = ''; + if ($project->getId() !== 'console') { + $teamInternalId = $project->getAttribute('teamInternalId', ''); + } else { + $route = $utopia->match($request); + $path = $route->getPath(); + if (str_starts_with($path, '/v1/projects/:projectId')) { + $uri = $request->getURI(); + $pid = explode('/', $uri)[3]; + $p = Authorization::skip(fn () => $dbForPlatform->getDocument('projects', $pid)); + $teamInternalId = $p->getAttribute('teamInternalId', ''); + } elseif ($path === '/v1/projects') { + $teamId = $request->getParam('teamId', ''); + $team = Authorization::skip(fn () => $dbForPlatform->getDocument('teams', $teamId)); + return $team; + } + } + + $team = Authorization::skip(function () use ($dbForPlatform, $teamInternalId) { + return $dbForPlatform->findOne('teams', [ + Query::equal('$internalId', [$teamInternalId]), + ]); + }); + + return $team; +}, ['project', 'dbForPlatform', 'utopia', 'request']); + +App::setResource( + 'isResourceBlocked', + fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false +); + +App::setResource('previewHostname', function (Request $request) { + if (App::isDevelopment()) { + $host = $request->getQuery('appwrite-hostname') ?? ''; + if (!empty($host)) { + return $host; + } + } + + return ''; +}, ['request']); diff --git a/app/init/worker.php b/app/init/worker.php new file mode 100644 index 0000000000..e9e0e3f9c3 --- /dev/null +++ b/app/init/worker.php @@ -0,0 +1,364 @@ + $register); + +Server::setResource('dbForPlatform', function (Cache $cache, Registry $register) { + $pools = $register->get('pools'); + $database = $pools + ->get('console') + ->pop() + ->getResource(); + + $adapter = new Database($database, $cache); + $adapter->setNamespace('_console'); + + return $adapter; +}, ['cache', 'register']); + +Server::setResource('project', function (Message $message, Database $dbForPlatform) { + $payload = $message->getPayload() ?? []; + $project = new Document($payload['project'] ?? []); + + if ($project->getId() === 'console') { + return $project; + } + + return $dbForPlatform->getDocument('projects', $project->getId()); +}, ['message', 'dbForPlatform']); + +Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForPlatform; + } + + $pools = $register->get('pools'); + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + $adapter = $pools + ->get($dsn->getHost()) + ->pop() + ->getResource(); + + $database = new Database($adapter, $cache); + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + if (\in_array($dsn->getHost(), $sharedTables)) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + + return $database; +}, ['cache', 'register', 'message', 'project', 'dbForPlatform']); + +Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { + $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools + + return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases): Database { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForPlatform; + } + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + if (isset($databases[$dsn->getHost()])) { + $database = $databases[$dsn->getHost()]; + + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + if (\in_array($dsn->getHost(), $sharedTables)) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + + return $database; + } + + $dbAdapter = $pools + ->get($dsn->getHost()) + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); + + $databases[$dsn->getHost()] = $database; + + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + if (\in_array($dsn->getHost(), $sharedTables)) { + $database + ->setSharedTables(true) + ->setTenant($project->getInternalId()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getInternalId()); + } + + return $database; + }; +}, ['pools', 'dbForPlatform', 'cache']); + +Server::setResource('abuseRetention', function () { + return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); +}); + +Server::setResource('auditRetention', function () { + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); +}); + +Server::setResource('executionRetention', function () { + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); +}); + +Server::setResource('cache', function (Registry $register) { + $pools = $register->get('pools'); + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +}, ['register']); + +Server::setResource('redis', function () { + $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); + $port = System::getEnv('_APP_REDIS_PORT', 6379); + $pass = System::getEnv('_APP_REDIS_PASS', ''); + + $redis = new \Redis(); + @$redis->pconnect($host, (int)$port); + if ($pass) { + $redis->auth($pass); + } + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); + + return $redis; +}); + +Server::setResource('timelimit', function (\Redis $redis) { + return function (string $key, int $limit, int $time) use ($redis) { + return new TimeLimitRedis($key, $limit, $time, $redis); + }; +}, ['redis']); + +Server::setResource('log', fn () => new Log()); + +Server::setResource('queueForUsage', function (Connection $queue) { + return new Usage($queue); +}, ['queue']); + +Server::setResource('queueForUsageDump', function (Connection $queue) { + return new UsageDump($queue); +}, ['queue']); + +Server::setResource('queue', function (Group $pools) { + return $pools->get('queue')->pop()->getResource(); +}, ['pools']); + +Server::setResource('queueForDatabase', function (Connection $queue) { + return new EventDatabase($queue); +}, ['queue']); + +Server::setResource('queueForMessaging', function (Connection $queue) { + return new Messaging($queue); +}, ['queue']); + +Server::setResource('queueForMails', function (Connection $queue) { + return new Mail($queue); +}, ['queue']); + +Server::setResource('queueForBuilds', function (Connection $queue) { + return new Build($queue); +}, ['queue']); + +Server::setResource('queueForDeletes', function (Connection $queue) { + return new Delete($queue); +}, ['queue']); + +Server::setResource('queueForEvents', function (Connection $queue) { + return new Event($queue); +}, ['queue']); + +Server::setResource('queueForAudits', function (Connection $queue) { + return new Audit($queue); +}, ['queue']); + +Server::setResource('queueForFunctions', function (Connection $queue) { + return new Func($queue); +}, ['queue']); + +Server::setResource('queueForCertificates', function (Connection $queue) { + return new Certificate($queue); +}, ['queue']); + +Server::setResource('queueForMigrations', function (Connection $queue) { + return new Migration($queue); +}, ['queue']); + +Server::setResource('logger', function (Registry $register) { + return $register->get('logger'); +}, ['register']); + +Server::setResource('pools', function (Registry $register) { + return $register->get('pools'); +}, ['register']); + +Server::setResource('deviceForFunctions', function (Document $project) { + return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); +}, ['project']); + +Server::setResource('deviceForFiles', function (Document $project) { + return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); +}, ['project']); + +Server::setResource('deviceForBuilds', function (Document $project) { + return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); +}, ['project']); + +Server::setResource('deviceForCache', function (Document $project) { + return getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); +}, ['project']); + +Server::setResource( + 'isResourceBlocked', + fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false +); + +Server::setResource('certificates', function () { + $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); + if (empty($email)) { + throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.'); + } + + return new LetsEncrypt($email); +}); + +Server::setResource('logError', function (Registry $register, Document $project) { + return function (Throwable $error, string $namespace, string $action, ?array $extras) use ($register, $project) { + $logger = $register->get('logger'); + + if ($logger) { + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); + + $log = new Log(); + $log->setNamespace($namespace); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + + $log->addTag('code', $error->getCode()); + $log->addTag('verboseType', get_class($error)); + $log->addTag('projectId', $project->getId() ?? ''); + + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + + + foreach ($extras as $key => $value) { + $log->addExtra($key, $value); + } + + $log->setAction($action); + + $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + try { + $responseCode = $logger->addLog($log); + Console::info('Error log pushed with status code: ' . $responseCode); + } catch (Throwable $th) { + Console::error('Error pushing log: ' . $th->getMessage()); + } + } + + Console::warning("Failed: {$error->getMessage()}"); + Console::warning($error->getTraceAsString()); + }; +}, ['register', 'project']); diff --git a/app/worker.php b/app/worker.php index 6eb1363e9b..04cd107a2f 100644 --- a/app/worker.php +++ b/app/worker.php @@ -1,364 +1,17 @@ $register); - -Server::setResource('dbForPlatform', function (Cache $cache, Registry $register) { - $pools = $register->get('pools'); - $database = $pools - ->get('console') - ->pop() - ->getResource(); - - $adapter = new Database($database, $cache); - $adapter->setNamespace('_console'); - - return $adapter; -}, ['cache', 'register']); - -Server::setResource('project', function (Message $message, Database $dbForPlatform) { - $payload = $message->getPayload() ?? []; - $project = new Document($payload['project'] ?? []); - - if ($project->getId() === 'console') { - return $project; - } - - return $dbForPlatform->getDocument('projects', $project->getId()); -}, ['message', 'dbForPlatform']); - -Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { - if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; - } - - $pools = $register->get('pools'); - - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - $adapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($adapter, $cache); - - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - $database - ->setSharedTables(true) - ->setTenant($project->getInternalId()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } - - return $database; -}, ['cache', 'register', 'message', 'project', 'dbForPlatform']); - -Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { - $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - - return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases): Database { - if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; - } - - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - if (isset($databases[$dsn->getHost()])) { - $database = $databases[$dsn->getHost()]; - - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - $database - ->setSharedTables(true) - ->setTenant($project->getInternalId()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } - - return $database; - } - - $dbAdapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); - - $databases[$dsn->getHost()] = $database; - - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - $database - ->setSharedTables(true) - ->setTenant($project->getInternalId()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getInternalId()); - } - - return $database; - }; -}, ['pools', 'dbForPlatform', 'cache']); - -Server::setResource('abuseRetention', function () { - return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); -}); - -Server::setResource('auditRetention', function () { - return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); -}); - -Server::setResource('executionRetention', function () { - return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600)); -}); - -Server::setResource('cache', function (Registry $register) { - $pools = $register->get('pools'); - $list = Config::getParam('pools-cache', []); - $adapters = []; - - foreach ($list as $value) { - $adapters[] = $pools - ->get($value) - ->pop() - ->getResource() - ; - } - - return new Cache(new Sharding($adapters)); -}, ['register']); - -Server::setResource('redis', function () { - $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); - $port = System::getEnv('_APP_REDIS_PORT', 6379); - $pass = System::getEnv('_APP_REDIS_PASS', ''); - - $redis = new \Redis(); - @$redis->pconnect($host, (int)$port); - if ($pass) { - $redis->auth($pass); - } - $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); - - return $redis; -}); - -Server::setResource('timelimit', function (\Redis $redis) { - return function (string $key, int $limit, int $time) use ($redis) { - return new TimeLimitRedis($key, $limit, $time, $redis); - }; -}, ['redis']); - -Server::setResource('log', fn () => new Log()); - -Server::setResource('queueForUsage', function (Connection $queue) { - return new Usage($queue); -}, ['queue']); - -Server::setResource('queueForUsageDump', function (Connection $queue) { - return new UsageDump($queue); -}, ['queue']); - -Server::setResource('queue', function (Group $pools) { - return $pools->get('queue')->pop()->getResource(); -}, ['pools']); - -Server::setResource('queueForDatabase', function (Connection $queue) { - return new EventDatabase($queue); -}, ['queue']); - -Server::setResource('queueForMessaging', function (Connection $queue) { - return new Messaging($queue); -}, ['queue']); - -Server::setResource('queueForMails', function (Connection $queue) { - return new Mail($queue); -}, ['queue']); - -Server::setResource('queueForBuilds', function (Connection $queue) { - return new Build($queue); -}, ['queue']); - -Server::setResource('queueForDeletes', function (Connection $queue) { - return new Delete($queue); -}, ['queue']); - -Server::setResource('queueForEvents', function (Connection $queue) { - return new Event($queue); -}, ['queue']); - -Server::setResource('queueForAudits', function (Connection $queue) { - return new Audit($queue); -}, ['queue']); - -Server::setResource('queueForFunctions', function (Connection $queue) { - return new Func($queue); -}, ['queue']); - -Server::setResource('queueForCertificates', function (Connection $queue) { - return new Certificate($queue); -}, ['queue']); - -Server::setResource('queueForMigrations', function (Connection $queue) { - return new Migration($queue); -}, ['queue']); - -Server::setResource('logger', function (Registry $register) { - return $register->get('logger'); -}, ['register']); - -Server::setResource('pools', function (Registry $register) { - return $register->get('pools'); -}, ['register']); - -Server::setResource('deviceForFunctions', function (Document $project) { - return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); -}, ['project']); - -Server::setResource('deviceForFiles', function (Document $project) { - return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); -}, ['project']); - -Server::setResource('deviceForBuilds', function (Document $project) { - return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); -}, ['project']); - -Server::setResource('deviceForCache', function (Document $project) { - return getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); -}, ['project']); - -Server::setResource( - 'isResourceBlocked', - fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false -); - -Server::setResource('certificates', function () { - $email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS')); - if (empty($email)) { - throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue a LetsEncrypt SSL certificate.'); - } - - return new LetsEncrypt($email); -}); - -Server::setResource('logError', function (Registry $register, Document $project) { - return function (Throwable $error, string $namespace, string $action, ?array $extras) use ($register, $project) { - $logger = $register->get('logger'); - - if ($logger) { - $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); - - $log = new Log(); - $log->setNamespace($namespace); - $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); - $log->setVersion($version); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($error->getMessage()); - - $log->addTag('code', $error->getCode()); - $log->addTag('verboseType', get_class($error)); - $log->addTag('projectId', $project->getId() ?? ''); - - $log->addExtra('file', $error->getFile()); - $log->addExtra('line', $error->getLine()); - $log->addExtra('trace', $error->getTraceAsString()); - - - foreach ($extras as $key => $value) { - $log->addExtra($key, $value); - } - - $log->setAction($action); - - $isProduction = System::getEnv('_APP_ENV', 'development') === 'production'; - $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); - - try { - $responseCode = $logger->addLog($log); - Console::info('Error log pushed with status code: ' . $responseCode); - } catch (Throwable $th) { - Console::error('Error pushing log: ' . $th->getMessage()); - } - } - - Console::warning("Failed: {$error->getMessage()}"); - Console::warning($error->getTraceAsString()); - }; -}, ['register', 'project']); - $pools = $register->get('pools'); $platform = new Appwrite(); $args = $platform->getEnv('argv');