Merge remote-tracking branch 'origin/1.7.x' into feat-file-tokens

This commit is contained in:
Damodar Lohani committed 2024-11-17 06:00:03 +00:00
commit c555a707e6
5186 files changed
+451784 -3700

No files matched your search

+150 -45
View File
@@ -33,6 +33,7 @@ use Appwrite\Event\Messaging;
use Appwrite\Event\Migration;
use Appwrite\Event\Usage;
use Appwrite\Extend\Exception;
use Appwrite\Functions\Specification;
use Appwrite\GraphQL\Promises\Adapter\Swoole;
use Appwrite\GraphQL\Schema;
use Appwrite\Hooks\Hooks;
@@ -40,6 +41,7 @@ use Appwrite\Network\Validator\Email;
use Appwrite\Network\Validator\Origin;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\URL\URL as AppwriteURL;
use Appwrite\Utopia\Request;
use MaxMind\Db\Reader;
use PHPMailer\PHPMailer\PHPMailer;
use Swoole\Database\PDOProxy;
@@ -62,6 +64,10 @@ use Utopia\Database\Validator\Structure;
use Utopia\Domains\Validator\PublicDomain;
use Utopia\DSN\DSN;
use Utopia\Locale\Locale;
use Utopia\Logger\Adapter\AppSignal;
use Utopia\Logger\Adapter\LogOwl;
use Utopia\Logger\Adapter\Raygun;
use Utopia\Logger\Adapter\Sentry;
use Utopia\Logger\Log;
use Utopia\Logger\Logger;
use Utopia\Pools\Group;
@@ -109,11 +115,12 @@ const APP_LIMIT_SUBSCRIBERS_SUBQUERY = 1_000_000;
const APP_LIMIT_WRITE_RATE_DEFAULT = 60; // Default maximum write rate per rate period
const APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT = 60; // Default maximum write rate period in seconds
const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return in list API calls
const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours
const APP_KEY_ACCESS = 24 * 60 * 60; // 24 hours
const APP_USER_ACCESS = 24 * 60 * 60; // 24 hours
const APP_PROJECT_ACCESS = 24 * 60 * 60; // 24 hours
const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
const APP_CACHE_BUSTER = 443;
const APP_VERSION_STABLE = '1.5.7';
const APP_CACHE_BUSTER = 4318;
const APP_VERSION_STABLE = '1.6.0';
const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
const APP_DATABASE_ATTRIBUTE_IP = 'ip';
@@ -123,6 +130,7 @@ const APP_DATABASE_ATTRIBUTE_INT_RANGE = 'intRange';
const APP_DATABASE_ATTRIBUTE_FLOAT_RANGE = 'floatRange';
const APP_DATABASE_ATTRIBUTE_STRING_MAX_LENGTH = 1_073_741_824; // 2^32 bits / 4 bits per char
const APP_DATABASE_TIMEOUT_MILLISECONDS = 15_000;
const APP_DATABASE_QUERY_MAX_VALUES = 500;
const APP_STORAGE_UPLOADS = '/storage/uploads';
const APP_STORAGE_FUNCTIONS = '/storage/functions';
const APP_STORAGE_BUILDS = '/storage/builds';
@@ -142,9 +150,12 @@ const APP_SOCIAL_DEV = 'https://dev.to/appwrite';
const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite';
const APP_SOCIAL_YOUTUBE = 'https://www.youtube.com/c/appwrite?sub_confirmation=1';
const APP_HOSTNAME_INTERNAL = 'appwrite';
// Databases
const DATABASE_SHARED_TABLES = 'database_db_fra1_self_hosted_16_0';
const APP_FUNCTION_SPECIFICATION_DEFAULT = Specification::S_05VCPU_512MB;
const APP_FUNCTION_CPUS_DEFAULT = 0.5;
const APP_FUNCTION_MEMORY_DEFAULT = 512;
const APP_PLATFORM_SERVER = 'server';
const APP_PLATFORM_CLIENT = 'client';
const APP_PLATFORM_CONSOLE = 'console';
// Database Reconnect
const DATABASE_RECONNECT_SLEEP = 2;
@@ -170,7 +181,7 @@ const DELETE_TYPE_PROJECTS = 'projects';
const DELETE_TYPE_FUNCTIONS = 'functions';
const DELETE_TYPE_DEPLOYMENTS = 'deployments';
const DELETE_TYPE_USERS = 'users';
const DELETE_TYPE_TEAMS = 'teams';
const DELETE_TYPE_TEAM_PROJECTS = 'teams_projects';
const DELETE_TYPE_EXECUTIONS = 'executions';
const DELETE_TYPE_AUDIT = 'audit';
const DELETE_TYPE_ABUSE = 'abuse';
@@ -211,18 +222,34 @@ const FUNCTION_ALLOWLIST_HEADERS_RESPONSE = ['content-type', 'content-length'];
const MESSAGE_TYPE_EMAIL = 'email';
const MESSAGE_TYPE_SMS = 'sms';
const MESSAGE_TYPE_PUSH = 'push';
// API key types
const API_KEY_STANDARD = 'standard';
const API_KEY_DYNAMIC = 'dynamic';
// Usage metrics
const METRIC_TEAMS = 'teams';
const METRIC_USERS = 'users';
const METRIC_MESSAGES = 'messages';
const METRIC_MESSAGES_COUNTRY_CODE = '{countryCode}.messages';
const METRIC_AUTH_METHOD_PHONE = 'auth.method.phone';
const METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE = METRIC_AUTH_METHOD_PHONE . '.{countryCode}';
const METRIC_MESSAGES = 'messages';
const METRIC_MESSAGES_SENT = METRIC_MESSAGES . '.sent';
const METRIC_MESSAGES_FAILED = METRIC_MESSAGES . '.failed';
const METRIC_MESSAGES_TYPE = METRIC_MESSAGES . '.{type}';
const METRIC_MESSAGES_TYPE_SENT = METRIC_MESSAGES . '.{type}.sent';
const METRIC_MESSAGES_TYPE_FAILED = METRIC_MESSAGES . '.{type}.failed';
const METRIC_MESSAGES_TYPE_PROVIDER = METRIC_MESSAGES . '.{type}.{provider}';
const METRIC_MESSAGES_TYPE_PROVIDER_SENT = METRIC_MESSAGES . '.{type}.{provider}.sent';
const METRIC_MESSAGES_TYPE_PROVIDER_FAILED = METRIC_MESSAGES . '.{type}.{provider}.failed';
const METRIC_SESSIONS = 'sessions';
const METRIC_DATABASES = 'databases';
const METRIC_COLLECTIONS = 'collections';
const METRIC_DATABASES_STORAGE = 'databases.storage';
const METRIC_DATABASE_ID_COLLECTIONS = '{databaseInternalId}.collections';
const METRIC_DATABASE_ID_STORAGE = '{databaseInternalId}.databases.storage';
const METRIC_DOCUMENTS = 'documents';
const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents';
const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents';
const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE = '{databaseInternalId}.{collectionInternalId}.databases.storage';
const METRIC_BUCKETS = 'buckets';
const METRIC_FILES = 'files';
const METRIC_FILES_STORAGE = 'files.storage';
@@ -232,17 +259,29 @@ const METRIC_FUNCTIONS = 'functions';
const METRIC_DEPLOYMENTS = 'deployments';
const METRIC_DEPLOYMENTS_STORAGE = 'deployments.storage';
const METRIC_BUILDS = 'builds';
const METRIC_BUILDS_SUCCESS = 'builds.success';
const METRIC_BUILDS_FAILED = 'builds.failed';
const METRIC_BUILDS_STORAGE = 'builds.storage';
const METRIC_BUILDS_COMPUTE = 'builds.compute';
const METRIC_BUILDS_COMPUTE_SUCCESS = 'builds.compute.success';
const METRIC_BUILDS_COMPUTE_FAILED = 'builds.compute.failed';
const METRIC_BUILDS_MB_SECONDS = 'builds.mbSeconds';
const METRIC_FUNCTION_ID_BUILDS = '{functionInternalId}.builds';
const METRIC_FUNCTION_ID_BUILDS_SUCCESS = '{functionInternalId}.builds.success';
const METRIC_FUNCTION_ID_BUILDS_FAILED = '{functionInternalId}.builds.failed';
const METRIC_FUNCTION_ID_BUILDS_STORAGE = '{functionInternalId}.builds.storage';
const METRIC_FUNCTION_ID_BUILDS_COMPUTE = '{functionInternalId}.builds.compute';
const METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS = '{functionInternalId}.builds.compute.success';
const METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED = '{functionInternalId}.builds.compute.failed';
const METRIC_FUNCTION_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments';
const METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage';
const METRIC_FUNCTION_ID_BUILDS_MB_SECONDS = '{functionInternalId}.builds.mbSeconds';
const METRIC_EXECUTIONS = 'executions';
const METRIC_EXECUTIONS_COMPUTE = 'executions.compute';
const METRIC_EXECUTIONS_MB_SECONDS = 'executions.mbSeconds';
const METRIC_FUNCTION_ID_EXECUTIONS = '{functionInternalId}.executions';
const METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE = '{functionInternalId}.executions.compute';
const METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS = '{functionInternalId}.executions.mbSeconds';
const METRIC_NETWORK_REQUESTS = 'network.requests';
const METRIC_NETWORK_INBOUND = 'network.inbound';
const METRIC_NETWORK_OUTBOUND = 'network.outbound';
@@ -290,6 +329,8 @@ Config::load('storage-logos', __DIR__ . '/config/storage/logos.php');
Config::load('storage-mimes', __DIR__ . '/config/storage/mimes.php');
Config::load('storage-inputs', __DIR__ . '/config/storage/inputs.php');
Config::load('storage-outputs', __DIR__ . '/config/storage/outputs.php');
Config::load('runtime-specifications', __DIR__ . '/config/runtimes/specifications.php');
Config::load('function-templates', __DIR__ . '/config/function-templates.php');
/**
* New DB Filters
@@ -344,8 +385,7 @@ Database::addFilter(
if (isset($formatOptions['min']) || isset($formatOptions['max'])) {
$attribute
->setAttribute('min', $formatOptions['min'])
->setAttribute('max', $formatOptions['max'])
;
->setAttribute('max', $formatOptions['max']);
}
return $value;
@@ -610,9 +650,9 @@ Database::addFilter(
])
));
if (\count($targetIds) > 0) {
return $database->find('targets', [
return $database->skipValidation(fn () => $database->find('targets', [
Query::equal('$internalId', $targetIds)
]);
]));
}
return [];
}
@@ -728,6 +768,27 @@ $register->set('logger', function () {
$providerName = System::getEnv('_APP_LOGGING_PROVIDER', '');
$providerConfig = System::getEnv('_APP_LOGGING_CONFIG', '');
try {
$loggingProvider = new DSN($providerConfig ?? '');
$providerName = $loggingProvider->getScheme();
$providerConfig = match ($providerName) {
'sentry' => ['key' => $loggingProvider->getPassword(), 'projectId' => $loggingProvider->getUser() ?? '', 'host' => 'https://' . $loggingProvider->getHost()],
'logowl' => ['ticket' => $loggingProvider->getUser() ?? '', 'host' => $loggingProvider->getHost()],
default => ['key' => $loggingProvider->getHost()],
};
} catch (Throwable $th) {
// Fallback for older Appwrite versions up to 1.5.x that use _APP_LOGGING_PROVIDER and _APP_LOGGING_CONFIG environment variables
Console::warning('Using deprecated logging configuration. Please update your configuration to use DSN format.' . $th->getMessage());
$configChunks = \explode(";", $providerConfig);
$providerConfig = match ($providerName) {
'sentry' => [ 'key' => $configChunks[0], 'projectId' => $configChunks[1] ?? '', 'host' => '',],
'logowl' => ['ticket' => $configChunks[0] ?? '', 'host' => ''],
default => ['key' => $providerConfig],
};
}
if (empty($providerName) || empty($providerConfig)) {
return;
}
@@ -736,20 +797,26 @@ $register->set('logger', function () {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Logging provider not supported. Logging is disabled");
}
// Old Sentry Format conversion. Fallback until the old syntax is completely deprecated.
if (str_contains($providerConfig, ';') && strtolower($providerName) == 'sentry') {
$configChunks = \explode(";", $providerConfig);
try {
$adapter = match ($providerName) {
'sentry' => new Sentry($providerConfig['projectId'], $providerConfig['key'], $providerConfig['host']),
'logowl' => new LogOwl($providerConfig['ticket'], $providerConfig['host']),
'raygun' => new Raygun($providerConfig['key']),
'appsignal' => new AppSignal($providerConfig['key']),
default => null
};
} catch (Throwable $th) {
$adapter = null;
}
$sentryKey = $configChunks[0];
$projectId = $configChunks[1];
$providerConfig = 'https://' . $sentryKey . '@sentry.io/' . $projectId;
if ($adapter === null) {
Console::error("Logging provider not supported. Logging is disabled");
return;
}
$classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName);
$adapter = new $classname($providerConfig);
return new Logger($adapter);
});
$register->set('pools', function () {
$group = new Group();
@@ -969,7 +1036,7 @@ $register->set('smtp', function () {
return $mail;
});
$register->set('geodb', function () {
return new Reader(__DIR__ . '/assets/dbip/dbip-country-lite-2024-02.mmdb');
return new Reader(__DIR__ . '/assets/dbip/dbip-country-lite-2024-09.mmdb');
});
$register->set('passwordsDictionary', function () {
$content = \file_get_contents(__DIR__ . '/assets/security/10k-common-passwords');
@@ -1236,18 +1303,18 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons
$user = new Document([]);
}
if (APP_MODE_ADMIN === $mode) {
if ($user->find('teamInternalId', $project->getAttribute('teamInternalId'), 'memberships')) {
Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users.
} else {
$user = new Document([]);
}
}
// if (APP_MODE_ADMIN === $mode) {
// if ($user->find('teamInternalId', $project->getAttribute('teamInternalId'), 'memberships')) {
// Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users.
// } else {
// $user = new Document([]);
// }
// }
$authJWT = $request->getHeader('x-appwrite-jwt', '');
if (!empty($authJWT) && !$project->isEmpty()) { // JWT authentication
$jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 900, 10); // Instantiate with key, algo, maxAge and leeway.
$jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0);
try {
$payload = $jwt->decode($authJWT);
@@ -1256,14 +1323,15 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons
}
$jwtUserId = $payload['userId'] ?? '';
$jwtSessionId = $payload['sessionId'] ?? '';
if ($jwtUserId && $jwtSessionId) {
if (!empty($jwtUserId)) {
$user = $dbForProject->getDocument('users', $jwtUserId);
}
if (empty($user->find('$id', $jwtSessionId, 'sessions'))) { // Match JWT to active token
$user = new Document([]);
$jwtSessionId = $payload['sessionId'] ?? '';
if (!empty($jwtSessionId)) {
if (empty($user->find('$id', $jwtSessionId, 'sessions'))) { // Match JWT to active token
$user = new Document([]);
}
}
}
@@ -1318,7 +1386,7 @@ App::setResource('console', function () {
'$collection' => ID::custom('projects'),
'description' => 'Appwrite core engine',
'logo' => '',
'teamId' => -1,
'teamId' => null,
'webhooks' => [],
'keys' => [],
'platforms' => [
@@ -1336,9 +1404,11 @@ App::setResource('console', function () {
'legalAddress' => '',
'legalTaxId' => '',
'auths' => [
'mockNumbers' => [],
'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled',
'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user
'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds
'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled'
],
'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [],
'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [],
@@ -1372,7 +1442,8 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole,
$database
->setMetadata('host', \gethostname())
->setMetadata('project', $project->getId())
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
try {
$dsn = new DSN($project->getAttribute('database'));
@@ -1381,7 +1452,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole,
$dsn = new DSN('mysql://' . $project->getAttribute('database'));
}
if ($dsn->getHost() === DATABASE_SHARED_TABLES) {
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
@@ -1408,7 +1479,8 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) {
->setNamespace('_console')
->setMetadata('host', \gethostname())
->setMetadata('project', 'console')
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
return $database;
}, ['pools', 'cache']);
@@ -1432,9 +1504,10 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
$database
->setMetadata('host', \gethostname())
->setMetadata('project', $project->getId())
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
if ($dsn->getHost() === DATABASE_SHARED_TABLES) {
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
@@ -1497,9 +1570,9 @@ App::setResource('deviceForBuilds', function ($project) {
return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId());
}, ['project']);
function getDevice($root): Device
function getDevice(string $root, string $connection = ''): Device
{
$connection = System::getEnv('_APP_CONNECTIONS_STORAGE', '');
$connection = !empty($connection) ? $connection : System::getEnv('_APP_CONNECTIONS_STORAGE', '');
if (!empty($connection)) {
$acl = 'private';
@@ -1744,3 +1817,35 @@ App::setResource('requestTimestamp', function ($request) {
App::setResource('plan', function (array $plan = []) {
return [];
});
App::setResource('team', function (Document $project, Database $dbForConsole, App $utopia, Request $request) {
$teamInternalId = '';
if ($project->getId() !== 'console') {
$teamInternalId = $project->getAttribute('teamInternalId', '');
} else {
$route = $utopia->match($request);
$path = $route->getPath();
if (str_starts_with($path, '/v1/projects/:projectId')) {
$uri = $request->getURI();
$pid = explode('/', $uri)[3];
$p = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $pid));
$teamInternalId = $p->getAttribute('teamInternalId', '');
} elseif ($path === '/v1/projects') {
$teamId = $request->getParam('teamId', '');
$team = Authorization::skip(fn () => $dbForConsole->getDocument('teams', $teamId));
return $team;
}
}
$team = Authorization::skip(function () use ($dbForConsole, $teamInternalId) {
return $dbForConsole->findOne('teams', [
Query::equal('$internalId', [$teamInternalId]),
]);
});
if (!$team) {
$team = new Document([]);
}
return $team;
}, ['project', 'dbForConsole', 'utopia', 'request']);