Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/1.6.x' into feat-docker-geo

Browse Source
This commit is contained in:
Damodar Lohani
2025-01-07 02:59:27 +00:00
parent c66d963200 3b9cbcb671
commit ad36bb32f2
88 changed files with 4592 additions and 3452 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Appwrite/Auth/OAuth2/Firebase.php
-389
View File
@@ -1,389 +0,0 @@
<?php
namespace Appwrite\Auth\OAuth2;
use Appwrite\Auth\OAuth2;
class Firebase extends OAuth2
{
/**
* @var array
*/
protected array $user = [];
/**
* @var array
*/
protected array $tokens = [];
/**
* @var array
*/
protected array $scopes = [
'https://www.googleapis.com/auth/firebase',
'https://www.googleapis.com/auth/datastore',
'https://www.googleapis.com/auth/cloud-platform',
'https://www.googleapis.com/auth/identitytoolkit',
'https://www.googleapis.com/auth/userinfo.profile'
];
/**
* @var array
*/
protected array $iamPermissions = [
// Database
'datastore.databases.get',
'datastore.databases.list',
'datastore.entities.get',
'datastore.entities.list',
'datastore.indexes.get',
'datastore.indexes.list',
// Generic Firebase permissions
'firebase.projects.get',
// Auth
'firebaseauth.configs.get',
'firebaseauth.configs.getHashConfig',
'firebaseauth.configs.getSecret',
'firebaseauth.users.get',
'identitytoolkit.tenants.get',
'identitytoolkit.tenants.list',
// IAM Assignment
'iam.serviceAccounts.list',
// Storage
'storage.buckets.get',
'storage.buckets.list',
'storage.objects.get',
'storage.objects.list'
];
/**
* @return string
*/
public function getName(): string
{
return 'firebase';
}
/**
* @return string
*/
public function getLoginURL(): string
{
return 'https://accounts.google.com/o/oauth2/v2/auth?' . \http_build_query([
'access_type' => 'offline',
'client_id' => $this->appID,
'redirect_uri' => $this->callback,
'scope' => \implode(' ', $this->getScopes()),
'state' => \json_encode($this->state),
'response_type' => 'code',
'prompt' => 'consent',
]);
}
/**
* @param string $code
*
* @return array
*/
protected function getTokens(string $code): array
{
if (empty($this->tokens)) {
$response = $this->request(
'POST',
'https://oauth2.googleapis.com/token',
[],
\http_build_query([
'client_id' => $this->appID,
'redirect_uri' => $this->callback,
'client_secret' => $this->appSecret,
'code' => $code,
'grant_type' => 'authorization_code'
])
);
$this->tokens = \json_decode($response, true);
}
return $this->tokens;
}
/**
* @param string $refreshToken
*
* @return array
*/
public function refreshTokens(string $refreshToken): array
{
$response = $this->request(
'POST',
'https://oauth2.googleapis.com/token',
[],
\http_build_query([
'client_id' => $this->appID,
'client_secret' => $this->appSecret,
'grant_type' => 'refresh_token',
'refresh_token' => $refreshToken
])
);
$output = [];
\parse_str($response, $output);
$this->tokens = $output;
if (empty($this->tokens['refresh_token'])) {
$this->tokens['refresh_token'] = $refreshToken;
}
return $this->tokens;
}
/**
* @param string $accessToken
*
* @return string
*/
public function getUserID(string $accessToken): string
{
$user = $this->getUser($accessToken);
return $user['id'] ?? '';
}
/**
* @param string $accessToken
*
* @return string
*/
public function getUserEmail(string $accessToken): string
{
$user = $this->getUser($accessToken);
return $user['email'] ?? '';
}
/**
* Check if the OAuth email is verified
*
* @link https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user
*
* @param string $accessToken
*
* @return bool
*/
public function isEmailVerified(string $accessToken): bool
{
$user = $this->getUser($accessToken);
if ($user['verified'] ?? false) {
return true;
}
return false;
}
/**
* @param string $accessToken
*
* @return string
*/
public function getUserName(string $accessToken): string
{
$user = $this->getUser($accessToken);
return $user['name'] ?? '';
}
/**
* @param string $accessToken
*
* @return array
*/
protected function getUser(string $accessToken)
{
if (empty($this->user)) {
$response = $this->request(
'GET',
'https://www.googleapis.com/oauth2/v1/userinfo?access_token=' . \urlencode($accessToken),
[],
);
$this->user = \json_decode($response, true);
}
return $this->user;
}
public function getProjects(string $accessToken): array
{
$projects = $this->request('GET', 'https://firebase.googleapis.com/v1beta1/projects', ['Authorization: Bearer ' . \urlencode($accessToken)]);
$projects = \json_decode($projects, true);
return $projects['results'];
}
/*
Be careful with the setIAMPolicy method, it will overwrite all existing policies
**/
public function assignIAMRole(string $accessToken, string $email, string $projectId, array $role)
{
// Get IAM Roles
$iamRoles = $this->request('POST', 'https://cloudresourcemanager.googleapis.com/v1/projects/' . $projectId . ':getIamPolicy', [
'Authorization: Bearer ' . \urlencode($accessToken),
'Content-Type: application/json'
]);
$iamRoles = \json_decode($iamRoles, true);
$iamRoles['bindings'][] = [
'role' => $role['name'],
'members' => [
'serviceAccount:' . $email
]
];
// Set IAM Roles
$this->request('POST', 'https://cloudresourcemanager.googleapis.com/v1/projects/' . $projectId . ':setIamPolicy', [
'Authorization: Bearer ' . \urlencode($accessToken),
'Content-Type: application/json'
], \json_encode([
'policy' => $iamRoles
]));
}
private function generateRandomString($length = 10): string
{
$characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
$charactersLength = strlen($characters);
$randomString = '';
for ($i = 0; $i < $length; $i++) {
$randomString .= $characters[random_int(0, $charactersLength - 1)];
}
return $randomString;
}
private function createCustomRole(string $accessToken, string $projectId): array
{
// Check if role already exists
try {
$role = $this->request('GET', 'https://iam.googleapis.com/v1/projects/' . $projectId . '/roles/appwriteMigrations', [
'Content-Type: application/json',
'Authorization: Bearer ' . \urlencode($accessToken),
]);
$role = \json_decode($role, true);
return $role;
} catch (\Throwable $e) {
if ($e->getCode() !== 404) {
throw $e;
}
}
// Create role if doesn't exist or isn't correct
$role = $this->request(
'POST',
'https://iam.googleapis.com/v1/projects/' . $projectId . '/roles/',
[
'Content-Type: application/json',
'Authorization: Bearer ' . \urlencode($accessToken),
],
\json_encode(
[
'roleId' => 'appwriteMigrations',
'role' => [
'title' => 'Appwrite Migrations',
'description' => 'A helper role for Appwrite Migrations',
'includedPermissions' => $this->iamPermissions,
'stage' => 'GA'
]
]
)
);
return json_decode($role, true);
}
public function createServiceAccount(string $accessToken, string $projectId): array
{
// Create Service Account
$uid = $this->generateRandomString();
$response = $this->request(
'POST',
'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts',
[
'Authorization: Bearer ' . \urlencode($accessToken),
'Content-Type: application/json'
],
\json_encode([
'accountId' => 'appwrite-' . $uid,
'serviceAccount' => [
'displayName' => 'Appwrite Migrations ' . $uid
]
])
);
$response = json_decode($response, true);
// Create and assign IAM Roles
$role = $this->createCustomRole($accessToken, $projectId);
\sleep(1); // Wait for IAM to propagate changes.
$this->assignIAMRole($accessToken, $response['email'], $projectId, $role);
// Create Service Account Key
$responseKey = $this->request(
'POST',
'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts/' . $response['email'] . '/keys',
[
'Authorization: Bearer ' . \urlencode($accessToken),
'Content-Type: application/json'
]
);
$responseKey = json_decode($responseKey, true);
return json_decode(base64_decode($responseKey['privateKeyData']), true);
}
public function cleanupServiceAccounts(string $accessToken, string $projectId)
{
// List Service Accounts
$response = $this->request(
'GET',
'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts',
[
'Authorization: Bearer ' . \urlencode($accessToken),
'Content-Type: application/json'
]
);
$response = json_decode($response, true);
if (empty($response['accounts'])) {
return false;
}
foreach ($response['accounts'] as $account) {
if (strpos($account['email'], 'appwrite-') !== false) {
$this->request(
'DELETE',
'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts/' . $account['email'],
[
'Authorization: Bearer ' . \urlencode($accessToken),
'Content-Type: application/json'
]
);
}
}
return true;
}
}
src/Appwrite/Certificates/Adapter.php
+14
View File
@@ -0,0 +1,14 @@
<?php
namespace Appwrite\Certificates;
use Utopia\Logger\Log;
interface Adapter
{
public function issueCertificate(string $certName, string $domain): ?string;
public function isRenewRequired(string $domain, Log $log): bool;
public function deleteCertificate(string $domain): void;
}
src/Appwrite/Certificates/LetsEncrypt.php
+125
View File
@@ -0,0 +1,125 @@
<?php
namespace Appwrite\Certificates;
use Exception;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Logger\Log;
class LetsEncrypt implements Adapter
{
private string $email;
public function __construct(string $email)
{
$this->email = $email;
}
public function issueCertificate(string $certName, string $domain): ?string
{
$stdout = '';
$stderr = '';
$staging = (App::isProduction()) ? '' : ' --dry-run';
$exit = Console::execute(
"certbot certonly -v --webroot --noninteractive --agree-tos{$staging}"
. " --email " . $this->email
. " --cert-name " . $certName
. " -w " . APP_STORAGE_CERTIFICATES
. " -d {$domain}",
'',
$stdout,
$stderr
);
// Unexpected error, usually 5XX, API limits, ...
if ($exit !== 0) {
throw new Exception('Failed to issue a certificate with message: ' . $stderr);
}
// Prepare folder in storage for domain
$path = APP_STORAGE_CERTIFICATES . '/' . $domain;
if (!\is_readable($path)) {
if (!\mkdir($path, 0755, true)) {
throw new Exception('Failed to create path for certificate.');
}
}
// Move generated files
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) {
throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) {
throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) {
throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
if (!@\rename('/etc/letsencrypt/live/' . $certName . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) {
throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $stderr . ' ; ' . $stdout);
}
$config = \implode(PHP_EOL, [
"tls:",
" certificates:",
" - certFile: /storage/certificates/{$domain}/fullchain.pem",
" keyFile: /storage/certificates/{$domain}/privkey.pem"
]);
// Save configuration into Traefik using our new cert files
if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) {
throw new Exception('Failed to save Traefik configuration.');
}
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? null;
$dt = (new \DateTime())->setTimestamp($validTo);
return DateTime::addSeconds($dt, -60 * 60 * 24 * 30);
}
public function isRenewRequired(string $domain, Log $log): bool
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
if (\file_exists($certPath)) {
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? 0;
if (empty($validTo)) {
$log->addTag('certificateDomain', $domain);
throw new Exception('Unable to read certificate file (cert.pem).');
}
// LetsEncrypt allows renewal 30 days before expiry
$expiryInAdvance = (60 * 60 * 24 * 30);
if ($validTo - $expiryInAdvance > \time()) {
$log->addTag('certificateDomain', $domain);
$log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
return false;
}
}
return true;
}
public function deleteCertificate(string $domain): void
{
$directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
$checkTraversal = realpath($directory) === $directory;
if ($checkTraversal && is_dir($directory)) {
// Delete files, so Traefik is aware of change
array_map('unlink', glob($directory . '/*.*'));
rmdir($directory);
Console::info("Deleted certificate files for {$domain}");
} else {
Console::info("No certificate files found for {$domain}");
}
}
}
src/Appwrite/Event/Usage.php
+7 -1
View File
@@ -42,6 +42,7 @@ class Usage extends Event
*/
public function addMetric(string $key, int $value): self
{
$this->metrics[] = [
'key' => $key,
'value' => $value,
@@ -62,10 +63,15 @@ class Usage extends Event
}
$client = new Client($this->queue, $this->connection);
return $client->enqueue([
$result = $client->enqueue([
'project' => $this->getProject(),
'reduce' => $this->reduce,
'metrics' => $this->metrics,
]);
$this->metrics = [];
return $result;
}
}
src/Appwrite/Event/Webhook.php
+13
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Event;
use Utopia\Database\Document;
use Utopia\Queue\Connection;
class Webhook extends Event
@@ -14,4 +15,16 @@ class Webhook extends Event
->setQueue(Event::WEBHOOK_QUEUE_NAME)
->setClass(Event::WEBHOOK_CLASS_NAME);
}
public function trigger(): string|bool
{
/** Filter out context and trim project to keep the payload small */
$this->context = [];
$this->project = new Document([
'$id' => $this->project->getId(),
'$internalId' => $this->project->getInternalId(),
]);
return parent::trigger();
}
}
src/Appwrite/Migration/Migration.php
+1
View File
@@ -91,6 +91,7 @@ abstract class Migration
'1.5.10' => 'V20',
'1.5.11' => 'V20',
'1.6.0' => 'V21',
'1.6.1' => 'V21',
];
/**
src/Appwrite/Platform/Tasks/Maintenance.php
+11 -18
View File
@@ -23,13 +23,13 @@ class Maintenance extends Action
{
$this
->desc('Schedules maintenance tasks and publishes them to our queues')
->inject('dbForConsole')
->inject('dbForPlatform')
->inject('queueForCertificates')
->inject('queueForDeletes')
->callback(fn (Database $dbForConsole, Certificate $queueForCertificates, Delete $queueForDeletes) => $this->action($dbForConsole, $queueForCertificates, $queueForDeletes));
->callback(fn (Database $dbForPlatform, Certificate $queueForCertificates, Delete $queueForDeletes) => $this->action($dbForPlatform, $queueForCertificates, $queueForDeletes));
}
public function action(Database $dbForConsole, Certificate $queueForCertificates, Delete $queueForDeletes): void
public function action(Database $dbForPlatform, Certificate $queueForCertificates, Delete $queueForDeletes): void
{
Console::title('Maintenance V1');
Console::success(APP_NAME . ' maintenance process v1 has started');
@@ -41,19 +41,19 @@ class Maintenance extends Action
$cacheRetention = (int) System::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
$schedulesDeletionRetention = (int) System::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForPlatform, $queueForDeletes, $queueForCertificates) {
$time = DateTime::now();
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
$this->foreachProject($dbForConsole, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
$this->foreachProject($dbForPlatform, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
$queueForDeletes->setProject($project);
$this->notifyProjects($queueForDeletes, $usageStatsRetentionHourly);
});
$this->notifyDeleteConnections($queueForDeletes);
$this->renewCertificates($dbForConsole, $queueForCertificates);
$this->renewCertificates($dbForPlatform, $queueForCertificates);
$this->notifyDeleteCache($cacheRetention, $queueForDeletes);
$this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
}, $interval, $delay);
@@ -66,13 +66,12 @@ class Maintenance extends Action
{
$this->notifyDeleteTargets($queueForDeletes);
$this->notifyDeleteExecutionLogs($queueForDeletes);
$this->notifyDeleteAbuseLogs($queueForDeletes);
$this->notifyDeleteAuditLogs($queueForDeletes);
$this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
$this->notifyDeleteExpiredSessions($queueForDeletes);
}
protected function foreachProject(Database $dbForConsole, callable $callback): void
protected function foreachProject(Database $dbForPlatform, callable $callback): void
{
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
$count = 0;
@@ -82,7 +81,7 @@ class Maintenance extends Action
$executionStart = \microtime(true);
while ($sum === $limit) {
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
$projects = $dbForPlatform->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
$chunk++;
@@ -106,13 +105,6 @@ class Maintenance extends Action
->trigger();
}
private function notifyDeleteAbuseLogs(Delete $queueForDeletes): void
{
$queueForDeletes
->setType(DELETE_TYPE_ABUSE)
->trigger();
}
private function notifyDeleteAuditLogs(Delete $queueForDeletes): void
{
$queueForDeletes
@@ -143,12 +135,13 @@ class Maintenance extends Action
->trigger();
}
private function renewCertificates(Database $dbForConsole, Certificate $queueForCertificate): void
private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void
{
$time = DateTime::now();
$certificates = $dbForConsole->find('certificates', [
$certificates = $dbForPlatform->find('certificates', [
Query::lessThan('attempts', 5), // Maximum 5 attempts
Query::isNotNull('renewDate'),
Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew)
Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains)
]);
src/Appwrite/Platform/Tasks/Migrate.php
+10 -10
View File
@@ -30,12 +30,12 @@ class Migrate extends Action
->desc('Migrate Appwrite to new version')
/** @TODO APP_VERSION_STABLE needs to be defined */
->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true)
->inject('dbForConsole')
->inject('dbForPlatform')
->inject('getProjectDB')
->inject('register')
->callback(function ($version, $dbForConsole, $getProjectDB, Registry $register) {
\Co\run(function () use ($version, $dbForConsole, $getProjectDB, $register) {
$this->action($version, $dbForConsole, $getProjectDB, $register);
->callback(function ($version, $dbForPlatform, $getProjectDB, Registry $register) {
\Co\run(function () use ($version, $dbForPlatform, $getProjectDB, $register) {
$this->action($version, $dbForPlatform, $getProjectDB, $register);
});
});
}
@@ -58,7 +58,7 @@ class Migrate extends Action
}
}
public function action(string $version, Database $dbForConsole, callable $getProjectDB, Registry $register)
public function action(string $version, Database $dbForPlatform, callable $getProjectDB, Registry $register)
{
Authorization::disable();
if (!array_key_exists($version, Migration::$versions)) {
@@ -93,10 +93,10 @@ class Migrate extends Action
$count = 0;
try {
$totalProjects = $dbForConsole->count('projects') + 1;
$totalProjects = $dbForPlatform->count('projects') + 1;
} catch (\Throwable $th) {
$dbForConsole->setNamespace('_console');
$totalProjects = $dbForConsole->count('projects') + 1;
$dbForPlatform->setNamespace('_console');
$totalProjects = $dbForPlatform->count('projects') + 1;
}
$class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
@@ -120,7 +120,7 @@ class Migrate extends Action
$projectDB = $getProjectDB($project);
$projectDB->disableValidation();
$migration
->setProject($project, $projectDB, $dbForConsole)
->setProject($project, $projectDB, $dbForPlatform)
->setPDO($register->get('db', true))
->execute();
} catch (\Throwable $th) {
@@ -132,7 +132,7 @@ class Migrate extends Action
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($offset)]);
$projects = $dbForPlatform->find('projects', [Query::limit($limit), Query::offset($offset)]);
$offset = $offset + $limit;
$count = $count + $sum;
src/Appwrite/Platform/Tasks/SDKs.php
+24 -9
View File
@@ -25,6 +25,9 @@ use Appwrite\Spec\Swagger2;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Platform\Action;
use Utopia\Validator\Nullable;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
class SDKs extends Action
{
@@ -37,23 +40,35 @@ class SDKs extends Action
{
$this
->desc('Generate Appwrite SDKs')
->callback(fn () => $this->action());
->param('platform', null, new Nullable(new Text(256)), 'Selected Platform', optional: true)
->param('sdk', null, new Nullable(new Text(256)), 'Selected SDK', optional:true)
->param('version', null, new Nullable(new Text(256)), 'Selected SDK', optional:true)
->param('git', null, new Nullable(new WhiteList(['yes', 'no'])), 'Should we use git push?', optional: true)
->param('production', null, new Nullable(new WhiteList(['yes', 'no'])), 'Should we push to production?', optional:true)
->param('message', null, new Nullable(new Text(256)), 'Commit Message', optional:true)
->callback([$this, 'action']);
}
public function action(): void
public function action(?string $selectedPlatform, ?string $selectedSDK, ?string $version, ?string $git, ?string $production, ?string $message)
{
$platforms = Config::getParam('platforms');
$selectedPlatform = Console::confirm('Choose Platform ("' . APP_PLATFORM_CLIENT . '", "' . APP_PLATFORM_SERVER . '", "' . APP_PLATFORM_CONSOLE . '" or "*" for all):');
$selectedSDK = \strtolower(Console::confirm('Choose SDK ("*" for all):'));
$version = Console::confirm('Choose an Appwrite version');
$git = (Console::confirm('Should we use git push? (yes/no)') == 'yes');
$production = ($git) ? (Console::confirm('Type "Appwrite" to push code to production git repos') == 'Appwrite') : false;
$message = ($git) ? Console::confirm('Please enter your commit message:') : '';
$selectedPlatform ??= Console::confirm('Choose Platform ("' . APP_PLATFORM_CLIENT . '", "' . APP_PLATFORM_SERVER . '", "' . APP_PLATFORM_CONSOLE . '" or "*" for all):');
$selectedSDK ??= \strtolower(Console::confirm('Choose SDK ("*" for all):'));
$version ??= Console::confirm('Choose an Appwrite version');
$git ??= Console::confirm('Should we use git push? (yes/no)');
$git = $git === 'yes';
if ($git) {
$production ??= Console::confirm('Type "Appwrite" to push code to production git repos');
$production = $production === 'Appwrite';
$message ??= Console::confirm('Please enter your commit message:');
}
if (!in_array($version, ['0.6.x', '0.7.x', '0.8.x', '0.9.x', '0.10.x', '0.11.x', '0.12.x', '0.13.x', '0.14.x', '0.15.x', '1.0.x', '1.1.x', '1.2.x', '1.3.x', '1.4.x', '1.5.x', '1.6.x', 'latest'])) {
throw new \Exception('Unknown version given');
}
$platforms = Config::getParam('platforms');
foreach ($platforms as $key => $platform) {
if ($selectedPlatform !== $key && $selectedPlatform !== '*') {
continue;
src/Appwrite/Platform/Tasks/ScheduleBase.php
+24 -12
View File
@@ -9,6 +9,7 @@ use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Action;
use Utopia\Pools\Group;
use Utopia\System\System;
@@ -25,7 +26,7 @@ abstract class ScheduleBase extends Action
abstract public static function getName(): string;
abstract public static function getSupportedResource(): string;
abstract public static function getCollectionId(): string;
abstract protected function enqueueResources(Group $pools, Database $dbForConsole, callable $getProjectDB): void;
abstract protected function enqueueResources(Group $pools, Database $dbForPlatform, callable $getProjectDB): void;
public function __construct()
{
@@ -34,9 +35,20 @@ abstract class ScheduleBase extends Action
$this
->desc("Execute {$type}s scheduled in Appwrite")
->inject('pools')
->inject('dbForConsole')
->inject('dbForPlatform')
->inject('getProjectDB')
->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB));
->callback(fn (Group $pools, Database $dbForPlatform, callable $getProjectDB) => $this->action($pools, $dbForPlatform, $getProjectDB));
}
protected function updateProjectAccess(Document $project, Database $dbForPlatform): void
{
if (!$project->isEmpty() && $project->getId() !== 'console') {
$accessedAt = $project->getAttribute('accessedAt', '');
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) {
$project->setAttribute('accessedAt', DateTime::now());
Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project));
}
}
}
/**
@@ -44,7 +56,7 @@ abstract class ScheduleBase extends Action
* 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute
* 3. Create timer that prepares coroutines for soon-to-execute schedules. When it's ready, coroutine sleeps until exact time before sending request to worker.
*/
public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void
public function action(Group $pools, Database $dbForPlatform, callable $getProjectDB): void
{
Console::title(\ucfirst(static::getSupportedResource()) . ' scheduler V1');
Console::success(APP_NAME . ' ' . \ucfirst(static::getSupportedResource()) . ' scheduler v1 has started');
@@ -56,8 +68,8 @@ abstract class ScheduleBase extends Action
* @throws Exception
* @var Document $schedule
*/
$getSchedule = function (Document $schedule) use ($dbForConsole, $getProjectDB): array {
$project = $dbForConsole->getDocument('projects', $schedule->getAttribute('projectId'));
$getSchedule = function (Document $schedule) use ($dbForPlatform, $getProjectDB): array {
$project = $dbForPlatform->getDocument('projects', $schedule->getAttribute('projectId'));
$resource = $getProjectDB($project)->getDocument(
static::getCollectionId(),
@@ -91,7 +103,7 @@ abstract class ScheduleBase extends Action
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
$results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
$results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [
Query::equal('region', [System::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', [static::getSupportedResource()]),
Query::equal('active', [true]),
@@ -119,11 +131,11 @@ abstract class ScheduleBase extends Action
Console::success("Starting timers at " . DateTime::now());
run(function () use ($dbForConsole, &$lastSyncUpdate, $getSchedule, $pools, $getProjectDB) {
run(function () use ($dbForPlatform, &$lastSyncUpdate, $getSchedule, $pools, $getProjectDB) {
/**
* The timer synchronize $schedules copy with database collection.
*/
Timer::tick(static::UPDATE_TIMER * 1000, function () use ($dbForConsole, &$lastSyncUpdate, $getSchedule, $pools) {
Timer::tick(static::UPDATE_TIMER * 1000, function () use ($dbForPlatform, &$lastSyncUpdate, $getSchedule, $pools) {
$time = DateTime::now();
$timerStart = \microtime(true);
@@ -141,7 +153,7 @@ abstract class ScheduleBase extends Action
$paginationQueries[] = Query::cursorAfter($latestDocument);
}
$results = $dbForConsole->find('schedules', \array_merge($paginationQueries, [
$results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [
Query::equal('region', [System::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', [static::getSupportedResource()]),
Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate),
@@ -179,10 +191,10 @@ abstract class ScheduleBase extends Action
Timer::tick(
static::ENQUEUE_TIMER * 1000,
fn () => $this->enqueueResources($pools, $dbForConsole, $getProjectDB)
fn () => $this->enqueueResources($pools, $dbForPlatform, $getProjectDB)
);
$this->enqueueResources($pools, $dbForConsole, $getProjectDB);
$this->enqueueResources($pools, $dbForPlatform, $getProjectDB);
});
}
}
src/Appwrite/Platform/Tasks/ScheduleExecutions.php
+6 -4
View File
@@ -27,7 +27,7 @@ class ScheduleExecutions extends ScheduleBase
return 'executions';
}
protected function enqueueResources(Group $pools, Database $dbForConsole, callable $getProjectDB): void
protected function enqueueResources(Group $pools, Database $dbForPlatform, callable $getProjectDB): void
{
$queue = $pools->get('queue')->pop();
$connection = $queue->getResource();
@@ -36,7 +36,7 @@ class ScheduleExecutions extends ScheduleBase
foreach ($this->schedules as $schedule) {
if (!$schedule['active']) {
$dbForConsole->deleteDocument(
$dbForPlatform->deleteDocument(
'schedules',
$schedule['$id'],
);
@@ -50,13 +50,15 @@ class ScheduleExecutions extends ScheduleBase
continue;
}
$data = $dbForConsole->getDocument(
$data = $dbForPlatform->getDocument(
'schedules',
$schedule['$id'],
)->getAttribute('data', []);
$delay = $scheduledAt->getTimestamp() - (new \DateTime())->getTimestamp();
$this->updateProjectAccess($schedule['project'], $dbForPlatform);
\go(function () use ($queueForFunctions, $schedule, $delay, $data) {
Co::sleep($delay);
@@ -74,7 +76,7 @@ class ScheduleExecutions extends ScheduleBase
->trigger();
});
$dbForConsole->deleteDocument(
$dbForPlatform->deleteDocument(
'schedules',
$schedule['$id'],
);
src/Appwrite/Platform/Tasks/ScheduleFunctions.php
+4 -2
View File
@@ -31,7 +31,7 @@ class ScheduleFunctions extends ScheduleBase
return 'functions';
}
protected function enqueueResources(Group $pools, Database $dbForConsole, callable $getProjectDB): void
protected function enqueueResources(Group $pools, Database $dbForPlatform, callable $getProjectDB): void
{
$timerStart = \microtime(true);
$time = DateTime::now();
@@ -70,7 +70,7 @@ class ScheduleFunctions extends ScheduleBase
}
foreach ($delayedExecutions as $delay => $scheduleKeys) {
\go(function () use ($delay, $scheduleKeys, $pools) {
\go(function () use ($delay, $scheduleKeys, $pools, $dbForPlatform) {
\sleep($delay); // in seconds
$queue = $pools->get('queue')->pop();
@@ -84,6 +84,8 @@ class ScheduleFunctions extends ScheduleBase
$schedule = $this->schedules[$scheduleKey];
$this->updateProjectAccess($schedule['project'], $dbForPlatform);
$queueForFunctions = new Func($connection);
$queueForFunctions
src/Appwrite/Platform/Tasks/ScheduleMessages.php
+5 -3
View File
@@ -26,7 +26,7 @@ class ScheduleMessages extends ScheduleBase
return 'messages';
}
protected function enqueueResources(Group $pools, Database $dbForConsole, callable $getProjectDB): void
protected function enqueueResources(Group $pools, Database $dbForPlatform, callable $getProjectDB): void
{
foreach ($this->schedules as $schedule) {
if (!$schedule['active']) {
@@ -40,18 +40,20 @@ class ScheduleMessages extends ScheduleBase
continue;
}
\go(function () use ($schedule, $pools, $dbForConsole) {
\go(function () use ($schedule, $pools, $dbForPlatform) {
$queue = $pools->get('queue')->pop();
$connection = $queue->getResource();
$queueForMessaging = new Messaging($connection);
$this->updateProjectAccess($schedule['project'], $dbForPlatform);
$queueForMessaging
->setType(MESSAGE_SEND_TYPE_EXTERNAL)
->setMessageId($schedule['resourceId'])
->setProject($schedule['project'])
->trigger();
$dbForConsole->deleteDocument(
$dbForPlatform->deleteDocument(
'schedules',
$schedule['$id'],
);
src/Appwrite/Platform/Tasks/Specs.php
+1 -1
View File
@@ -61,7 +61,7 @@ class Specs extends Action
// Mock dependencies
App::setResource('request', fn () => $this->getRequest());
App::setResource('response', fn () => $response);
App::setResource('dbForConsole', fn () => new Database(new MySQL(''), new Cache(new None())));
App::setResource('dbForPlatform', fn () => new Database(new MySQL(''), new Cache(new None())));
App::setResource('dbForProject', fn () => new Database(new MySQL(''), new Cache(new None())));
$platforms = [
src/Appwrite/Platform/Workers/Builds.php
+23 -20
View File
@@ -46,7 +46,7 @@ class Builds extends Action
$this
->desc('Builds worker')
->inject('message')
->inject('dbForConsole')
->inject('dbForPlatform')
->inject('queueForEvents')
->inject('queueForFunctions')
->inject('queueForUsage')
@@ -54,12 +54,12 @@ class Builds extends Action
->inject('dbForProject')
->inject('deviceForFunctions')
->inject('log')
->callback(fn ($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log));
->callback(fn ($message, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log));
}
/**
* @param Message $message
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param Usage $queueForUsage
@@ -70,7 +70,7 @@ class Builds extends Action
* @return void
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void
public function action(Message $message, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void
{
$payload = $message->getPayload() ?? [];
@@ -92,7 +92,7 @@ class Builds extends Action
case BUILD_TYPE_RETRY:
Console::info('Creating build for deployment: ' . $deployment->getId());
$github = new GitHub($cache);
$this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template, $log);
$this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log);
break;
default:
@@ -105,7 +105,7 @@ class Builds extends Action
* @param Func $queueForFunctions
* @param Event $queueForEvents
* @param Usage $queueForUsage
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Database $dbForProject
* @param GitHub $github
* @param Document $project
@@ -117,7 +117,7 @@ class Builds extends Action
* @throws \Utopia\Database\Exception
* @throws Exception
*/
protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void
protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void
{
$executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
@@ -199,7 +199,7 @@ class Builds extends Action
$repositoryName = '';
if ($isVcsEnabled) {
$installation = $dbForConsole->getDocument('installations', $installationId);
$installation = $dbForPlatform->getDocument('installations', $installationId);
$providerInstallationId = $installation->getAttribute('providerInstallationId');
$privateKey = System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY');
$githubAppId = System::getEnv('_APP_VCS_GITHUB_APP_ID');
@@ -209,8 +209,7 @@ class Builds extends Action
try {
if ($isNewBuild && !$isVcsEnabled) {
// Non-vcs+Template
// Non-VCS + Template
$templateRepositoryName = $template->getAttribute('repositoryName', '');
$templateOwnerName = $template->getAttribute('ownerName', '');
$templateVersion = $template->getAttribute('version', '');
@@ -233,6 +232,8 @@ class Builds extends Action
throw new \Exception('Unable to clone code repository: ' . $stderr);
}
Console::execute('find ' . \escapeshellarg($tmpTemplateDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr);
// Ensure directories
Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr);
@@ -398,6 +399,8 @@ class Builds extends Action
throw new \Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / 1048576, 2) . ' MBs.');
}
Console::execute('find ' . \escapeshellarg($tmpDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr);
$tarParamDirectory = '/tmp/builds/' . $buildId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory);
Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax
@@ -415,7 +418,7 @@ class Builds extends Action
$directorySize = $deviceForFunctions->getFileSize($source);
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize));
$this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
$this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform);
}
/** Request the executor to build the code... */
@@ -423,7 +426,7 @@ class Builds extends Action
$build = $dbForProject->updateDocument('builds', $buildId, $build);
if ($isVcsEnabled) {
$this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
$this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform);
}
/** Trigger Webhook */
@@ -637,7 +640,7 @@ class Builds extends Action
$build = $dbForProject->updateDocument('builds', $buildId, $build);
if ($isVcsEnabled) {
$this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
$this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform);
}
Console::success("Build id: $buildId created");
@@ -658,12 +661,12 @@ class Builds extends Action
/** Update function schedule */
// Inform scheduler if function is still active
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule = $dbForPlatform->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
Authorization::skip(fn () => $dbForPlatform->updateDocument('schedules', $schedule->getId(), $schedule));
} catch (\Throwable $th) {
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
@@ -680,7 +683,7 @@ class Builds extends Action
$build = $dbForProject->updateDocument('builds', $buildId, $build);
if ($isVcsEnabled) {
$this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
$this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForPlatform);
}
} finally {
/**
@@ -739,7 +742,7 @@ class Builds extends Action
* @param Document $function
* @param string $deploymentId
* @param Database $dbForProject
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @return void
* @throws Structure
* @throws \Utopia\Database\Exception
@@ -747,7 +750,7 @@ class Builds extends Action
* @throws Conflict
* @throws Restricted
*/
protected function runGitAction(string $status, GitHub $github, string $providerCommitHash, string $owner, string $repositoryName, Document $project, Document $function, string $deploymentId, Database $dbForProject, Database $dbForConsole): void
protected function runGitAction(string $status, GitHub $github, string $providerCommitHash, string $owner, string $repositoryName, Document $project, Document $function, string $deploymentId, Database $dbForProject, Database $dbForPlatform): void
{
if ($function->getAttribute('providerSilentMode', false) === true) {
return;
@@ -792,7 +795,7 @@ class Builds extends Action
$retries++;
try {
$dbForConsole->createDocument('vcsCommentLocks', new Document([
$dbForPlatform->createDocument('vcsCommentLocks', new Document([
'$id' => $commentId
]));
break;
@@ -812,7 +815,7 @@ class Builds extends Action
$comment->addBuild($project, $function, $status, $deployment->getId(), ['type' => 'logs']);
$github->updateComment($owner, $repositoryName, $commentId, $comment->generateComment());
} finally {
$dbForConsole->deleteDocument('vcsCommentLocks', $commentId);
$dbForPlatform->deleteDocument('vcsCommentLocks', $commentId);
}
}
}
src/Appwrite/Platform/Workers/Certificates.php
+57 -197
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Certificates\Adapter as CertificatesAdapter;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Mail;
@@ -11,7 +12,6 @@ use Appwrite\Template\Template;
use Appwrite\Utopia\Response\Model\Rule;
use Exception;
use Throwable;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
@@ -44,28 +44,31 @@ class Certificates extends Action
$this
->desc('Certificates worker')
->inject('message')
->inject('project')
->inject('dbForConsole')
->inject('dbForPlatform')
->inject('queueForMails')
->inject('queueForEvents')
->inject('queueForFunctions')
->inject('log')
->callback(fn (Message $message, Document $project, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $project, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log));
->inject('certificates')
->callback(
fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates) =>
$this->action($message, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates)
);
}
/**
* @param Message $message
* @param Document $project
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Mail $queueForMails
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param Log $log
* @param CertificatesAdapter $certificates
* @return void
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Document $project, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void
public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates): void
{
$payload = $message->getPayload() ?? [];
@@ -79,33 +82,33 @@ class Certificates extends Action
$log->addTag('domain', $domain->get());
$this->execute($domain, $project, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck);
$this->execute($domain, $dbForPlatform, $queueForMails, $queueForEvents, $queueForFunctions, $log, $certificates, $skipRenewCheck);
}
/**
* @param Domain $domain
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Mail $queueForMails
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param CertificatesAdapter $certificates
* @param bool $skipRenewCheck
* @return void
* @throws Throwable
* @throws \Utopia\Database\Exception
*/
private function execute(Domain $domain, Document $project, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void
private function execute(Domain $domain, Database $dbForPlatform, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, CertificatesAdapter $certificates, bool $skipRenewCheck = false): void
{
/**
* 1. Read arguments and validate domain
* 2. Get main domain
* 3. Validate CNAME DNS if parameter is not main domain (meaning it's custom domain)
* 4. Validate security email. Cannot be empty, required by LetsEncrypt
* 5. Validate renew date with certificate file, unless requested to skip by parameter
* 6. Issue a certificate using certbot CLI
* 7. Update 'log' attribute on certificate document with Certbot message
* 8. Create storage folder for certificate, if not ready already
* 9. Move certificates from Certbot location to our Storage
* 10. Create/Update our Storage with new Traefik config with new certificate paths
* 4. Validate renew date with certificate file, unless requested to skip by parameter
* 5. Issue a certificate using certbot CLI
* 6. Update 'log' attribute on certificate document with Certbot message
* 7. Create storage folder for certificate, if not ready already
* 8. Move certificates from Certbot location to our Storage
* 9. Create/Update our Storage with new Traefik config with new certificate paths
* 11. Read certificate file and update 'renewDate' on certificate document
* 12. Update 'issueDate' and 'attempts' on certificate
*
@@ -122,11 +125,11 @@ class Certificates extends Action
* 2. Save document to database
* 3. Update all domains documents with current certificate ID
*
* Note: Renewals are checked and scheduled from maintenence worker
* Note: Renewals are checked and scheduled from maintenance worker
*/
// Get current certificate
$certificate = $dbForConsole->findOne('certificates', [Query::equal('domain', [$domain->get()])]);
$certificate = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain->get()])]);
// If we don't have certificate for domain yet, let's create new document. At the end we save it
if ($certificate->isEmpty()) {
@@ -137,50 +140,27 @@ class Certificates extends Action
$success = false;
try {
// Email for alerts is required by LetsEncrypt
$email = System::getEnv('_APP_EMAIL_CERTIFICATES', System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'));
if (empty($email)) {
throw new Exception('You must set a valid security email address (_APP_EMAIL_CERTIFICATES) to issue an SSL certificate.');
}
// Validate domain and DNS records. Skip if job is forced
if (!$skipRenewCheck) {
$mainDomain = $this->getMainDomain();
$isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain;
$this->validateDomain($domain, $isMainDomain, $log);
// If certificate exists already, double-check expiry date. Skip if job is forced
if (!$certificates->isRenewRequired($domain->get(), $log)) {
throw new Exception('Renew isn\'t required.');
}
}
// If certificate exists already, double-check expiry date. Skip if job is forced
if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) {
throw new Exception('Renew isn\'t required.');
}
// Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate
$folder = ID::unique();
try {
// Generate certificate files using Let's Encrypt
$letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email);
// Give certificates to Traefik
$this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData);
} catch (\Throwable $th) {
Console::error('Failed to generate Lets Encrypt certificate');
}
// Prepare unique cert name. Using this helps prevent miss-match in configuration when renewing certificates.
$certName = ID::unique();
$renewDate = $certificates->issueCertificate($certName, $domain->get());
// Command succeeded, store all data into document
$logs = 'Certificate successfully generated.';
$certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB
try {
// TEMP: add custom hostnames to cloudflare
$this->addCustomHostnameToRegistrar($project, $domain->get());
} catch (\Throwable $th) {
Console::error('Failed to add custom hostname to registrar: ' . $th->getMessage());
}
$certificate->setAttribute('logs', 'Certificate successfully generated.');
// Update certificate info stored in database
$certificate->setAttribute('renewDate', $this->getRenewDate($domain->get()));
$certificate->setAttribute('renewDate', $renewDate);
$certificate->setAttribute('attempts', 0);
$certificate->setAttribute('issueDate', DateTime::now());
$success = true;
@@ -194,7 +174,7 @@ class Certificates extends Action
$attempts = $certificate->getAttribute('attempts', 0) + 1;
$certificate->setAttribute('attempts', $attempts);
// Store cuttent time as renew date to ensure another attempt in next maintenance cycle
// Store current time as renew date to ensure another attempt in next maintenance cycle.
$certificate->setAttribute('renewDate', DateTime::now());
// Send email to security email
@@ -206,7 +186,7 @@ class Certificates extends Action
$certificate->setAttribute('updated', DateTime::now());
// Save all changes we made to certificate document into database
$this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForConsole, $queueForEvents, $queueForFunctions);
$this->saveCertificateDocument($domain->get(), $certificate, $success, $dbForPlatform, $queueForEvents, $queueForFunctions);
}
}
@@ -245,7 +225,7 @@ class Certificates extends Action
* @param string $domain Domain name that certificate is for
* @param Document $certificate Certificate document that we need to save
* @param bool $success
* @param Database $dbForConsole Database connection for console
* @param Database $dbForPlatform Database connection for console
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @return void
@@ -254,21 +234,21 @@ class Certificates extends Action
* @throws Conflict
* @throws Structure
*/
private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void
private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions): void
{
// Check if update or insert required
$certificateDocument = $dbForConsole->findOne('certificates', [Query::equal('domain', [$domain])]);
$certificateDocument = $dbForPlatform->findOne('certificates', [Query::equal('domain', [$domain])]);
if (!$certificateDocument->isEmpty()) {
// Merge new data with current data
$certificate = new Document(\array_merge($certificateDocument->getArrayCopy(), $certificate->getArrayCopy()));
$certificate = $dbForConsole->updateDocument('certificates', $certificate->getId(), $certificate);
$certificate = $dbForPlatform->updateDocument('certificates', $certificate->getId(), $certificate);
} else {
$certificate->removeAttribute('$internalId');
$certificate = $dbForConsole->createDocument('certificates', $certificate);
$certificate = $dbForPlatform->createDocument('certificates', $certificate);
}
$certificateId = $certificate->getId();
$this->updateDomainDocuments($certificateId, $domain, $success, $dbForConsole, $queueForEvents, $queueForFunctions);
$this->updateDomainDocuments($certificateId, $domain, $success, $dbForPlatform, $queueForEvents, $queueForFunctions);
}
/**
@@ -287,8 +267,8 @@ class Certificates extends Action
}
/**
* Internal domain validation functionality to prevent unnecessary attempts failed from Let's Encrypt side. We check:
* - Domain needs to be public and valid (prevents NFT domains that are not supported by Let's Encrypt)
* Internal domain validation functionality to prevent unnecessary attempts. We check:
* - Domain needs to be public and valid (prevents NFT domains that are not supported)
* - Domain must have proper DNS record
*
* @param Domain $domain Domain which we validate
@@ -334,136 +314,6 @@ class Certificates extends Action
}
}
/**
* Reads expiry date of certificate from file and decides if renewal is required or not.
*
* @param string $domain Domain for which we check certificate file
* @return bool True, if certificate needs to be renewed
* @throws Exception
*/
private function isRenewRequired(string $domain, Log $log): bool
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
if (\file_exists($certPath)) {
$validTo = null;
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? 0;
if (empty($validTo)) {
$log->addTag('certificateDomain', $domain);
throw new Exception('Unable to read certificate file (cert.pem).');
}
// LetsEncrypt allows renewal 30 days before expiry
$expiryInAdvance = (60 * 60 * 24 * 30);
if ($validTo - $expiryInAdvance > \time()) {
$log->addTag('certificateDomain', $domain);
$log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
return false;
}
}
return true;
}
/**
* LetsEncrypt communication to issue certificate (using certbot CLI)
*
* @param string $folder Folder into which certificates should be generated
* @param string $domain Domain to generate certificate for
* @return array Named array with keys 'stdout' and 'stderr', both string
* @throws Exception
*/
private function issueCertificate(string $folder, string $domain, string $email): array
{
$stdout = '';
$stderr = '';
$staging = (App::isProduction()) ? '' : ' --dry-run';
$exit = Console::execute("certbot certonly -v --webroot --noninteractive --agree-tos{$staging}"
. " --email " . $email
. " --cert-name " . $folder
. " -w " . APP_STORAGE_CERTIFICATES
. " -d {$domain}", '', $stdout, $stderr);
// Unexpected error, usually 5XX, API limits, ...
if ($exit !== 0) {
throw new Exception('Failed to issue a certificate with message: ' . $stderr);
}
return [
'stdout' => $stdout,
'stderr' => $stderr
];
}
/**
* Read new renew date from certificate file generated by Let's Encrypt
*
* @param string $domain Domain which certificate was generated for
* @return string
* @throws \Utopia\Database\Exception
*/
private function getRenewDate(string $domain): string
{
$certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
$certData = openssl_x509_parse(file_get_contents($certPath));
$validTo = $certData['validTo_time_t'] ?? null;
$dt = (new \DateTime())->setTimestamp($validTo);
return DateTime::addSeconds($dt, -60 * 60 * 24 * 30); // -30 days
}
/**
* Method to take files from Let's Encrypt, and put it into Traefik.
*
* @param string $domain Domain which certificate was generated for
* @param string $folder Folder in which certificates were generated
* @param array $letsEncryptData Let's Encrypt logs to use for additional info when throwing error
* @return void
* @throws Exception
*/
private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void
{
// Prepare folder in storage for domain
$path = APP_STORAGE_CERTIFICATES . '/' . $domain;
if (!\is_readable($path)) {
if (!\mkdir($path, 0755, true)) {
throw new Exception('Failed to create path for certificate.');
}
}
// Move generated files
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/cert.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem')) {
throw new Exception('Failed to rename certificate cert.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/chain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/chain.pem')) {
throw new Exception('Failed to rename certificate chain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/fullchain.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/fullchain.pem')) {
throw new Exception('Failed to rename certificate fullchain.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
if (!@\rename('/etc/letsencrypt/live/' . $folder . '/privkey.pem', APP_STORAGE_CERTIFICATES . '/' . $domain . '/privkey.pem')) {
throw new Exception('Failed to rename certificate privkey.pem. Let\'s Encrypt log: ' . $letsEncryptData['stderr'] . ' ; ' . $letsEncryptData['stdout']);
}
$config = \implode(PHP_EOL, [
"tls:",
" certificates:",
" - certFile: /storage/certificates/{$domain}/fullchain.pem",
" keyFile: /storage/certificates/{$domain}/privkey.pem"
]);
// Save configuration into Traefik using our new cert files
if (!\file_put_contents(APP_STORAGE_CONFIG . '/' . $domain . '.yml', $config)) {
throw new Exception('Failed to save Traefik configuration.');
}
}
/**
* Method to make sure information about error is delivered to admnistrator.
*
@@ -517,15 +367,21 @@ class Certificates extends Action
*
* @return void
*/
private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void
private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions): void
{
$rule = $dbForConsole->getDocument('rules', md5($domain));
// TODO: @christyjacob remove once we migrate the rules in 1.7.x
if (System::getEnv('_APP_RULES_FORMAT') === 'md5') {
$rule = $dbForPlatform->getDocument('rules', md5($domain));
} else {
$rule = $dbForPlatform->findOne('rules', [
Query::equal('domain', [$domain]),
]);
}
if (!$rule->isEmpty()) {
$rule->setAttribute('certificateId', $certificateId);
$rule->setAttribute('status', $success ? 'verified' : 'unverified');
$dbForConsole->updateDocument('rules', $rule->getId(), $rule);
$dbForPlatform->updateDocument('rules', $rule->getId(), $rule);
$projectId = $rule->getAttribute('projectId');
@@ -534,7 +390,11 @@ class Certificates extends Action
return;
}
$project = $dbForConsole->getDocument('projects', $projectId);
$project = $dbForPlatform->getDocument('projects', $projectId);
if ($project->isEmpty()) {
return;
}
/** Trigger Webhook */
$ruleModel = new Rule();
src/Appwrite/Platform/Workers/Databases.php
+47 -38
View File
@@ -11,6 +11,7 @@ use Utopia\Database\Document;
use Utopia\Database\Exception as DatabaseException;
use Utopia\Database\Exception\Authorization;
use Utopia\Database\Exception\Conflict;
use Utopia\Database\Exception\NotFound;
use Utopia\Database\Exception\Restricted;
use Utopia\Database\Exception\Structure;
use Utopia\Database\Query;
@@ -33,21 +34,21 @@ class Databases extends Action
$this
->desc('Databases worker')
->inject('message')
->inject('dbForConsole')
->inject('dbForPlatform')
->inject('dbForProject')
->inject('log')
->callback(fn (Message $message, Database $dbForConsole, Database $dbForProject, Log $log) => $this->action($message, $dbForConsole, $dbForProject, $log));
->callback(fn (Message $message, Database $dbForPlatform, Database $dbForProject, Log $log) => $this->action($message, $dbForPlatform, $dbForProject, $log));
}
/**
* @param Message $message
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Database $dbForProject
* @param Log $log
* @return void
* @throws \Exception
*/
public function action(Message $message, Database $dbForConsole, Database $dbForProject, Log $log): void
public function action(Message $message, Database $dbForPlatform, Database $dbForProject, Log $log): void
{
$payload = $message->getPayload() ?? [];
@@ -73,10 +74,10 @@ class Databases extends Action
match (\strval($type)) {
DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject),
DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject),
DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject),
DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject),
DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject),
DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject),
DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject),
DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject),
DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject),
DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject),
default => throw new \Exception('No database operation for type: ' . \strval($type)),
};
}
@@ -86,7 +87,7 @@ class Databases extends Action
* @param Document $collection
* @param Document $attribute
* @param Document $project
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Database $dbForProject
* @return void
* @throws Authorization
@@ -94,7 +95,7 @@ class Databases extends Action
* @throws \Exception
* @throws \Throwable
*/
private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void
private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject): void
{
if ($collection->isEmpty()) {
throw new Exception('Missing collection');
@@ -133,7 +134,7 @@ class Databases extends Action
$formatOptions = $attribute->getAttribute('formatOptions', []);
$filters = $attribute->getAttribute('filters', []);
$options = $attribute->getAttribute('options', []);
$project = $dbForConsole->getDocument('projects', $projectId);
$project = $dbForPlatform->getDocument('projects', $projectId);
try {
switch ($type) {
@@ -210,7 +211,7 @@ class Databases extends Action
* @param Document $collection
* @param Document $attribute
* @param Document $project
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Database $dbForProject
* @return void
* @throws Authorization
@@ -218,7 +219,7 @@ class Databases extends Action
* @throws \Exception
* @throws \Throwable
**/
private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void
private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForProject): void
{
if ($collection->isEmpty()) {
throw new Exception('Missing collection');
@@ -238,7 +239,7 @@ class Databases extends Action
$key = $attribute->getAttribute('key', '');
$status = $attribute->getAttribute('status', '');
$type = $attribute->getAttribute('type', '');
$project = $dbForConsole->getDocument('projects', $projectId);
$project = $dbForPlatform->getDocument('projects', $projectId);
$options = $attribute->getAttribute('options', []);
$relatedAttribute = new Document();
$relatedCollection = new Document();
@@ -251,23 +252,21 @@ class Databases extends Action
try {
try {
if ($status !== 'failed') {
if ($type === Database::VAR_RELATIONSHIP) {
if ($options['twoWay']) {
$relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']);
if ($relatedCollection->isEmpty()) {
throw new DatabaseException('Collection not found');
}
$relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']);
if ($type === Database::VAR_RELATIONSHIP) {
if ($options['twoWay']) {
$relatedCollection = $dbForProject->getDocument('database_' . $database->getInternalId(), $options['relatedCollection']);
if ($relatedCollection->isEmpty()) {
throw new DatabaseException('Collection not found');
}
if (!$dbForProject->deleteRelationship('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
$dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'stuck'));
throw new DatabaseException('Failed to delete Relationship');
}
} elseif (!$dbForProject->deleteAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
throw new DatabaseException('Failed to delete Attribute');
$relatedAttribute = $dbForProject->getDocument('attributes', $database->getInternalId() . '_' . $relatedCollection->getInternalId() . '_' . $options['twoWayKey']);
}
if (!$dbForProject->deleteRelationship('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
$dbForProject->updateDocument('attributes', $relatedAttribute->getId(), $relatedAttribute->setAttribute('status', 'stuck'));
throw new DatabaseException('Failed to delete Relationship');
}
} elseif (!$dbForProject->deleteAttribute('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
throw new DatabaseException('Failed to delete Attribute');
}
$dbForProject->deleteDocument('attributes', $attribute->getId());
@@ -275,6 +274,18 @@ class Databases extends Action
if (!$relatedAttribute->isEmpty()) {
$dbForProject->deleteDocument('attributes', $relatedAttribute->getId());
}
} catch (NotFound $e) {
Console::error($e->getMessage());
$dbForProject->deleteDocument('attributes', $attribute->getId());
if (!$relatedAttribute->isEmpty()) {
$dbForProject->deleteDocument('attributes', $relatedAttribute->getId());
}
throw $e;
} catch (\Throwable $e) {
Console::error($e->getMessage());
@@ -345,7 +356,7 @@ class Databases extends Action
}
if ($exists) { // Delete the duplicate if created, else update in db
$this->deleteIndex($database, $collection, $index, $project, $dbForConsole, $dbForProject);
$this->deleteIndex($database, $collection, $index, $project, $dbForPlatform, $dbForProject);
} else {
$dbForProject->updateDocument('indexes', $index->getId(), $index);
}
@@ -354,11 +365,9 @@ class Databases extends Action
}
} finally {
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
$dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) {
$dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId());
$dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId());
}
}
}
@@ -368,7 +377,7 @@ class Databases extends Action
* @param Document $collection
* @param Document $index
* @param Document $project
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Database $dbForProject
* @return void
* @throws Authorization
@@ -377,7 +386,7 @@ class Databases extends Action
* @throws DatabaseException
* @throws \Throwable
*/
private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void
private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject): void
{
if ($collection->isEmpty()) {
throw new Exception('Missing collection');
@@ -399,7 +408,7 @@ class Databases extends Action
$attributes = $index->getAttribute('attributes', []);
$lengths = $index->getAttribute('lengths', []);
$orders = $index->getAttribute('orders', []);
$project = $dbForConsole->getDocument('projects', $projectId);
$project = $dbForPlatform->getDocument('projects', $projectId);
try {
if (!$dbForProject->createIndex('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key, $type, $attributes, $lengths, $orders)) {
@@ -429,7 +438,7 @@ class Databases extends Action
* @param Document $collection
* @param Document $index
* @param Document $project
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Database $dbForProject
* @return void
* @throws Authorization
@@ -438,7 +447,7 @@ class Databases extends Action
* @throws DatabaseException
* @throws \Throwable
*/
private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void
private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject): void
{
if ($collection->isEmpty()) {
throw new Exception('Missing collection');
@@ -456,7 +465,7 @@ class Databases extends Action
]);
$key = $index->getAttribute('key');
$status = $index->getAttribute('status', '');
$project = $dbForConsole->getDocument('projects', $projectId);
$project = $dbForPlatform->getDocument('projects', $projectId);
try {
if ($status !== 'failed' && !$dbForProject->deleteIndex('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $key)) {
src/Appwrite/Platform/Workers/Deletes.php
+87 -112
View File
@@ -3,11 +3,10 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Auth\Auth;
use Appwrite\Certificates\Adapter as CertificatesAdapter;
use Appwrite\Extend\Exception;
use Executor\Executor;
use Throwable;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\Database\TimeLimit;
use Utopia\Audit\Audit;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
@@ -44,24 +43,28 @@ class Deletes extends Action
$this
->desc('Deletes worker')
->inject('message')
->inject('dbForConsole')
->inject('dbForPlatform')
->inject('getProjectDB')
->inject('timelimit')
->inject('deviceForFiles')
->inject('deviceForFunctions')
->inject('deviceForBuilds')
->inject('deviceForCache')
->inject('abuseRetention')
->inject('certificates')
->inject('executionRetention')
->inject('auditRetention')
->inject('log')
->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log));
->callback(
fn ($message, $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) =>
$this->action($message, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log)
);
}
/**
* @throws Exception
* @throws Throwable
*/
public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void
public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void
{
$payload = $message->getPayload() ?? [];
@@ -84,10 +87,10 @@ class Deletes extends Action
case DELETE_TYPE_DOCUMENT:
switch ($document->getCollection()) {
case DELETE_TYPE_PROJECTS:
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document);
$this->deleteProject($dbForPlatform, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document);
break;
case DELETE_TYPE_FUNCTIONS:
$this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project);
$this->deleteFunction($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project);
break;
case DELETE_TYPE_DEPLOYMENTS:
$this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project);
@@ -99,10 +102,10 @@ class Deletes extends Action
$this->deleteBucket($getProjectDB, $deviceForFiles, $document, $project);
break;
case DELETE_TYPE_INSTALLATIONS:
$this->deleteInstallation($dbForConsole, $getProjectDB, $document, $project);
$this->deleteInstallation($dbForPlatform, $getProjectDB, $document, $project);
break;
case DELETE_TYPE_RULES:
$this->deleteRule($dbForConsole, $document);
$this->deleteRule($dbForPlatform, $document, $certificates);
break;
default:
Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
@@ -110,7 +113,7 @@ class Deletes extends Action
}
break;
case DELETE_TYPE_TEAM_PROJECTS:
$this->deleteProjectsByTeam($dbForConsole, $getProjectDB, $document);
$this->deleteProjectsByTeam($dbForPlatform, $getProjectDB, $certificates, $document);
break;
case DELETE_TYPE_EXECUTIONS:
$this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
@@ -120,11 +123,8 @@ class Deletes extends Action
$this->deleteAuditLogs($project, $getProjectDB, $auditRetention);
}
break;
case DELETE_TYPE_ABUSE:
$this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention);
break;
case DELETE_TYPE_REALTIME:
$this->deleteRealtimeUsage($dbForConsole, $datetime);
$this->deleteRealtimeUsage($dbForPlatform, $datetime);
break;
case DELETE_TYPE_SESSIONS:
$this->deleteExpiredSessions($project, $getProjectDB);
@@ -139,7 +139,7 @@ class Deletes extends Action
$this->deleteCacheByDate($project, $getProjectDB, $datetime);
break;
case DELETE_TYPE_SCHEDULES:
$this->deleteSchedules($dbForConsole, $getProjectDB, $datetime);
$this->deleteSchedules($dbForPlatform, $getProjectDB, $datetime);
break;
case DELETE_TYPE_TOPIC:
$this->deleteTopic($project, $getProjectDB, $document);
@@ -159,7 +159,7 @@ class Deletes extends Action
}
/**
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param callable $getProjectDB
* @param string $datetime
* @param Document|null $document
@@ -170,7 +170,7 @@ class Deletes extends Action
* @throws Structure
* @throws DatabaseException
*/
private function deleteSchedules(Database $dbForConsole, callable $getProjectDB, string $datetime): void
private function deleteSchedules(Database $dbForPlatform, callable $getProjectDB, string $datetime): void
{
$this->listByGroup(
'schedules',
@@ -179,12 +179,12 @@ class Deletes extends Action
Query::lessThanEqual('resourceUpdatedAt', $datetime),
Query::equal('active', [false]),
],
$dbForConsole,
function (Document $document) use ($dbForConsole, $getProjectDB) {
$project = $dbForConsole->getDocument('projects', $document->getAttribute('projectId'));
$dbForPlatform,
function (Document $document) use ($dbForPlatform, $getProjectDB) {
$project = $dbForPlatform->getDocument('projects', $document->getAttribute('projectId'));
if ($project->isEmpty()) {
$dbForConsole->deleteDocument('schedules', $document->getId());
$dbForPlatform->deleteDocument('schedules', $document->getId());
Console::success('Deleted schedule for deleted project ' . $document->getAttribute('projectId'));
return;
}
@@ -208,7 +208,7 @@ class Deletes extends Action
}
if ($delete) {
$dbForConsole->deleteDocument('schedules', $document->getId());
$dbForPlatform->deleteDocument('schedules', $document->getId());
Console::success('Deleting schedule for ' . $document->getAttribute('resourceType') . ' ' . $document->getAttribute('resourceId'));
}
}
@@ -264,7 +264,7 @@ class Deletes extends Action
MESSAGE_TYPE_EMAIL => 'emailTotal',
MESSAGE_TYPE_SMS => 'smsTotal',
MESSAGE_TYPE_PUSH => 'pushTotal',
default => throw new Exception('Invalid target provider type'),
default => throw new Exception('Invalid target CertificatesAdapter type'),
};
$dbForProject->decreaseDocumentAttribute(
'topics',
@@ -389,7 +389,7 @@ class Deletes extends Action
}
/**
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param callable $getProjectDB
* @param string $hourlyUsageRetentionDatetime
* @return void
@@ -432,7 +432,7 @@ class Deletes extends Action
}
/**
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Document $document
* @return void
* @throws Authorization
@@ -442,10 +442,10 @@ class Deletes extends Action
* @throws Structure
* @throws Exception
*/
private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, Document $document): void
private function deleteProjectsByTeam(Database $dbForPlatform, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void
{
$projects = $dbForConsole->find('projects', [
$projects = $dbForPlatform->find('projects', [
Query::equal('teamInternalId', [$document->getInternalId()])
]);
@@ -455,13 +455,13 @@ class Deletes extends Action
$deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId());
$deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId());
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project);
$dbForConsole->deleteDocument('projects', $project->getId());
$this->deleteProject($dbForPlatform, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $project);
$dbForPlatform->deleteDocument('projects', $project->getId());
}
}
/**
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param callable $getProjectDB
* @param Device $deviceForFiles
* @param Device $deviceForFunctions
@@ -473,7 +473,7 @@ class Deletes extends Action
* @throws Authorization
* @throws DatabaseException
*/
private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void
private function deleteProject(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Document $document): void
{
$projectInternalId = $document->getInternalId();
$projectId = $document->getId();
@@ -489,35 +489,35 @@ class Deletes extends Action
$projectCollectionIds = [
...\array_keys(Config::getParam('collections', [])['projects']),
Audit::COLLECTION,
TimeLimit::COLLECTION,
Audit::COLLECTION
];
$limit = \count($projectCollectionIds) + 25;
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
$sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', ''));
$projectTables = !\in_array($dsn->getHost(), $sharedTables);
$sharedTablesV1 = \in_array($dsn->getHost(), $sharedTablesV1);
$sharedTablesV2 = !$projectTables && !$sharedTablesV1;
$sharedTables = $sharedTablesV1 || $sharedTablesV2;
while (true) {
$collections = $dbForProject->listCollections($limit);
foreach ($collections as $collection) {
if ($dsn->getHost() !== System::getEnv('_APP_DATABASE_SHARED_TABLES', '') || !\in_array($collection->getId(), $projectCollectionIds)) {
try {
try {
if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) {
$dbForProject->deleteCollection($collection->getId());
} catch (Throwable $e) {
Console::error('Error deleting '.$collection->getId().' '.$e->getMessage());
/**
* Ignore junction tables;
*/
if (!preg_match('/^_\d+_\d+$/', $collection->getId())) {
throw $e;
}
} else {
$this->deleteByGroup($collection->getId(), [], database: $dbForProject);
}
} else {
$this->deleteByGroup($collection->getId(), [], database: $dbForProject);
} catch (Throwable $e) {
Console::error('Error deleting '.$collection->getId().' '.$e->getMessage());
}
}
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
if ($sharedTables) {
$collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections);
if (empty(\array_diff($collectionsIds, $projectCollectionIds))) {
@@ -531,50 +531,57 @@ class Deletes extends Action
// Delete Platforms
$this->deleteByGroup('platforms', [
Query::equal('projectInternalId', [$projectInternalId])
], $dbForConsole);
], $dbForPlatform);
// Delete project and function rules
$this->deleteByGroup('rules', [
Query::equal('projectInternalId', [$projectInternalId])
], $dbForConsole, function (Document $document) use ($dbForConsole) {
$this->deleteRule($dbForConsole, $document);
], $dbForPlatform, function (Document $document) use ($dbForPlatform, $certificates) {
$this->deleteRule($dbForPlatform, $document, $certificates);
});
// Delete Keys
$this->deleteByGroup('keys', [
Query::equal('projectInternalId', [$projectInternalId])
], $dbForConsole);
], $dbForPlatform);
// Delete Webhooks
$this->deleteByGroup('webhooks', [
Query::equal('projectInternalId', [$projectInternalId])
], $dbForConsole);
], $dbForPlatform);
// Delete VCS Installations
$this->deleteByGroup('installations', [
Query::equal('projectInternalId', [$projectInternalId])
], $dbForConsole);
], $dbForPlatform);
// Delete VCS Repositories
$this->deleteByGroup('repositories', [
Query::equal('projectInternalId', [$projectInternalId]),
], $dbForConsole);
], $dbForPlatform);
// Delete VCS comments
$this->deleteByGroup('vcsComments', [
Query::equal('projectInternalId', [$projectInternalId]),
], $dbForConsole);
], $dbForPlatform);
// Delete Schedules (No projectInternalId in this collection)
$this->deleteByGroup('schedules', [
Query::equal('projectId', [$projectId]),
], $dbForConsole);
], $dbForPlatform);
// Delete metadata table
if ($dsn->getHost() !== System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$dbForProject->deleteCollection('_metadata');
} else {
$this->deleteByGroup('_metadata', [], $dbForProject);
if ($projectTables) {
$dbForProject->deleteCollection(Database::METADATA);
} elseif ($sharedTablesV1) {
$this->deleteByGroup(Database::METADATA, [], $dbForProject);
} elseif ($sharedTablesV2) {
$queries = \array_map(
fn ($id) => Query::notEqual('$id', $id),
$projectCollectionIds
);
$this->deleteByGroup(Database::METADATA, $queries, $dbForProject);
}
// Delete all storage directories
@@ -641,7 +648,7 @@ class Deletes extends Action
}
/**
* @param database $dbForConsole
* @param database $dbForPlatform
* @param callable $getProjectDB
* @param string $datetime
* @return void
@@ -657,7 +664,7 @@ class Deletes extends Action
}
/**
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param callable $getProjectDB
* @return void
* @throws Exception|Throwable
@@ -675,42 +682,21 @@ class Deletes extends Action
}
/**
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param string $datetime
* @return void
* @throws Exception
*/
private function deleteRealtimeUsage(Database $dbForConsole, string $datetime): void
private function deleteRealtimeUsage(Database $dbForPlatform, string $datetime): void
{
// Delete Dead Realtime Logs
$this->deleteByGroup('realtime', [
Query::lessThan('timestamp', $datetime)
], $dbForConsole);
], $dbForPlatform);
}
/**
* @param Database $dbForConsole
* @param callable $getProjectDB
* @param string $datetime
* @return void
* @throws Exception
*/
private function deleteAbuseLogs(Document $project, callable $getProjectDB, string $abuseRetention): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
$abuse = new Abuse($timeLimit);
try {
$abuse->cleanup($abuseRetention);
} catch (DatabaseException $e) {
Console::error('Failed to delete abuse logs for project ' . $projectId . ': ' . $e->getMessage());
}
}
/**
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param callable $getProjectDB
* @param string $datetime
* @return void
@@ -738,7 +724,7 @@ class Deletes extends Action
* @return void
* @throws Exception
*/
private function deleteFunction(Database $dbForConsole, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void
private function deleteFunction(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
@@ -753,8 +739,8 @@ class Deletes extends Action
Query::equal('resourceType', ['function']),
Query::equal('resourceInternalId', [$functionInternalId]),
Query::equal('projectInternalId', [$project->getInternalId()])
], $dbForConsole, function (Document $document) use ($project, $dbForConsole) {
$this->deleteRule($dbForConsole, $document);
], $dbForPlatform, function (Document $document) use ($project, $dbForPlatform, $certificates) {
$this->deleteRule($dbForPlatform, $document, $certificates);
});
/**
@@ -808,13 +794,13 @@ class Deletes extends Action
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::equal('resourceInternalId', [$functionInternalId]),
Query::equal('resourceType', ['function']),
], $dbForConsole, function (Document $document) use ($dbForConsole) {
], $dbForPlatform, function (Document $document) use ($dbForPlatform) {
$providerRepositoryId = $document->getAttribute('providerRepositoryId', '');
$projectInternalId = $document->getAttribute('projectInternalId', '');
$this->deleteByGroup('vcsComments', [
Query::equal('providerRepositoryId', [$providerRepositoryId]),
Query::equal('projectInternalId', [$projectInternalId]),
], $dbForConsole);
], $dbForPlatform);
});
/**
@@ -1036,29 +1022,18 @@ class Deletes extends Action
}
/**
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Document $document rule document
* @return void
*/
private function deleteRule(Database $dbForConsole, Document $document): void
private function deleteRule(Database $dbForPlatform, Document $document, CertificatesAdapter $certificates): void
{
$domain = $document->getAttribute('domain');
$directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
$checkTraversal = realpath($directory) === $directory;
if ($checkTraversal && is_dir($directory)) {
// Delete files, so Traefik is aware of change
array_map('unlink', glob($directory . '/*.*'));
rmdir($directory);
Console::info("Deleted certificate files for {$domain}");
} else {
Console::info("No certificate files found for {$domain}");
}
$certificates->deleteCertificate($domain);
// Delete certificate document, so Appwrite is aware of change
if (isset($document['certificateId'])) {
$dbForConsole->deleteDocument('certificates', $document['certificateId']);
$dbForPlatform->deleteDocument('certificates', $document['certificateId']);
}
}
@@ -1079,21 +1054,21 @@ class Deletes extends Action
}
/**
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param callable $getProjectDB
* @param Document $document
* @param Document $project
* @return void
* @throws Exception
*/
private function deleteInstallation(Database $dbForConsole, callable $getProjectDB, Document $document, Document $project): void
private function deleteInstallation(Database $dbForPlatform, callable $getProjectDB, Document $document, Document $project): void
{
$dbForProject = $getProjectDB($project);
$this->listByGroup('functions', [
Query::equal('installationInternalId', [$document->getInternalId()])
], $dbForProject, function ($function) use ($dbForProject, $dbForConsole) {
$dbForConsole->deleteDocument('repositories', $function->getAttribute('repositoryId'));
], $dbForProject, function ($function) use ($dbForProject, $dbForPlatform) {
$dbForPlatform->deleteDocument('repositories', $function->getAttribute('repositoryId'));
$function = $function
->setAttribute('installationId', '')
src/Appwrite/Platform/Workers/Migrations.php
+51 -27
View File
@@ -16,7 +16,6 @@ use Utopia\Database\Exception\Conflict;
use Utopia\Database\Exception\Restricted;
use Utopia\Database\Exception\Structure;
use Utopia\Database\Helpers\ID;
use Utopia\Logger\Log;
use Utopia\Migration\Destination;
use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite;
use Utopia\Migration\Exception as MigrationException;
@@ -33,10 +32,12 @@ class Migrations extends Action
{
protected Database $dbForProject;
protected Database $dbForConsole;
protected Database $dbForPlatform;
protected Document $project;
protected $logError;
public static function getName(): string
{
return 'migrations';
@@ -51,15 +52,15 @@ class Migrations extends Action
->desc('Migrations worker')
->inject('message')
->inject('dbForProject')
->inject('dbForConsole')
->inject('log')
->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, Log $log) => $this->action($message, $dbForProject, $dbForConsole, $log));
->inject('dbForPlatform')
->inject('logError')
->callback(fn (Message $message, Database $dbForProject, Database $dbForPlatform, callable $logError) => $this->action($message, $dbForProject, $dbForPlatform, $logError));
}
/**
* @throws Exception
*/
public function action(Message $message, Database $dbForProject, Database $dbForConsole, Log $log): void
public function action(Message $message, Database $dbForProject, Database $dbForPlatform, callable $logError): void
{
$payload = $message->getPayload() ?? [];
@@ -76,8 +77,9 @@ class Migrations extends Action
}
$this->dbForProject = $dbForProject;
$this->dbForConsole = $dbForConsole;
$this->dbForPlatform = $dbForPlatform;
$this->project = $project;
$this->logError = $logError;
/**
* Handle Event execution.
@@ -86,10 +88,7 @@ class Migrations extends Action
return;
}
$log->addTag('migrationId', $migration->getId());
$log->addTag('projectId', $project->getId());
$this->processMigration($migration, $log);
$this->processMigration($migration);
}
/**
@@ -198,7 +197,7 @@ class Migrations extends Action
*/
protected function removeAPIKey(Document $apiKey): void
{
$this->dbForConsole->deleteDocument('keys', $apiKey->getId());
$this->dbForPlatform->deleteDocument('keys', $apiKey->getId());
}
/**
@@ -245,8 +244,8 @@ class Migrations extends Action
'secret' => $generatedSecret,
]);
$this->dbForConsole->createDocument('keys', $key);
$this->dbForConsole->purgeCachedDocument('projects', $project->getId());
$this->dbForPlatform->createDocument('keys', $key);
$this->dbForPlatform->purgeCachedDocument('projects', $project->getId());
return $key;
}
@@ -259,10 +258,10 @@ class Migrations extends Action
* @throws \Utopia\Database\Exception
* @throws Exception
*/
protected function processMigration(Document $migration, Log $log): void
protected function processMigration(Document $migration): void
{
$project = $this->project;
$projectDocument = $this->dbForConsole->getDocument('projects', $project->getId());
$projectDocument = $this->dbForPlatform->getDocument('projects', $project->getId());
$tempAPIKey = $this->generateAPIKey($projectDocument);
$transfer = $source = $destination = null;
@@ -285,8 +284,6 @@ class Migrations extends Action
$migration->setAttribute('status', 'processing');
$this->updateMigrationDocument($migration, $projectDocument);
$log->addTag('type', $migration->getAttribute('source'));
$source = $this->processSource($migration);
$destination = $this->processDestination($migration, $tempAPIKey->getAttribute('secret'));
@@ -324,7 +321,6 @@ class Migrations extends Action
$errorMessages = [];
foreach ($sourceErrors as $error) {
/** @var $sourceErrors $error */
$message = "Error occurred while fetching '{$error->getResourceName()}:{$error->getResourceId()}' from source with message: '{$error->getMessage()}'";
if ($error->getPrevious()) {
$message .= " Message: ".$error->getPrevious()->getMessage() . " File: ".$error->getPrevious()->getFile() . " Line: ".$error->getPrevious()->getLine();
@@ -344,7 +340,6 @@ class Migrations extends Action
}
$migration->setAttribute('errors', $errorMessages);
$log->addExtra('migrationErrors', json_encode($errorMessages));
$this->updateMigrationDocument($migration, $projectDocument);
return;
@@ -359,7 +354,12 @@ class Migrations extends Action
if (! $migration->isEmpty()) {
$migration->setAttribute('status', 'failed');
$migration->setAttribute('stage', 'finished');
$migration->setAttribute('errors', [$th->getMessage()]);
call_user_func($this->logError, $th, 'appwrite-worker', 'appwrite-queue-'.self::getName(), [
'migrationId' => $migration->getId(),
'source' => $migration->getAttribute('source') ?? '',
'destination' => $migration->getAttribute('destination') ?? '',
]);
return;
}
@@ -379,7 +379,6 @@ class Migrations extends Action
}
$migration->setAttribute('errors', $errorMessages);
$log->addTag('migrationErrors', json_encode($errorMessages));
}
} finally {
if (! $tempAPIKey->isEmpty()) {
@@ -391,15 +390,40 @@ class Migrations extends Action
if ($migration->getAttribute('status', '') === 'failed') {
Console::error('Migration('.$migration->getInternalId().':'.$migration->getId().') failed, Project('.$this->project->getInternalId().':'.$this->project->getId().')');
$destination->error();
$source->error();
if ($destination) {
$destination->error();
throw new Exception('Migration failed');
foreach ($destination->getErrors() as $error) {
/** @var MigrationException $error */
call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [
'migrationId' => $migration->getId(),
'source' => $migration->getAttribute('source') ?? '',
'destination' => $migration->getAttribute('destination') ?? '',
'resourceName' => $error->getResourceName(),
'resourceGroup' => $error->getResourceGroup()
]);
}
}
if ($source) {
$source->error();
foreach ($source->getErrors() as $error) {
/** @var MigrationException $error */
call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [
'migrationId' => $migration->getId(),
'source' => $migration->getAttribute('source') ?? '',
'destination' => $migration->getAttribute('destination') ?? '',
'resourceName' => $error->getResourceName(),
'resourceGroup' => $error->getResourceGroup()
]);
}
}
}
if ($migration->getAttribute('status', '') === 'completed') {
$destination->success();
$source->success();
$destination?->success();
$source?->success();
}
}
}
src/Appwrite/Platform/Workers/Usage.php
+18 -6
View File
@@ -15,9 +15,10 @@ class Usage extends Action
{
private array $stats = [];
private int $lastTriggeredTime = 0;
private int $aggregationInterval = 20;
private int $keys = 0;
private const INFINITY_PERIOD = '_inf_';
private const KEYS_THRESHOLD = 10000;
private const KEYS_THRESHOLD = 20000;
public static function getName(): string
{
@@ -39,6 +40,7 @@ class Usage extends Action
$this->action($message, $getProjectDB, $queueForUsageDump);
});
$this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
$this->lastTriggeredTime = time();
}
@@ -56,10 +58,15 @@ class Usage extends Action
if (empty($payload)) {
throw new Exception('Missing payload');
}
//Todo Figure out way to preserve keys when the container is being recreated @shimonewman
$aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
$project = new Document($payload['project'] ?? []);
$document = $payload['project'] ?? [];
$project = new Document($document);
if (empty($project->getAttribute('database'))) {
var_dump($payload);
return;
}
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
if (empty($document)) {
@@ -74,7 +81,12 @@ class Usage extends Action
);
}
$this->stats[$projectId]['project'] = $project;
$this->stats[$projectId]['project'] = [
'$id' => $project->getId(),
'$internalId' => $project->getInternalId(),
'database' => $project->getAttribute('database'),
];
$this->stats[$projectId]['receivedAt'] = DateTime::now();
foreach ($payload['metrics'] ?? [] as $metric) {
$this->keys++;
@@ -89,7 +101,7 @@ class Usage extends Action
// If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
if (
$this->keys >= self::KEYS_THRESHOLD ||
(time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0)
(time() - $this->lastTriggeredTime > $this->aggregationInterval && $this->keys > 0)
) {
Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
src/Appwrite/Platform/Workers/UsageDump.php
+22 -3
View File
@@ -57,16 +57,35 @@ class UsageDump extends Action
throw new Exception('Missing payload');
}
// TODO: rename both usage workers @shimonewman
foreach ($payload['stats'] ?? [] as $stats) {
$project = new Document($stats['project'] ?? []);
//$project = new Document($stats['project'] ?? []);
/**
* Start temp bug fallback
*/
$document = $stats['project'] ?? [];
if (!empty($document['$uid'])) {
$document['$id'] = $document['$uid'];
}
$project = new Document($document);
if (empty($project->getAttribute('database'))) {
continue;
}
/**
* End temp bug fallback
*/
$numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
$receivedAt = $stats['receivedAt'] ?? 'NONE';
if ($numberOfKeys === 0) {
continue;
}
console::log('[' . DateTime::now() . '] ProjectId [' . $project->getInternalId() . '] ReceivedAt [' . $receivedAt . '] ' . $numberOfKeys . ' keys');
console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys);
try {
$dbForProject = $getProjectDB($project);
src/Appwrite/Platform/Workers/Webhooks.php
+36 -19
View File
@@ -3,6 +3,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Event\Mail;
use Appwrite\Event\Usage;
use Appwrite\Template\Template;
use Exception;
use Utopia\Database\Database;
@@ -31,21 +32,22 @@ class Webhooks extends Action
$this
->desc('Webhooks worker')
->inject('message')
->inject('dbForConsole')
->inject('dbForPlatform')
->inject('queueForMails')
->inject('queueForUsage')
->inject('log')
->callback(fn (Message $message, Database $dbForConsole, Mail $queueForMails, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $log));
->callback(fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log) => $this->action($message, $dbForPlatform, $queueForMails, $queueForUsage, $log));
}
/**
* @param Message $message
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Mail $queueForMails
* @param Log $log
* @return void
* @throws Exception
*/
public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Log $log): void
public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log): void
{
$this->errors = [];
$payload = $message->getPayload() ?? [];
@@ -56,14 +58,15 @@ class Webhooks extends Action
$events = $payload['events'];
$webhookPayload = json_encode($payload['payload']);
$project = new Document($payload['project']);
$user = new Document($payload['user'] ?? []);
$project = new Document($payload['project']);
$project = $dbForPlatform->getDocument('projects', $project->getId());
$log->addTag('projectId', $project->getId());
foreach ($project->getAttribute('webhooks', []) as $webhook) {
if (array_intersect($webhook->getAttribute('events', []), $events)) {
$this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForConsole, $queueForMails);
$this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForUsage);
}
}
@@ -78,11 +81,11 @@ class Webhooks extends Action
* @param Document $webhook
* @param Document $user
* @param Document $project
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Mail $queueForMails
* @return void
*/
private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForConsole, Mail $queueForMails): void
private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage): void
{
if ($webhook->getAttribute('enabled') !== true) {
return;
@@ -138,8 +141,8 @@ class Webhooks extends Action
\curl_close($ch);
if (!empty($curlError) || $statusCode >= 400) {
$dbForConsole->increaseDocumentAttribute('webhooks', $webhook->getId(), 'attempts', 1);
$webhook = $dbForConsole->getDocument('webhooks', $webhook->getId());
$dbForPlatform->increaseDocumentAttribute('webhooks', $webhook->getId(), 'attempts', 1);
$webhook = $dbForPlatform->getDocument('webhooks', $webhook->getId());
$attempts = $webhook->getAttribute('attempts');
$logs = '';
@@ -158,18 +161,32 @@ class Webhooks extends Action
if ($attempts >= \intval(System::getEnv('_APP_WEBHOOK_MAX_FAILED_ATTEMPTS', '10'))) {
$webhook->setAttribute('enabled', false);
$this->sendEmailAlert($attempts, $statusCode, $webhook, $project, $dbForConsole, $queueForMails);
$this->sendEmailAlert($attempts, $statusCode, $webhook, $project, $dbForPlatform, $queueForMails);
}
$dbForConsole->updateDocument('webhooks', $webhook->getId(), $webhook);
$dbForConsole->purgeCachedDocument('projects', $project->getId());
$dbForPlatform->updateDocument('webhooks', $webhook->getId(), $webhook);
$dbForPlatform->purgeCachedDocument('projects', $project->getId());
$this->errors[] = $logs;
$queueForUsage
->addMetric(METRIC_WEBHOOKS_FAILED, 1)
->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_FAILED), 1)
;
} else {
$webhook->setAttribute('attempts', 0); // Reset attempts on success
$dbForConsole->updateDocument('webhooks', $webhook->getId(), $webhook);
$dbForConsole->purgeCachedDocument('projects', $project->getId());
$dbForPlatform->updateDocument('webhooks', $webhook->getId(), $webhook);
$dbForPlatform->purgeCachedDocument('projects', $project->getId());
$queueForUsage
->addMetric(METRIC_WEBHOOKS_SENT, 1)
->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_SENT), 1)
;
}
$queueForUsage
->setProject($project)
->trigger();
}
/**
@@ -177,20 +194,20 @@ class Webhooks extends Action
* @param mixed $statusCode
* @param Document $webhook
* @param Document $project
* @param Database $dbForConsole
* @param Database $dbForPlatform
* @param Mail $queueForMails
* @return void
*/
public function sendEmailAlert(int $attempts, mixed $statusCode, Document $webhook, Document $project, Database $dbForConsole, Mail $queueForMails): void
public function sendEmailAlert(int $attempts, mixed $statusCode, Document $webhook, Document $project, Database $dbForPlatform, Mail $queueForMails): void
{
$memberships = $dbForConsole->find('memberships', [
$memberships = $dbForPlatform->find('memberships', [
Query::equal('teamInternalId', [$project->getAttribute('teamInternalId')]),
Query::limit(APP_LIMIT_SUBQUERY)
]);
$userIds = array_column(\array_map(fn ($membership) => $membership->getArrayCopy(), $memberships), 'userId');
$users = $dbForConsole->find('users', [
$users = $dbForPlatform->find('users', [
Query::equal('$id', $userIds),
]);
src/Appwrite/Specification/Format/OpenAPI3.php
+1
View File
@@ -323,6 +323,7 @@ class OpenAPI3 extends Format
case 'Utopia\Validator\JSON':
case 'Utopia\Validator\Mock':
case 'Utopia\Validator\Assoc':
case 'Appwrite\Functions\Validator\Payload':
$param['default'] = (empty($param['default'])) ? new \stdClass() : $param['default'];
$node['schema']['type'] = 'object';
$node['schema']['x-example'] = '{}';
src/Appwrite/Specification/Format/Swagger2.php
+1
View File
@@ -349,6 +349,7 @@ class Swagger2 extends Format
case 'Utopia\Validator\JSON':
case 'Utopia\Validator\Mock':
case 'Utopia\Validator\Assoc':
case 'Appwrite\Functions\Validator\Payload':
$node['type'] = 'object';
$node['default'] = (empty($param['default'])) ? new \stdClass() : $param['default'];
$node['x-example'] = '{}';
src/Appwrite/Utopia/Response/Model/Func.php
+1 -1
View File
@@ -94,7 +94,7 @@ class Func extends Model
])
->addRule('schedule', [
'type' => self::TYPE_STRING,
'description' => 'Function execution schedult in CRON format.',
'description' => 'Function execution schedule in CRON format.',
'default' => '',
'example' => '5 4 * * *',
])