Merge branch 'feat-sites' into feat-add-fallbackFile-attribute

2026-05-26 13:51:13 +00:00 · 2024-11-20 14:14:28 +01:00
parent cb359e40ba 2e9e6d38e2
commit 09e2ded206
28 changed files with 2229 additions and 464 deletions
@@ -10,6 +10,7 @@ use Utopia\Database\Document;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
+use Utopia\Database\Validator\Authorization;
 use Utopia\Platform\Action;
 use Utopia\Swoole\Request;
 use Utopia\System\System;
@@ -92,7 +93,7 @@ class Base extends Action
            ->setTemplate($template);
    }

-    public function redeployVcsSite(Request $request, Document $site, Document $project, Document $installation, Database $dbForProject, Build $queueForBuilds, Document $template, GitHub $github)
+    public function redeployVcsSite(Request $request, Document $site, Document $project, Document $installation, Database $dbForProject, Database $dbForConsole, Build $queueForBuilds, Document $template, GitHub $github)
    {
        $deploymentId = ID::unique();
        $providerInstallationId = $installation->getAttribute('providerInstallationId', '');
@@ -160,6 +161,27 @@ class Base extends Action
            'activate' => true,
        ]));

+        // Preview deployments for sites
+        $projectId = $project->getId();
+
+        $sitesDomain = System::getEnv('_APP_DOMAIN_SITES', '');
+        $domain = "{$deploymentId}-{$projectId}.{$sitesDomain}";
+        $ruleId = md5($domain);
+
+        $rule = Authorization::skip(
+            fn() => $dbForConsole->createDocument('rules', new Document([
+                '$id' => $ruleId,
+                'projectId' => $project->getId(),
+                'projectInternalId' => $project->getInternalId(),
+                'domain' => $domain,
+                'resourceType' => 'deployment',
+                'resourceId' => $deploymentId,
+                'resourceInternalId' => $deployment->getInternalId(),
+                'status' => 'verified',
+                'certificateId' => '',
+            ]))
+        );
+
        $queueForBuilds
            ->setType(BUILD_TYPE_DEPLOYMENT)
            ->setResource($site)
@@ -691,31 +691,6 @@ class Builds extends Action

            $build = $dbForProject->updateDocument('builds', $buildId, $build);

-            // Preview deployments for sites
-            if ($resource->getCollection() === 'sites') {
-                $ruleId = ID::unique();
-
-                $deploymentId = $deployment->getId();
-                $projectId = $project->getId();
-
-                $sitesDomain = System::getEnv('_APP_DOMAIN_SITES', '');
-                $domain = "{$deploymentId}-{$projectId}.{$sitesDomain}";
-
-                $rule = Authorization::skip(
-                    fn () => $dbForConsole->createDocument('rules', new Document([
-                        '$id' => $ruleId,
-                        'projectId' => $project->getId(),
-                        'projectInternalId' => $project->getInternalId(),
-                        'domain' => $domain,
-                        'resourceType' => 'deployment',
-                        'resourceId' => $deployment->getId(),
-                        'resourceInternalId' => $deployment->getInternalId(),
-                        'status' => 'verified',
-                        'certificateId' => '',
-                    ]))
-                );
-            }
-
            if ($isVcsEnabled) {
                $this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForConsole);
            }
@@ -12,6 +12,7 @@ use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
 use Utopia\Database\Query;
+use Utopia\Database\Validator\Authorization;
 use Utopia\Database\Validator\UID;
 use Utopia\Platform\Action;
 use Utopia\Platform\Scope\HTTP;
@@ -64,6 +65,8 @@ class CreateDeployment extends Action
            ->inject('request')
            ->inject('response')
            ->inject('dbForProject')
+            ->inject('dbForConsole')
+            ->inject('project')
            ->inject('queueForEvents')
            ->inject('deviceForSites')
            ->inject('deviceForFunctions') // TODO: Remove this later once volume is added to executor
@@ -72,7 +75,7 @@ class CreateDeployment extends Action
            ->callback([$this, 'action']);
    }

-    public function action(string $siteId, ?string $installCommand, ?string $buildCommand, ?string $outputDirectory, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Event $queueForEvents, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForLocal, Build $queueForBuilds)
+    public function action(string $siteId, ?string $installCommand, ?string $buildCommand, ?string $outputDirectory, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Database $dbForConsole, Document $project, Event $queueForEvents, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForLocal, Build $queueForBuilds)
    {
        $activate = \strval($activate) === 'true' || \strval($activate) === '1';

@@ -213,6 +216,27 @@ class CreateDeployment extends Action
                    'metadata' => $metadata,
                    'type' => $type
                ]));
+
+                // Preview deployments for sites
+                $projectId = $project->getId();
+
+                $sitesDomain = System::getEnv('_APP_DOMAIN_SITES', '');
+                $domain = "{$deploymentId}-{$projectId}.{$sitesDomain}";
+                $ruleId = md5($domain);
+
+                $rule = Authorization::skip(
+                    fn () => $dbForConsole->createDocument('rules', new Document([
+                        '$id' => $ruleId,
+                        'projectId' => $project->getId(),
+                        'projectInternalId' => $project->getInternalId(),
+                        'domain' => $domain,
+                        'resourceType' => 'deployment',
+                        'resourceId' => $deploymentId,
+                        'resourceInternalId' => $deployment->getInternalId(),
+                        'status' => 'verified',
+                        'certificateId' => '',
+                    ]))
+                );
            } else {
                $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('size', $fileSize)->setAttribute('metadata', $metadata));
            }
@@ -247,6 +271,27 @@ class CreateDeployment extends Action
                    'metadata' => $metadata,
                    'type' => $type
                ]));
+
+                // Preview deployments for sites
+                $projectId = $project->getId();
+
+                $sitesDomain = System::getEnv('_APP_DOMAIN_SITES', '');
+                $domain = "{$deploymentId}-{$projectId}.{$sitesDomain}";
+                $ruleId = md5($domain);
+
+                $rule = Authorization::skip(
+                    fn () => $dbForConsole->createDocument('rules', new Document([
+                        '$id' => $ruleId,
+                        'projectId' => $project->getId(),
+                        'projectInternalId' => $project->getInternalId(),
+                        'domain' => $domain,
+                        'resourceType' => 'deployment',
+                        'resourceId' => $deploymentId,
+                        'resourceInternalId' => $deployment->getInternalId(),
+                        'status' => 'verified',
+                        'certificateId' => '',
+                    ]))
+                );
            } else {
                $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('chunksUploaded', $chunksUploaded)->setAttribute('metadata', $metadata));
            }
@@ -99,20 +99,16 @@ class CreateSite extends Base
    public function action(string $siteId, string $name, string $framework, bool $enabled, int $timeout, string $installCommand, string $buildCommand, string $outputDirectory, string $subdomain, string $buildRuntime, string $serveRuntime, string $installationId, string $fallbackFile, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github)
    {
        $sitesDomain = System::getEnv('_APP_DOMAIN_SITES', '');
-        $ruleId = '';
        $routeSubdomain = '';
        $domain = '';

        if (!empty($sitesDomain)) {
-            $ruleId = ID::unique();
            $routeSubdomain = $subdomain ?: ID::unique();
            $domain = "{$routeSubdomain}.{$sitesDomain}";

-            $subdomain = Authorization::skip(fn () => $dbForConsole->findOne('rules', [
-                Query::equal('domain', [$domain])
-            ]));
+            $subdomain = Authorization::skip(fn () => $dbForConsole->getDocument('rules', \md5($domain)));

-            if (!empty($subdomain)) {
+            if ($subdomain && !$subdomain->isEmpty()) {
                throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Subdomain already exists. Please choose a different subdomain.');
            }
        }
@@ -208,7 +204,7 @@ class CreateSite extends Base

        if (!empty($providerRepositoryId)) {
            // Deploy VCS
-            $this->redeployVcsSite($request, $site, $project, $installation, $dbForProject, $queueForBuilds, $template, $github);
+            $this->redeployVcsSite($request, $site, $project, $installation, $dbForProject, $dbForConsole, $queueForBuilds, $template, $github);
        } elseif (!$template->isEmpty()) {
            // Deploy non-VCS from template
            $deploymentId = ID::unique();
@@ -230,6 +226,26 @@ class CreateSite extends Base
                'activate' => true,
            ]));

+            // Preview deployments url
+            $projectId = $project->getId();
+
+            $sitesDomain = System::getEnv('_APP_DOMAIN_SITES', '');
+            $previewDomain = "{$deploymentId}-{$projectId}.{$sitesDomain}";
+
+            $rule = Authorization::skip(
+                fn() => $dbForConsole->createDocument('rules', new Document([
+                    '$id' => \md5($previewDomain),
+                    'projectId' => $project->getId(),
+                    'projectInternalId' => $project->getInternalId(),
+                    'domain' => $previewDomain,
+                    'resourceType' => 'deployment',
+                    'resourceId' => $deploymentId,
+                    'resourceInternalId' => $deployment->getInternalId(),
+                    'status' => 'verified',
+                    'certificateId' => '',
+                ]))
+            );
+
            $queueForBuilds
                ->setType(BUILD_TYPE_DEPLOYMENT)
                ->setResource($site)
@@ -240,7 +256,7 @@ class CreateSite extends Base
        if (!empty($sitesDomain)) {
            $rule = Authorization::skip(
                fn () => $dbForConsole->createDocument('rules', new Document([
-                    '$id' => $ruleId,
+                    '$id' => \md5($domain),
                    'projectId' => $project->getId(),
                    'projectInternalId' => $project->getInternalId(),
                    'domain' => $domain,
@@ -25,6 +25,9 @@ use Appwrite\Spec\Swagger2;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Utopia\Platform\Action;
+use Utopia\Validator\Nullable;
+use Utopia\Validator\Text;
+use Utopia\Validator\WhiteList;

 class SDKs extends Action
 {
@@ -37,23 +40,35 @@ class SDKs extends Action
    {
        $this
            ->desc('Generate Appwrite SDKs')
-            ->callback(fn () => $this->action());
+            ->param('platform', null, new Nullable(new Text(256)), 'Selected Platform', optional: true)
+            ->param('sdk', null, new Nullable(new Text(256)), 'Selected SDK', optional:true)
+            ->param('version', null, new Nullable(new Text(256)), 'Selected SDK', optional:true)
+            ->param('git', null, new Nullable(new WhiteList(['yes', 'no'])), 'Should we use git push?', optional: true)
+            ->param('production', null, new Nullable(new WhiteList(['yes', 'no'])), 'Should we push to production?', optional:true)
+            ->param('message', null, new Nullable(new Text(256)), 'Commit Message', optional:true)
+            ->callback([$this, 'action']);
    }

-    public function action(): void
+    public function action(?string $selectedPlatform, ?string $selectedSDK, ?string $version, ?string $git, ?string $production, ?string $message)
    {
-        $platforms = Config::getParam('platforms');
-        $selectedPlatform = Console::confirm('Choose Platform ("' . APP_PLATFORM_CLIENT . '", "' . APP_PLATFORM_SERVER . '", "' . APP_PLATFORM_CONSOLE . '" or "*" for all):');
-        $selectedSDK = \strtolower(Console::confirm('Choose SDK ("*" for all):'));
-        $version = Console::confirm('Choose an Appwrite version');
-        $git = (Console::confirm('Should we use git push? (yes/no)') == 'yes');
-        $production = ($git) ? (Console::confirm('Type "Appwrite" to push code to production git repos') == 'Appwrite') : false;
-        $message = ($git) ? Console::confirm('Please enter your commit message:') : '';
+        $selectedPlatform ??= Console::confirm('Choose Platform ("' . APP_PLATFORM_CLIENT . '", "' . APP_PLATFORM_SERVER . '", "' . APP_PLATFORM_CONSOLE . '" or "*" for all):');
+        $selectedSDK ??= \strtolower(Console::confirm('Choose SDK ("*" for all):'));
+        $version ??= Console::confirm('Choose an Appwrite version');
+
+        $git ??= Console::confirm('Should we use git push? (yes/no)');
+        $git = $git === 'yes';
+
+        if ($git) {
+            $production ??= Console::confirm('Type "Appwrite" to push code to production git repos');
+            $production = $production === 'Appwrite';
+            $message ??= Console::confirm('Please enter your commit message:');
+        }

        if (!in_array($version, ['0.6.x', '0.7.x', '0.8.x', '0.9.x', '0.10.x', '0.11.x', '0.12.x', '0.13.x', '0.14.x', '0.15.x', '1.0.x', '1.1.x', '1.2.x', '1.3.x', '1.4.x', '1.5.x', '1.6.x', 'latest'])) {
            throw new \Exception('Unknown version given');
        }

+        $platforms = Config::getParam('platforms');
        foreach ($platforms as $key => $platform) {
            if ($selectedPlatform !== $key && $selectedPlatform !== '*') {
                continue;
@@ -354,11 +354,9 @@ class Databases extends Action
            }
        } finally {
            $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $collectionId);
-            $dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());

            if (!$relatedCollection->isEmpty() && !$relatedAttribute->isEmpty()) {
                $dbForProject->purgeCachedDocument('database_' . $database->getInternalId(), $relatedCollection->getId());
-                $dbForProject->purgeCachedCollection('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId());
            }
        }
    }
@@ -16,7 +16,6 @@ use Utopia\Database\Exception\Conflict;
 use Utopia\Database\Exception\Restricted;
 use Utopia\Database\Exception\Structure;
 use Utopia\Database\Helpers\ID;
-use Utopia\Logger\Log;
 use Utopia\Migration\Destination;
 use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite;
 use Utopia\Migration\Exception as MigrationException;
@@ -37,6 +36,8 @@ class Migrations extends Action

    protected Document $project;

+    protected $logError;
+
    public static function getName(): string
    {
        return 'migrations';
@@ -52,14 +53,14 @@ class Migrations extends Action
            ->inject('message')
            ->inject('dbForProject')
            ->inject('dbForConsole')
-            ->inject('log')
-            ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, Log $log) => $this->action($message, $dbForProject, $dbForConsole, $log));
+            ->inject('logError')
+            ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, callable $logError) => $this->action($message, $dbForProject, $dbForConsole, $logError));
    }

    /**
     * @throws Exception
     */
-    public function action(Message $message, Database $dbForProject, Database $dbForConsole, Log $log): void
+    public function action(Message $message, Database $dbForProject, Database $dbForConsole, callable $logError): void
    {
        $payload = $message->getPayload() ?? [];

@@ -78,6 +79,7 @@ class Migrations extends Action
        $this->dbForProject = $dbForProject;
        $this->dbForConsole = $dbForConsole;
        $this->project = $project;
+        $this->logError = $logError;

        /**
         * Handle Event execution.
@@ -86,10 +88,7 @@ class Migrations extends Action
            return;
        }

-        $log->addTag('migrationId', $migration->getId());
-        $log->addTag('projectId', $project->getId());
-
-        $this->processMigration($migration, $log);
+        $this->processMigration($migration);
    }

    /**
@@ -259,7 +258,7 @@ class Migrations extends Action
     * @throws \Utopia\Database\Exception
     * @throws Exception
     */
-    protected function processMigration(Document $migration, Log $log): void
+    protected function processMigration(Document $migration): void
    {
        $project = $this->project;
        $projectDocument = $this->dbForConsole->getDocument('projects', $project->getId());
@@ -285,8 +284,6 @@ class Migrations extends Action
            $migration->setAttribute('status', 'processing');
            $this->updateMigrationDocument($migration, $projectDocument);

-            $log->addTag('type', $migration->getAttribute('source'));
-
            $source = $this->processSource($migration);
            $destination = $this->processDestination($migration, $tempAPIKey->getAttribute('secret'));

@@ -324,7 +321,6 @@ class Migrations extends Action

                $errorMessages = [];
                foreach ($sourceErrors as $error) {
-                    /** @var $sourceErrors $error */
                    $message = "Error occurred while fetching '{$error->getResourceName()}:{$error->getResourceId()}' from source with message: '{$error->getMessage()}'";
                    if ($error->getPrevious()) {
                        $message .= " Message: ".$error->getPrevious()->getMessage() . " File: ".$error->getPrevious()->getFile() . " Line: ".$error->getPrevious()->getLine();
@@ -344,7 +340,6 @@ class Migrations extends Action
                }

                $migration->setAttribute('errors', $errorMessages);
-                $log->addExtra('migrationErrors', json_encode($errorMessages));
                $this->updateMigrationDocument($migration, $projectDocument);

                return;
@@ -359,7 +354,6 @@ class Migrations extends Action
            if (! $migration->isEmpty()) {
                $migration->setAttribute('status', 'failed');
                $migration->setAttribute('stage', 'finished');
-                $migration->setAttribute('errors', [$th->getMessage()]);

                return;
            }
@@ -379,7 +373,6 @@ class Migrations extends Action
                }

                $migration->setAttribute('errors', $errorMessages);
-                $log->addTag('migrationErrors', json_encode($errorMessages));
            }
        } finally {
            if (! $tempAPIKey->isEmpty()) {
@@ -394,7 +387,23 @@ class Migrations extends Action
                $destination->error();
                $source->error();

-                throw new Exception('Migration failed');
+                foreach ($source->getErrors() as $error) {
+                    call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [
+                        'migrationId' => $migration->getId() ?? '',
+                        'source' => $migration->getAttribute('source') ?? '',
+                        'resourceName' => $error->getResourceName(),
+                        'resourceGroup' => $error->getResourceGroup()
+                    ]);
+                }
+
+                foreach ($destination->getErrors() as $error) {
+                    call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [
+                        'migrationId' => $migration->getId() ?? '',
+                        'source' => $migration->getAttribute('source') ?? '',
+                        'resourceName' => $error->getResourceName(),
+                        'resourceGroup' => $error->getResourceGroup()
+                    ]);
+                }
            }

            if ($migration->getAttribute('status', '') === 'completed') {
@@ -323,6 +323,7 @@ class OpenAPI3 extends Format
                    case 'Utopia\Validator\JSON':
                    case 'Utopia\Validator\Mock':
                    case 'Utopia\Validator\Assoc':
+                    case 'Appwrite\Functions\Validator\Payload':
                        $param['default'] = (empty($param['default'])) ? new \stdClass() : $param['default'];
                        $node['schema']['type'] = 'object';
                        $node['schema']['x-example'] = '{}';
@@ -349,6 +349,7 @@ class Swagger2 extends Format
                    case 'Utopia\Validator\JSON':
                    case 'Utopia\Validator\Mock':
                    case 'Utopia\Validator\Assoc':
+                    case 'Appwrite\Functions\Validator\Payload':
                        $node['type'] = 'object';
                        $node['default'] = (empty($param['default'])) ? new \stdClass() : $param['default'];
                        $node['x-example'] = '{}';