periodic task for rule verification & certificate generation

.env

@@ -101,6 +101,8 @@ _APP_USAGE_AGGREGATION_INTERVAL=30
 _APP_STATS_RESOURCES_INTERVAL=30
 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400
+_APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL=60
+_APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL=86400
 _APP_USAGE_STATS=enabled
 _APP_LOGGING_CONFIG=
 _APP_LOGGING_CONFIG_REALTIME=

Dockerfile

@@ -58,6 +58,7 @@ RUN mkdir -p /storage/uploads && \
 RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/install && \
     chmod +x /usr/local/bin/maintenance &&  \
+    chmod +x /usr/local/bin/rules && \
     chmod +x /usr/local/bin/migrate && \
     chmod +x /usr/local/bin/realtime && \
     chmod +x /usr/local/bin/schedule-functions && \

bin/rules

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php rules $@

docker-compose.yml

@@ -786,6 +786,43 @@ services:
       - _APP_MAINTENANCE_START_TIME
       - _APP_DATABASE_SHARED_TABLES
 
+  appwrite-task-rules:
+    entrypoint: rules
+    <<: *x-logging
+    container_name: appwrite-task-rules
+    image: appwrite-dev
+    networks:
+      - appwrite
+    volumes:
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+    depends_on:
+      - redis
+    environment:
+      - _APP_ENV
+      - _APP_WORKER_PER_CORE
+      - _APP_DOMAIN
+      - _APP_DOMAIN_TARGET_CNAME
+      - _APP_DOMAIN_TARGET_AAAA
+      - _APP_DOMAIN_TARGET_A
+      - _APP_DOMAIN_TARGET_CAA
+      - _APP_DNS
+      - _APP_DOMAIN_FUNCTIONS
+      - _APP_DOMAIN_SITES
+      - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_DB_HOST
+      - _APP_DB_PORT
+      - _APP_DB_SCHEMA
+      - _APP_DB_USER
+      - _APP_DB_PASS
+      - _APP_DATABASE_SHARED_TABLES
+      - _APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL
+      - _APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL
+
   appwrite-task-stats-resources:
     container_name: appwrite-task-stats-resources
     entrypoint: stats-resources

src/Appwrite/Event/Certificate.php

@@ -7,7 +7,10 @@ use Utopia\Queue\Publisher;
 
 class Certificate extends Event
 {
+    public const string ACTION_DOMAIN_VERIFICATION = 'verification';
+    public const string ACTION_GENERATION = 'generation';
     protected bool $skipRenewCheck = false;
+    protected string $action = self::ACTION_GENERATION;
     protected ?Document $domain = null;
     protected ?string $validationDomain = null;
 
@@ -90,6 +93,28 @@ class Certificate extends Event
         return $this->skipRenewCheck;
     }
 
+    /**
+     * Set action for this certificate event.
+     *
+     * @param string $action
+     * @return self
+     */
+    public function setAction(string $action): self
+    {
+        $this->action = $action;
+        return $this;
+    }
+
+    /**
+     * Get action for this certificate event.
+     *
+     * @return string
+     */
+    public function getAction(): string
+    {
+        return $this->action;
+    }
+
 
     /**
      * Prepare the payload for the event

src/Appwrite/Platform/Modules/Proxy/Http/Rules/API/Create.php

@@ -125,6 +125,7 @@ class Create extends Action
                     'domain' => $rule->getAttribute('domain'),
                     'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')),
                 ]))
+                ->setAction(Certificate::ACTION_GENERATION)
                 ->trigger();
         }
 

src/Appwrite/Platform/Modules/Proxy/Http/Rules/Function/Create.php

@@ -143,6 +143,7 @@ class Create extends Action
                     'domain' => $rule->getAttribute('domain'),
                     'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')),
                 ]))
+                ->setAction(Certificate::ACTION_GENERATION)
                 ->trigger();
         }
 

src/Appwrite/Platform/Modules/Proxy/Http/Rules/Get.php

@@ -3,6 +3,7 @@
 namespace Appwrite\Platform\Modules\Proxy\Http\Rules;
 
 use Appwrite\Extend\Exception;
+use Appwrite\Platform\Modules\Proxy\Action;
 use Appwrite\SDK\AuthType;
 use Appwrite\SDK\Method;
 use Appwrite\SDK\Response as SDKResponse;

src/Appwrite/Platform/Modules/Proxy/Http/Rules/Redirect/Create.php

@@ -147,6 +147,7 @@ class Create extends Action
                     'domain' => $rule->getAttribute('domain'),
                     'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')),
                 ]))
+                ->setAction(Certificate::ACTION_GENERATION)
                 ->trigger();
         }
 

src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php

@@ -143,6 +143,7 @@ class Create extends Action
                     'domain' => $rule->getAttribute('domain'),
                     'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')),
                 ]))
+                ->setAction(Certificate::ACTION_GENERATION)
                 ->trigger();
         }
 

src/Appwrite/Platform/Modules/Proxy/Http/Rules/XList.php

@@ -3,6 +3,7 @@
 namespace Appwrite\Platform\Modules\Proxy\Http\Rules;
 
 use Appwrite\Extend\Exception;
+use Appwrite\Platform\Modules\Proxy\Action;
 use Appwrite\SDK\AuthType;
 use Appwrite\SDK\Method;
 use Appwrite\SDK\Response as SDKResponse;

src/Appwrite/Platform/Services/Tasks.php

@@ -7,6 +7,7 @@ use Appwrite\Platform\Tasks\Install;
 use Appwrite\Platform\Tasks\Maintenance;
 use Appwrite\Platform\Tasks\Migrate;
 use Appwrite\Platform\Tasks\QueueRetry;
+use Appwrite\Platform\Tasks\Rules;
 use Appwrite\Platform\Tasks\ScheduleExecutions;
 use Appwrite\Platform\Tasks\ScheduleFunctions;
 use Appwrite\Platform\Tasks\ScheduleMessages;
@@ -29,6 +30,7 @@ class Tasks extends Service
             ->addAction(Doctor::getName(), new Doctor())
             ->addAction(Install::getName(), new Install())
             ->addAction(Maintenance::getName(), new Maintenance())
+            ->addAction(Rules::getName(), new Rules())
             ->addAction(Migrate::getName(), new Migrate())
             ->addAction(QueueRetry::getName(), new QueueRetry())
             ->addAction(SDKs::getName(), new SDKs())

src/Appwrite/Platform/Tasks/Maintenance.php

@@ -92,7 +92,6 @@ class Maintenance extends Action
                 ->trigger();
 
             $this->notifyDeleteConnections($queueForDeletes);
-            $this->renewCertificates($dbForPlatform, $queueForCertificates);
             $this->notifyDeleteCache($cacheRetention, $queueForDeletes);
             $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
             $this->notifyDeleteCSVExports($queueForDeletes);
@@ -114,47 +113,6 @@ class Maintenance extends Action
             ->trigger();
     }
 
-    private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void
-    {
-        $time = DatabaseDateTime::now();
-
-        $certificates = $dbForPlatform->find('certificates', [
-            Query::lessThan('attempts', 5), // Maximum 5 attempts
-            Query::isNotNull('renewDate'),
-            Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew)
-            Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains)
-        ]);
-
-
-        if (\count($certificates) > 0) {
-            Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs.");
-
-            // TODO: (@Meldiron) Remove after 1.7.x migration
-            $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5';
-
-            foreach ($certificates as $certificate) {
-                $domain = $certificate->getAttribute('domain');
-                $rule = $isMd5
-                    ? $dbForPlatform->getDocument('rules', md5($domain))
-                    : $dbForPlatform->findOne('rules', [
-                        Query::equal('domain', [$domain]),
-                    ]);
-
-                if ($rule->isEmpty() || $rule->getAttribute('region') !== System::getEnv('_APP_REGION', 'default')) {
-                    continue;
-                }
-
-                $queueForCertificate
-                    ->setDomain(new Document([
-                        'domain' => $certificate->getAttribute('domain')
-                    ]))
-                    ->trigger();
-            }
-        } else {
-            Console::info("[{$time}] No certificates for renewal.");
-        }
-    }
-
     private function notifyDeleteCache($interval, Delete $queueForDeletes): void
     {
         $queueForDeletes

src/Appwrite/Platform/Tasks/Rules.php

@@ -0,0 +1,126 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Appwrite\Event\Certificate;
+use DateTime;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\DateTime as DatabaseDateTime;
+use Utopia\Database\Document;
+use Utopia\Database\Query;
+use Utopia\Platform\Action;
+use Utopia\System\System;
+
+class Rules extends Action
+{
+    public static function getName(): string
+    {
+        return 'rules';
+    }
+
+    public function __construct()
+    {
+        $this
+          ->desc('Schedules periodic tasks for rule verification and certificate renewal')
+          ->inject('dbForPlatform')
+          ->inject('queueForCertificates')
+          ->callback($this->action(...));
+    }
+
+    public function action(Database $dbForPlatform, Certificate $queueForCertificates): void
+    {
+        Console::title('Interval V1');
+        Console::success(APP_NAME . ' interval process v1 has started');
+
+        $intervalRuleVerification = (int) System::getEnv('_APP_MAINTENANCE_RULE_VERIFICATION_INTERVAL', '60'); // 1 minute
+        $intervalCertificateRenewal = (int) System::getEnv('_APP_MAINTENANCE_CERTIFICATE_RENEWAL_INTERVAL', '86400'); // 1 day
+
+        \go(function () use ($dbForPlatform, $queueForCertificates, $intervalRuleVerification) {
+            Console::loop(function () use ($dbForPlatform, $queueForCertificates) {
+                $this->checkRuleVerification($dbForPlatform, $queueForCertificates);
+            }, $intervalRuleVerification);
+        });
+
+        \go(function () use ($dbForPlatform, $queueForCertificates, $intervalCertificateRenewal) {
+            Console::loop(function () use ($dbForPlatform, $queueForCertificates) {
+                $this->renewCertificates($dbForPlatform, $queueForCertificates);
+            }, $intervalCertificateRenewal);
+        });
+    }
+
+    private function checkRuleVerification(Database $dbForPlatform, Certificate $queueForCertificate): void
+    {
+        $time = DatabaseDateTime::now();
+        $fromTime = new DateTime('-3 days'); // Max 3 days old
+
+        $rules = $dbForPlatform->find('rules', [
+            Query::createdAfter(DatabaseDateTime::format($fromTime)),
+            Query::equal('status', [RULE_STATUS_CREATED]), // Created but not verified yet
+            Query::orderAsc('$updatedAt'), // Pick the ones waiting for another attempt for longest
+            Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), // Only current region
+            Query::limit(30), // Reasonable pagination limit, processable within a minute
+        ]);
+
+        if (\count($rules) === 0) {
+            Console::info("[{$time}] No rules for verification.");
+            return; // No rules to verify
+        }
+
+        Console::info("[{$time}] Found " . \count($rules) . " rules for verification, scheduling jobs.");
+
+        foreach ($rules as $rule) {
+            $queueForCertificate
+                ->setDomain(new Document([
+                    'domain' => $rule->getAttribute('domain'),
+                    'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')),
+                ]))
+                ->setAction(Certificate::ACTION_DOMAIN_VERIFICATION)
+                ->trigger();
+        }
+    }
+
+    private function renewCertificates(Database $dbForPlatform, Certificate $queueForCertificate): void
+    {
+        $time = DatabaseDateTime::now();
+
+        $certificates = $dbForPlatform->find('certificates', [
+            Query::lessThan('attempts', 5), // Maximum 5 attempts
+            Query::isNotNull('renewDate'),
+            Query::lessThanEqual('renewDate', $time), // includes 60 days cooldown (we have 30 days to renew)
+            Query::limit(200), // Limit 200 comes from LetsEncrypt (300 orders per 3 hours, keeping some for new domains)
+        ]);
+
+        if (\count($certificates) === 0) {
+            Console::info("[{$time}] No certificates for renewal.");
+            return;
+        }
+
+        Console::info("[{$time}] Found " . \count($certificates) . " certificates for renewal, scheduling jobs.");
+
+        $isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5';
+        $appRegion = System::getEnv('_APP_REGION', 'default');
+
+        foreach ($certificates as $certificate) {
+            $domain = $certificate->getAttribute('domain');
+            $rule = $isMd5 ?
+                $dbForPlatform->getDocument('rules', md5($domain)) :
+                    $dbForPlatform->findOne('rules', [
+                        Query::equal('domain', [$domain]),
+                        Query::limit(1)
+                    ]);
+
+            if ($rule->isEmpty() || $rule->getAttribute('region') !== $appRegion) {
+                continue;
+            }
+
+            $queueForCertificate
+                ->setDomain(new Document([
+                    'domain' => $rule->getAttribute('domain'),
+                    'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')),
+                ]))
+                ->setAction(Certificate::ACTION_GENERATION)
+                ->trigger();
+        }
+    }
+}

src/Appwrite/Platform/Workers/Certificates.php

@@ -3,11 +3,13 @@
 namespace Appwrite\Platform\Workers;
 
 use Appwrite\Certificates\Adapter as CertificatesAdapter;
+use Appwrite\Event\Certificate;
 use Appwrite\Event\Event;
 use Appwrite\Event\Func;
 use Appwrite\Event\Mail;
 use Appwrite\Event\Realtime;
 use Appwrite\Event\Webhook;
+use Appwrite\Extend\Exception as AppwriteException;
 use Appwrite\Platform\Modules\Proxy\Action;
 use Appwrite\Template\Template;
 use Appwrite\Utopia\Response\Model\Rule;
@@ -20,9 +22,9 @@ use Utopia\Database\Document;
 use Utopia\Database\Exception\Authorization;
 use Utopia\Database\Exception\Conflict;
 use Utopia\Database\Exception\Structure;
-use Utopia\Database\Validator\Authorization as ValidatorAuthorization;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Query;
+use Utopia\Database\Validator\Authorization as ValidatorAuthorization;
 use Utopia\Domains\Domain;
 use Utopia\Locale\Locale;
 use Utopia\Logger\Log;
@@ -52,6 +54,7 @@ class Certificates extends Action
             ->inject('queueForWebhooks')
             ->inject('queueForFunctions')
             ->inject('queueForRealtime')
+            ->inject('queueForCertificates')
             ->inject('log')
             ->inject('certificates')
             ->inject('plan')
@@ -66,6 +69,7 @@ class Certificates extends Action
      * @param Webhook $queueForWebhooks
      * @param Func $queueForFunctions
      * @param Realtime $queueForRealtime
+     * @param Certificate $queueForCertificates
      * @param Log $log
      * @param CertificatesAdapter $certificates
      * @return void
@@ -80,6 +84,7 @@ class Certificates extends Action
         Webhook $queueForWebhooks,
         Func $queueForFunctions,
         Realtime $queueForRealtime,
+        Certificate $queueForCertificates,
         Log $log,
         CertificatesAdapter $certificates,
         array $plan
@@ -95,10 +100,88 @@ class Certificates extends Action
         $domainType = $document->getAttribute('domainType');
         $skipRenewCheck = $payload['skipRenewCheck'] ?? false;
         $validationDomain = $payload['validationDomain'] ?? null;
+        $action = $payload['action'] ?? Certificate::ACTION_GENERATION;
 
         $log->addTag('domain', $domain->get());
 
-        $this->execute($domain, $domainType, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck, $plan, $validationDomain);
+        switch ($action) {
+            case Certificate::ACTION_DOMAIN_VERIFICATION:
+                $this->handleDomainVerificationAction($domain, $dbForPlatform, $log, $queueForCertificates, $validationDomain);
+                break;
+
+            case Certificate::ACTION_GENERATION:
+                $this->handleCertificateGenerationAction($domain, $domainType, $dbForPlatform, $queueForMails, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $log, $certificates, $skipRenewCheck, $plan, $validationDomain);
+                break;
+
+            default:
+                throw new Exception('Invalid action: ' . $action);
+        }
+
+
+    }
+
+    /**
+     * @param Domain $domain
+     * @param Database $dbForPlatform
+     * @param Log $log
+     * @param Certificate $queueForCertificates
+     * @return void
+     * @throws Throwable
+     * @throws \Utopia\Database\Exception
+     */
+    private function handleDomainVerificationAction(
+        Domain $domain,
+        Database $dbForPlatform,
+        Log $log,
+        Certificate $queueForCertificates,
+        ?string $validationDomain = null,
+    ): void {
+        // Get rule
+        $rule = System::getEnv('_APP_RULES_FORMAT') === 'md5'
+            ? ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('rules', md5($domain->get())))
+            : ValidatorAuthorization::skip(fn () => $dbForPlatform->findOne('rules', [
+                Query::equal('domain', [$domain->get()]),
+                Query::limit(1),
+            ]));
+
+        // Skip if rule is not desired state (created but not verified yet).
+        if ($rule->getAttribute('status', '') !== RULE_STATUS_CREATED) {
+            Console::warning('Domain verification for ' . $rule->getAttribute('domain', '') . ' is not needed.');
+            return;
+        }
+
+        Console::info('Domain verification for ' . $rule->getAttribute('domain', '') . ' started.');
+
+        $updates = new Document();
+        try {
+            // Verify DNS records
+            $this->validateDomain($rule, $domain, $log, $validationDomain);
+            // Reset logs and status for the rule
+            $updates
+                ->setAttribute('logs', '')
+                ->setAttribute('status', RULE_STATUS_CERTIFICATE_GENERATING);
+
+            Console::success('Domain verification succeeded.');
+        } catch (AppwriteException $err) {
+            Console::warning('Domain verification failed: ' . $err->getMessage());
+            $updates->setAttribute('logs', $err->getMessage());
+        }
+
+        echo "updating rule with updates: " . \var_dump($updates);
+        $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), $updates);
+
+        // Issue a TLS certificate when domain is verified
+        if ($rule->getAttribute('status', '') === RULE_STATUS_CERTIFICATE_GENERATING) {
+            $queueForCertificates
+                ->setDomain(new Document([
+                    'domain' => $rule->getAttribute('domain'),
+                    'domainType' => $rule->getAttribute('deploymentResourceType', $rule->getAttribute('type')),
+                ]))
+                ->setAction(Certificate::ACTION_GENERATION)
+                ->trigger();
+
+            Console::success('Certificate generation triggered successfully.');
+        }
     }
 
     /**
@@ -117,7 +200,7 @@ class Certificates extends Action
      * @throws Throwable
      * @throws \Utopia\Database\Exception
      */
-    private function execute(
+    private function handleCertificateGenerationAction(
         Domain $domain,
         ?string $domainType,
         Database $dbForPlatform,
@@ -172,7 +255,7 @@ class Certificates extends Action
 
         // Rule not found (or) not in the expected state
         if ($rule->isEmpty() || $rule->getAttribute('status') !== RULE_STATUS_CERTIFICATE_GENERATING) {
-            Console::warning('Certificate generation for ' . $domain . ' is skipped as the associated rule is either empty or not in the expected state.');
+            Console::warning('Certificate generation for ' . $domain->get() . ' is skipped as the associated rule is either empty or not in the expected state.');
         }
 
         // Get associated certificate for the rule
@@ -195,7 +278,7 @@ class Certificates extends Action
 
             // Validate domain and DNS records. Skip if job is forced
             if (!$skipRenewCheck) {
-                $this->validateDomain($rule, $domain, $validationDomain, $log);
+                $this->validateDomain($rule, $domain, $log, $validationDomain);
 
                 // If certificate exists already, double-check expiry date. Skip if job is forced
                 if (!$certificates->isRenewRequired($domain->get(), $domainType, $log)) {
@@ -355,13 +438,13 @@ class Certificates extends Action
      *
      * @param Document $rule Rule to validate
      * @param Domain $domain Domain to validate
-     * @param string|null $validationDomain Override for main domain check
      * @param Log $log Logger for adding metrics
+     * @param string|null $validationDomain Override for main domain check
      *
      * @return void
      * @throws Exception
      */
-    private function validateDomain(Document $rule, Domain $domain, ?string $validationDomain = null, Log $log): void
+    private function validateDomain(Document $rule, Domain $domain, Log $log, ?string $validationDomain = null): void
     {
         $mainDomain = $validationDomain ?? $this->getMainDomain();
         $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain;