failed request log

2026-05-26 13:51:13 +00:00 · 2022-10-18 18:42:41 +03:00
parent 70102a18f7
commit 462b30b721
13 changed files with 263 additions and 77 deletions
@@ -18,6 +18,62 @@ $auth = Config::getParam('auth', []);
 */

 $collections = [
+    'syncs' => [
+        '$collection' => ID::custom(Database::METADATA),
+        '$id' => ID::custom('syncs'),
+        'name' => 'Syncs',
+        'attributes' => [
+            [
+                '$id' => ID::custom('region'),
+                'type' => Database::VAR_STRING,
+                'size' => 256,
+                'required' => true,
+                'signed' => true,
+                'array' => false,
+                'filters' => [],
+            ],
+            [
+                '$id' => ID::custom('keys'),
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 16384,
+                'signed' => true,
+                'required' => true,
+                'default' => [],
+                'array' => false,
+                'filters' => ['json'],
+            ],
+            [
+                '$id' => ID::custom('requestedAt'),
+                'type' => Database::VAR_DATETIME,
+                'format' => '',
+                'size' => 0,
+                'signed' => false,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => ['datetime'],
+            ],
+            [
+                '$id' => ID::custom('status'),
+                'type' => Database::VAR_INTEGER,
+                'size' => 256,
+                'required' => true,
+                'signed' => true,
+                'array' => false,
+                'filters' => [],
+            ],
+        ],
+        'indexes' => [
+            [
+                '$id' => ID::custom('_key_requestedAt_status'),
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['requestedAt', 'status'],
+                'lengths' => [],
+                'orders' => [],
+            ],
+        ],
+    ],
    'databases' => [
        '$collection' => ID::custom(Database::METADATA),
        '$id' => ID::custom('databases'),
@@ -18,13 +18,13 @@ App::post('/v1/syncs')
    ->inject('response')
    ->action(function (array $keys, Request $request, Response $response) {

-        if (empty($keys)) {
+        //if (empty($keys)) {
            throw new Exception(Exception::KEY_NOT_FOUND);
-        }
+        //}

        $token = $request->getHeader('authorization');
        $token = str_replace(["Bearer"," "], "", $token);
-        $jwt = new JWT(App::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 900, 10);
+        $jwt = new JWT(App::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 600, 10);
        try {
            $payload = $jwt->decode($token);
        } catch (JWTException $error) {
@@ -154,6 +154,7 @@ const DELETE_TYPE_BUCKETS = 'buckets';
 const DELETE_TYPE_SESSIONS = 'sessions';
 const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp';
 const DELETE_TYPE_CACHE_BY_RESOURCE  = 'cacheByResource';
+const DELETE_TYPE_SYNCS = 'syncs';
 // Compression type
 const COMPRESSION_TYPE_NONE = 'none';
 const COMPRESSION_TYPE_GZIP = 'gzip';
@@ -930,6 +931,10 @@ $register->set('syncOut', function () {
    return new SyncOut();
 });

+$register->set('deletes', function () {
+    return new Delete();
+});
+
 App::setResource('dbForProject', function ($db, $cache, Document $project, $register) {

    $cache = new Cache(new RedisCache($cache));
@@ -940,12 +945,19 @@ App::setResource('dbForProject', function ($db, $cache, Document $project, $regi
            ->trigger();
    });

-//    $cache->on(cache::EVENT_PURGE, function ($key) use ($register) {
-//        $register
-//            ->get('syncOut')
-//            ->addKey($key)
-//            ->trigger();
-//    });
+    $cache->on(cache::EVENT_PURGE, function ($key) use ($register) {
+        $register
+            ->get('syncOut')
+            ->addKey($key)
+            ->trigger();
+    });
+
+    $cache->on(cache::EVENT_FLUSH, function ($region) use ($register) {
+        $register
+            ->get('deletes')
+            ->setRegion($region)
+            ->trigger();
+    });

    $database = new Database(new MariaDB($db), $cache);
    $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
@@ -958,19 +970,26 @@ App::setResource('dbForConsole', function ($db, $cache, $register) {

    $cache = new Cache(new RedisCache($cache));

-//    $cache->on(cache::EVENT_SAVE, function ($key) use ($register) {
-//        $register
-//            ->get('syncOut')
-//            ->addKey($key)
-//            ->trigger();
-//    });
-//
-//    $cache->on(cache::EVENT_PURGE, function ($key) use ($register) {
-//        $register
-//            ->get('syncOut')
-//            ->addKey($key)
-//            ->trigger();
-//    });
+    $cache->on(cache::EVENT_SAVE, function ($key) use ($register) {
+        $register
+            ->get('syncOut')
+            ->addKey($key)
+            ->trigger();
+    });
+
+    $cache->on(cache::EVENT_PURGE, function ($key) use ($register) {
+        $register
+            ->get('syncOut')
+            ->addKey($key)
+            ->trigger();
+    });
+
+    $cache->on(cache::EVENT_FLUSH, function ($region) use ($register) {
+        $register
+            ->get('deletes')
+            ->setRegion($region)
+            ->trigger();
+    });

    $database = new Database(new MariaDB($db), $cache);
    $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
@@ -35,6 +35,7 @@ foreach (
    realpath(__DIR__ . '/../vendor/symfony'),
    realpath(__DIR__ . '/../vendor/mongodb'),
    realpath(__DIR__ . '/../vendor/utopia-php/websocket'), // TODO: remove workerman autoload
+    realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: Remove when memcached ext issue get fixed
    ] as $key => $value
 ) {
    if ($value !== false) {
@@ -171,6 +171,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
     */
    Timer::tick(5000, function () use ($register, $stats, &$statsDocument, $logError) {
        $payload = [];
+
        foreach ($stats as $projectId => $value) {
            $payload[$projectId] = $stats->get($projectId, 'connectionsTotal');
        }
@@ -187,6 +188,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume

            Authorization::skip(fn () => $database->updateDocument('realtime', $statsDocument->getId(), $statsDocument));
        } catch (\Throwable $th) {
+
            call_user_func($logError, $th, "updateWorkerDocument");
        } finally {
            call_user_func($returnDatabase);
@@ -6,6 +6,7 @@ global $register;
 use Appwrite\Auth\Auth;
 use Appwrite\Event\Certificate;
 use Appwrite\Event\Delete;
+use Appwrite\Event\SyncOut;
 use Utopia\App;
 use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
@@ -54,7 +55,7 @@ $cli
        Console::title('Maintenance V1');
        Console::success(APP_NAME . ' maintenance process v1 has started');

-        function notifyDeleteExecutionLogs(int $interval)
+        function notifyDeleteExecutionLogs(int $interval): void
        {
            (new Delete())
                ->setType(DELETE_TYPE_EXECUTIONS)
@@ -62,7 +63,7 @@ $cli
                ->trigger();
        }

-        function notifyDeleteAbuseLogs(int $interval)
+        function notifyDeleteAbuseLogs(int $interval): void
        {
            (new Delete())
                ->setType(DELETE_TYPE_ABUSE)
@@ -70,7 +71,7 @@ $cli
                ->trigger();
        }

-        function notifyDeleteAuditLogs(int $interval)
+        function notifyDeleteAuditLogs(int $interval): void
        {
            (new Delete())
                ->setType(DELETE_TYPE_AUDIT)
@@ -78,7 +79,7 @@ $cli
                ->trigger();
        }

-        function notifyDeleteUsageStats(int $interval30m, int $interval1d)
+        function notifyDeleteUsageStats(int $interval30m, int $interval1d): void
        {
            (new Delete())
                ->setType(DELETE_TYPE_USAGE)
@@ -87,7 +88,7 @@ $cli
                ->trigger();
        }

-        function notifyDeleteConnections()
+        function notifyDeleteConnections(): void
        {
            (new Delete())
                ->setType(DELETE_TYPE_REALTIME)
@@ -95,7 +96,7 @@ $cli
                ->trigger();
        }

-        function notifyDeleteExpiredSessions()
+        function notifyDeleteExpiredSessions(): void
        {
            (new Delete())
                ->setType(DELETE_TYPE_SESSIONS)
@@ -103,7 +104,7 @@ $cli
                ->trigger();
        }

-        function renewCertificates($dbForConsole)
+        function renewCertificates($dbForConsole): void
        {
            $time = DateTime::now();

@@ -139,6 +140,29 @@ $cli
                ->trigger();
        }

+        function syncRegionalCache($dbForConsole): void
+        {
+            $time = DateTime::now();
+
+            $chunks = $dbForConsole->find('syncs', [
+                Query::notEqual('status', 200),
+                Query::limit(300)
+            ]);
+
+            if (\count($chunks) > 0) {
+                Console::info("[{$time}] Found " . \count($chunks) . " cache chunks to purge.");
+                foreach ($chunks as $chunk) {
+                    $keys = $chunk->getAttribute('keys');
+//                    (new SyncOut())
+ //                       ->setRegion($chunk->getAttribute('region'))
+//                        ->addKey($key)
+//                        ->trigger();
+                }
+            } else {
+                Console::info("[{$time}] No certificates for renewal.");
+            }
+        }
+
        // # of days in seconds (1 day = 86400s)
        $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
        $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
@@ -147,8 +171,9 @@ $cli
        $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours
        $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days
        $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
+        $regionalCacheSyncRetention = (int) App::getEnv('_APP_MAINTENANCE_CACHE_SYNC', '300'); // 5 minutes

-        Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention) {
+        Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention, $regionalCacheSyncRetention) {
            $database = getConsoleDB();

            $time = DateTime::now();
@@ -162,5 +187,6 @@ $cli
            notifyDeleteExpiredSessions();
            renewCertificates($database);
            notifyDeleteCache($cacheRetention);
+            syncRegionalCache($database);
        }, $interval);
    });
@@ -114,6 +114,9 @@ class DeletesV1 extends Worker
            case DELETE_TYPE_CACHE_BY_TIMESTAMP:
                $this->deleteCacheByDate();
                break;
+            case DELETE_TYPE_SYNCS:
+                $this->deleteRegionalCache();
+                break;
            default:
                Console::error('No delete operation for type: ' . $type);
                break;
@@ -675,4 +678,11 @@ class DeletesV1 extends Worker

        $device->deletePath($document->getId());
    }
+
+    protected function deleteRegionalCache()
+    {
+        $this->deleteByGroup('syncs', [
+            Query::equal('region', [$this->args['region']])
+        ], $this->getConsoleDB);
+    }
 }
@@ -4,6 +4,8 @@ use Ahc\Jwt\JWT;
 use Appwrite\Resque\Worker;
 use Utopia\App;
 use Utopia\CLI\Console;
+use Utopia\Database\DateTime;
+use Utopia\Database\Document;

 require_once __DIR__ . '/../init.php';

@@ -12,8 +14,6 @@ Console::success(APP_NAME . ' syncs out worker v1 has started');

 class SyncsOutV1 extends Worker
 {
-    protected array $errors = [];
-
    private array $regions = [
       'fra1' => '172.17.0.1',
       'nyc1' => '172.17.0.1',
@@ -35,36 +35,57 @@ class SyncsOutV1 extends Worker
        $currentRegion = 'nyc1';

        $data['keys'][] = $this->args['key'];
-        $jwt = new JWT(App::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 900, 10);
+        $jwt = new JWT(App::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 600, 10);
        $token = $jwt->encode($data);

+        if (!empty($this->args['region'])) {
+            $this->regions = $this->regions[$this->args['region']];
+        }
+
        foreach ($this->regions as $region => $host) {
            if ($currentRegion === $region) {
                continue;
            }
-                $ch = curl_init($host . '/v1/syncs');
-                curl_setopt($ch, CURLOPT_HTTPHEADER, [
-                    'Authorization: Bearer ' . $token,
-                    'Content-Type: application/json'
-                ]);
-                curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
-                curl_setopt($ch, CURLOPT_TIMEOUT, 5);
-                curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
-                curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data));

-            for ($attempts = 0; $attempts < 6; $attempts++) {
-                    curl_exec($ch);
-                    $responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE);
-                if ($responseStatus === 200) {
-                    break;
-                }
-
-                sleep(2);
+            $status = $this->send($host, $token, $data);
+            if ($status !== 200) {
+                $this->getConsoleDB()->createDocument('syncs', new Document([
+                    'requestedAt' => DateTime::now(),
+                    'region' => $region,
+                    'keys' => $data,
+                    'status' => $status,
+                ]));
            }
-                curl_close($ch);
        }
    }

+    private function send($host, $token, $data): int
+    {
+
+        $ch = curl_init($host . '/v1/syncs');
+        curl_setopt($ch, CURLOPT_HTTPHEADER, [
+            'Authorization: Bearer ' . $token,
+            'Content-Type: application/json'
+        ]);
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+        curl_setopt($ch, CURLOPT_TIMEOUT, 5);
+        curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
+        curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data));
+
+        for ($attempts = 0; $attempts < 3; $attempts++) {
+            curl_exec($ch);
+            $responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+
+            if ($responseStatus === 200) {
+                return $responseStatus;
+            }
+
+            sleep(2);
+        }
+        curl_close($ch);
+        return $responseStatus;
+    }
+

    public function shutdown(): void
    {
@@ -48,7 +48,7 @@
        "utopia-php/abuse": "0.14.*",
        "utopia-php/analytics": "0.2.*",
        "utopia-php/audit": "0.15.*",
-        "utopia-php/cache": "0.6.*",
+        "utopia-php/cache": "dev-feat-redis-sync as 0.6.1",
        "utopia-php/cli": "0.13.*",
        "utopia-php/config": "0.2.*",
        "utopia-php/database": "0.26.*",
@@ -4,7 +4,7 @@
        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
        "This file is @generated automatically"
    ],
-    "content-hash": "08fdd139ad1285b02c4b4e555679e7de",
+    "content-hash": "1ceb54089b20b863e8685e1644ba51f9",
    "packages": [
        {
            "name": "adhocore/jwt",
@@ -1897,24 +1897,26 @@
        },
        {
            "name": "utopia-php/cache",
-            "version": "0.6.1",
+            "version": "dev-feat-redis-sync",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/cache.git",
-                "reference": "9889235a6d3da6cbb1f435201529da4d27c30e79"
+                "reference": "99e7085eb229d0f0159a4f2107ea5ea123f7b32b"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/cache/zipball/9889235a6d3da6cbb1f435201529da4d27c30e79",
-                "reference": "9889235a6d3da6cbb1f435201529da4d27c30e79",
+                "url": "https://api.github.com/repos/utopia-php/cache/zipball/99e7085eb229d0f0159a4f2107ea5ea123f7b32b",
+                "reference": "99e7085eb229d0f0159a4f2107ea5ea123f7b32b",
                "shasum": ""
            },
            "require": {
                "ext-json": "*",
+                "ext-memcached": "*",
                "ext-redis": "*",
                "php": ">=8.0"
            },
            "require-dev": {
+                "laravel/pint": "1.2.*",
                "phpunit/phpunit": "^9.3",
                "vimeo/psalm": "4.13.1"
            },
@@ -1928,12 +1930,6 @@
            "license": [
                "MIT"
            ],
-            "authors": [
-                {
-                    "name": "Eldad Fux",
-                    "email": "eldad@appwrite.io"
-                }
-            ],
            "description": "A simple cache library to manage application cache storing, loading and purging",
            "keywords": [
                "cache",
@@ -1944,9 +1940,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/cache/issues",
-                "source": "https://github.com/utopia-php/cache/tree/0.6.1"
+                "source": "https://github.com/utopia-php/cache/tree/feat-redis-sync"
            },
-            "time": "2022-08-10T08:12:46+00:00"
+            "time": "2022-10-18T06:58:42+00:00"
        },
        {
            "name": "utopia-php/cli",
@@ -3413,25 +3409,30 @@
        },
        {
            "name": "phpdocumentor/type-resolver",
-            "version": "1.6.1",
+            "version": "1.6.2",
            "source": {
                "type": "git",
                "url": "https://github.com/phpDocumentor/TypeResolver.git",
-                "reference": "77a32518733312af16a44300404e945338981de3"
+                "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/77a32518733312af16a44300404e945338981de3",
-                "reference": "77a32518733312af16a44300404e945338981de3",
+                "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/48f445a408c131e38cab1c235aa6d2bb7a0bb20d",
+                "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d",
                "shasum": ""
            },
            "require": {
-                "php": "^7.2 || ^8.0",
+                "php": "^7.4 || ^8.0",
                "phpdocumentor/reflection-common": "^2.0"
            },
            "require-dev": {
                "ext-tokenizer": "*",
-                "psalm/phar": "^4.8"
+                "phpstan/extension-installer": "^1.1",
+                "phpstan/phpstan": "^1.8",
+                "phpstan/phpstan-phpunit": "^1.1",
+                "phpunit/phpunit": "^9.5",
+                "rector/rector": "^0.13.9",
+                "vimeo/psalm": "^4.25"
            },
            "type": "library",
            "extra": {
@@ -3457,9 +3458,9 @@
            "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names",
            "support": {
                "issues": "https://github.com/phpDocumentor/TypeResolver/issues",
-                "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.1"
+                "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.2"
            },
-            "time": "2022-03-15T21:29:03+00:00"
+            "time": "2022-10-14T12:47:21+00:00"
        },
        {
            "name": "phpspec/prophecy",
@@ -5352,9 +5353,18 @@
            "time": "2022-09-28T08:42:51+00:00"
        }
    ],
-    "aliases": [],
+    "aliases": [
+        {
+            "package": "utopia-php/cache",
+            "version": "dev-feat-redis-sync",
+            "alias": "0.6.1",
+            "alias_normalized": "0.6.1.0"
+        }
+    ],
    "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {
+        "utopia-php/cache": 20
+    },
    "prefer-stable": false,
    "prefer-lowest": false,
    "platform": {
@@ -259,6 +259,7 @@ services:
    entrypoint: worker-syncs-out
    <<: *x-logging
    container_name: appwrite-worker-syncs-out
+    image: appwrite-dev
    build:
      context: .
    networks:
@@ -266,8 +267,8 @@ services:
    volumes:
      - ./app:/usr/src/code/app
      - ./src:/usr/src/code/src
-      - ./vendor/utopia-php/cache:/usr/src/code/vendor/utopia-php/cache
    depends_on:
+      - mariadb
      - redis
    environment:
        - _APP_ENV
@@ -275,11 +276,17 @@ services:
        - _APP_REDIS_PORT
        - _APP_REDIS_USER
        - _APP_REDIS_PASS
+        - _APP_DB_HOST
+        - _APP_DB_PORT
+        - _APP_DB_SCHEMA
+        - _APP_DB_USER
+        - _APP_DB_PASS

  appwrite-worker-syncs-in:
    entrypoint: worker-syncs-in
    <<: *x-logging
    container_name: appwrite-worker-syncs-in
+    image: appwrite-dev
    build:
     context: .
    networks:
@@ -287,7 +294,6 @@ services:
    volumes:
      - ./app:/usr/src/code/app
      - ./src:/usr/src/code/src
-      - ./vendor/utopia-php/cache:/usr/src/code/vendor/utopia-php/cache
    depends_on:
      - redis
    environment:
@@ -13,6 +13,7 @@ class Delete extends Event
    protected ?string $datetime = null;
    protected ?string $dateTime30m = null;
    protected ?string $dateTime1d = null;
+    protected ?string $region = null;


    public function __construct()
@@ -92,6 +93,16 @@ class Delete extends Event
        return $this;
    }

+    /**
+     * Sets cloud region.
+     *
+     * @param string $region
+     */
+    public function setRegion($region): void
+    {
+        $this->region = $region;
+    }
+
    /**
     * Returns the resource for the delete event.
     *
@@ -7,6 +7,7 @@ use Resque;
 class SyncOut extends Event
 {
    protected string $key = '';
+    protected string $region = '';

    public function __construct()
    {
@@ -36,6 +37,29 @@ class SyncOut extends Event
        return $this->key;
    }

+    /**
+     * Sets cloud region.
+     *
+     * @param string $region
+     * @return self
+     */
+    public function setRegion(string $region): self
+    {
+        $this->host = $region;
+
+        return $this;
+    }
+
+    /**
+     * Returns cloud region.
+     *
+     * @return string
+     */
+    public function getRegion(): string
+    {
+        return $this->region;
+    }
+
    /**
     * Executes the event and sends it to the messaging worker.
     *