From 462b30b721ab32addc6276eb274e6262d417bc9d Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 18 Oct 2022 18:42:41 +0300 Subject: [PATCH] failed request log --- app/config/collections.php | 56 ++++++++++++++++++++++++++++++ app/controllers/api/syncs.php | 6 ++-- app/init.php | 57 ++++++++++++++++++++---------- app/preload.php | 1 + app/realtime.php | 2 ++ app/tasks/maintenance.php | 42 ++++++++++++++++++----- app/workers/deletes.php | 10 ++++++ app/workers/syncsOut.php | 63 ++++++++++++++++++++++------------ composer.json | 2 +- composer.lock | 56 +++++++++++++++++------------- docker-compose.yml | 10 ++++-- src/Appwrite/Event/Delete.php | 11 ++++++ src/Appwrite/Event/SyncOut.php | 24 +++++++++++++ 13 files changed, 263 insertions(+), 77 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index d8f65da788..58d00c85aa 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -18,6 +18,62 @@ $auth = Config::getParam('auth', []); */ $collections = [ + 'syncs' => [ + '$collection' => ID::custom(Database::METADATA), + '$id' => ID::custom('syncs'), + 'name' => 'Syncs', + 'attributes' => [ + [ + '$id' => ID::custom('region'), + 'type' => Database::VAR_STRING, + 'size' => 256, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('keys'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => true, + 'default' => [], + 'array' => false, + 'filters' => ['json'], + ], + [ + '$id' => ID::custom('requestedAt'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => ID::custom('status'), + 'type' => Database::VAR_INTEGER, + 'size' => 256, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => ID::custom('_key_requestedAt_status'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['requestedAt', 'status'], + 'lengths' => [], + 'orders' => [], + ], + ], + ], 'databases' => [ '$collection' => ID::custom(Database::METADATA), '$id' => ID::custom('databases'), diff --git a/app/controllers/api/syncs.php b/app/controllers/api/syncs.php index e6181de62e..ccd6f82c8f 100644 --- a/app/controllers/api/syncs.php +++ b/app/controllers/api/syncs.php @@ -18,13 +18,13 @@ App::post('/v1/syncs') ->inject('response') ->action(function (array $keys, Request $request, Response $response) { - if (empty($keys)) { + //if (empty($keys)) { throw new Exception(Exception::KEY_NOT_FOUND); - } + //} $token = $request->getHeader('authorization'); $token = str_replace(["Bearer"," "], "", $token); - $jwt = new JWT(App::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 900, 10); + $jwt = new JWT(App::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 600, 10); try { $payload = $jwt->decode($token); } catch (JWTException $error) { diff --git a/app/init.php b/app/init.php index e621956c15..efb6554c54 100644 --- a/app/init.php +++ b/app/init.php @@ -154,6 +154,7 @@ const DELETE_TYPE_BUCKETS = 'buckets'; const DELETE_TYPE_SESSIONS = 'sessions'; const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp'; const DELETE_TYPE_CACHE_BY_RESOURCE = 'cacheByResource'; +const DELETE_TYPE_SYNCS = 'syncs'; // Compression type const COMPRESSION_TYPE_NONE = 'none'; const COMPRESSION_TYPE_GZIP = 'gzip'; @@ -930,6 +931,10 @@ $register->set('syncOut', function () { return new SyncOut(); }); +$register->set('deletes', function () { + return new Delete(); +}); + App::setResource('dbForProject', function ($db, $cache, Document $project, $register) { $cache = new Cache(new RedisCache($cache)); @@ -940,12 +945,19 @@ App::setResource('dbForProject', function ($db, $cache, Document $project, $regi ->trigger(); }); -// $cache->on(cache::EVENT_PURGE, function ($key) use ($register) { -// $register -// ->get('syncOut') -// ->addKey($key) -// ->trigger(); -// }); + $cache->on(cache::EVENT_PURGE, function ($key) use ($register) { + $register + ->get('syncOut') + ->addKey($key) + ->trigger(); + }); + + $cache->on(cache::EVENT_FLUSH, function ($region) use ($register) { + $register + ->get('deletes') + ->setRegion($region) + ->trigger(); + }); $database = new Database(new MariaDB($db), $cache); $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); @@ -958,19 +970,26 @@ App::setResource('dbForConsole', function ($db, $cache, $register) { $cache = new Cache(new RedisCache($cache)); -// $cache->on(cache::EVENT_SAVE, function ($key) use ($register) { -// $register -// ->get('syncOut') -// ->addKey($key) -// ->trigger(); -// }); -// -// $cache->on(cache::EVENT_PURGE, function ($key) use ($register) { -// $register -// ->get('syncOut') -// ->addKey($key) -// ->trigger(); -// }); + $cache->on(cache::EVENT_SAVE, function ($key) use ($register) { + $register + ->get('syncOut') + ->addKey($key) + ->trigger(); + }); + + $cache->on(cache::EVENT_PURGE, function ($key) use ($register) { + $register + ->get('syncOut') + ->addKey($key) + ->trigger(); + }); + + $cache->on(cache::EVENT_FLUSH, function ($region) use ($register) { + $register + ->get('deletes') + ->setRegion($region) + ->trigger(); + }); $database = new Database(new MariaDB($db), $cache); $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); diff --git a/app/preload.php b/app/preload.php index bf8b0bfd1d..fb63a381b6 100644 --- a/app/preload.php +++ b/app/preload.php @@ -35,6 +35,7 @@ foreach ( realpath(__DIR__ . '/../vendor/symfony'), realpath(__DIR__ . '/../vendor/mongodb'), realpath(__DIR__ . '/../vendor/utopia-php/websocket'), // TODO: remove workerman autoload + realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: Remove when memcached ext issue get fixed ] as $key => $value ) { if ($value !== false) { diff --git a/app/realtime.php b/app/realtime.php index be87c3d6e6..2263f3c396 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -171,6 +171,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume */ Timer::tick(5000, function () use ($register, $stats, &$statsDocument, $logError) { $payload = []; + foreach ($stats as $projectId => $value) { $payload[$projectId] = $stats->get($projectId, 'connectionsTotal'); } @@ -187,6 +188,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume Authorization::skip(fn () => $database->updateDocument('realtime', $statsDocument->getId(), $statsDocument)); } catch (\Throwable $th) { + call_user_func($logError, $th, "updateWorkerDocument"); } finally { call_user_func($returnDatabase); diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 42b5ed00dc..c60a7f24dd 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -6,6 +6,7 @@ global $register; use Appwrite\Auth\Auth; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; +use Appwrite\Event\SyncOut; use Utopia\App; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -54,7 +55,7 @@ $cli Console::title('Maintenance V1'); Console::success(APP_NAME . ' maintenance process v1 has started'); - function notifyDeleteExecutionLogs(int $interval) + function notifyDeleteExecutionLogs(int $interval): void { (new Delete()) ->setType(DELETE_TYPE_EXECUTIONS) @@ -62,7 +63,7 @@ $cli ->trigger(); } - function notifyDeleteAbuseLogs(int $interval) + function notifyDeleteAbuseLogs(int $interval): void { (new Delete()) ->setType(DELETE_TYPE_ABUSE) @@ -70,7 +71,7 @@ $cli ->trigger(); } - function notifyDeleteAuditLogs(int $interval) + function notifyDeleteAuditLogs(int $interval): void { (new Delete()) ->setType(DELETE_TYPE_AUDIT) @@ -78,7 +79,7 @@ $cli ->trigger(); } - function notifyDeleteUsageStats(int $interval30m, int $interval1d) + function notifyDeleteUsageStats(int $interval30m, int $interval1d): void { (new Delete()) ->setType(DELETE_TYPE_USAGE) @@ -87,7 +88,7 @@ $cli ->trigger(); } - function notifyDeleteConnections() + function notifyDeleteConnections(): void { (new Delete()) ->setType(DELETE_TYPE_REALTIME) @@ -95,7 +96,7 @@ $cli ->trigger(); } - function notifyDeleteExpiredSessions() + function notifyDeleteExpiredSessions(): void { (new Delete()) ->setType(DELETE_TYPE_SESSIONS) @@ -103,7 +104,7 @@ $cli ->trigger(); } - function renewCertificates($dbForConsole) + function renewCertificates($dbForConsole): void { $time = DateTime::now(); @@ -139,6 +140,29 @@ $cli ->trigger(); } + function syncRegionalCache($dbForConsole): void + { + $time = DateTime::now(); + + $chunks = $dbForConsole->find('syncs', [ + Query::notEqual('status', 200), + Query::limit(300) + ]); + + if (\count($chunks) > 0) { + Console::info("[{$time}] Found " . \count($chunks) . " cache chunks to purge."); + foreach ($chunks as $chunk) { + $keys = $chunk->getAttribute('keys'); +// (new SyncOut()) + // ->setRegion($chunk->getAttribute('region')) +// ->addKey($key) +// ->trigger(); + } + } else { + Console::info("[{$time}] No certificates for renewal."); + } + } + // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); @@ -147,8 +171,9 @@ $cli $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days + $regionalCacheSyncRetention = (int) App::getEnv('_APP_MAINTENANCE_CACHE_SYNC', '300'); // 5 minutes - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention) { + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention, $regionalCacheSyncRetention) { $database = getConsoleDB(); $time = DateTime::now(); @@ -162,5 +187,6 @@ $cli notifyDeleteExpiredSessions(); renewCertificates($database); notifyDeleteCache($cacheRetention); + syncRegionalCache($database); }, $interval); }); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b015043b1d..06205a961f 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -114,6 +114,9 @@ class DeletesV1 extends Worker case DELETE_TYPE_CACHE_BY_TIMESTAMP: $this->deleteCacheByDate(); break; + case DELETE_TYPE_SYNCS: + $this->deleteRegionalCache(); + break; default: Console::error('No delete operation for type: ' . $type); break; @@ -675,4 +678,11 @@ class DeletesV1 extends Worker $device->deletePath($document->getId()); } + + protected function deleteRegionalCache() + { + $this->deleteByGroup('syncs', [ + Query::equal('region', [$this->args['region']]) + ], $this->getConsoleDB); + } } diff --git a/app/workers/syncsOut.php b/app/workers/syncsOut.php index af596e46d4..2498d9ebf6 100644 --- a/app/workers/syncsOut.php +++ b/app/workers/syncsOut.php @@ -4,6 +4,8 @@ use Ahc\Jwt\JWT; use Appwrite\Resque\Worker; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Database\DateTime; +use Utopia\Database\Document; require_once __DIR__ . '/../init.php'; @@ -12,8 +14,6 @@ Console::success(APP_NAME . ' syncs out worker v1 has started'); class SyncsOutV1 extends Worker { - protected array $errors = []; - private array $regions = [ 'fra1' => '172.17.0.1', 'nyc1' => '172.17.0.1', @@ -35,36 +35,57 @@ class SyncsOutV1 extends Worker $currentRegion = 'nyc1'; $data['keys'][] = $this->args['key']; - $jwt = new JWT(App::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 900, 10); + $jwt = new JWT(App::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 600, 10); $token = $jwt->encode($data); + if (!empty($this->args['region'])) { + $this->regions = $this->regions[$this->args['region']]; + } + foreach ($this->regions as $region => $host) { if ($currentRegion === $region) { continue; } - $ch = curl_init($host . '/v1/syncs'); - curl_setopt($ch, CURLOPT_HTTPHEADER, [ - 'Authorization: Bearer ' . $token, - 'Content-Type: application/json' - ]); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_TIMEOUT, 5); - curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); - curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data)); - for ($attempts = 0; $attempts < 6; $attempts++) { - curl_exec($ch); - $responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($responseStatus === 200) { - break; - } - - sleep(2); + $status = $this->send($host, $token, $data); + if ($status !== 200) { + $this->getConsoleDB()->createDocument('syncs', new Document([ + 'requestedAt' => DateTime::now(), + 'region' => $region, + 'keys' => $data, + 'status' => $status, + ])); } - curl_close($ch); } } + private function send($host, $token, $data): int + { + + $ch = curl_init($host . '/v1/syncs'); + curl_setopt($ch, CURLOPT_HTTPHEADER, [ + 'Authorization: Bearer ' . $token, + 'Content-Type: application/json' + ]); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_TIMEOUT, 5); + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); + curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data)); + + for ($attempts = 0; $attempts < 3; $attempts++) { + curl_exec($ch); + $responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE); + + if ($responseStatus === 200) { + return $responseStatus; + } + + sleep(2); + } + curl_close($ch); + return $responseStatus; + } + public function shutdown(): void { diff --git a/composer.json b/composer.json index 47317ed5e2..f302008567 100644 --- a/composer.json +++ b/composer.json @@ -48,7 +48,7 @@ "utopia-php/abuse": "0.14.*", "utopia-php/analytics": "0.2.*", "utopia-php/audit": "0.15.*", - "utopia-php/cache": "0.6.*", + "utopia-php/cache": "dev-feat-redis-sync as 0.6.1", "utopia-php/cli": "0.13.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.26.*", diff --git a/composer.lock b/composer.lock index b4bbda3cef..3805593bd6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "08fdd139ad1285b02c4b4e555679e7de", + "content-hash": "1ceb54089b20b863e8685e1644ba51f9", "packages": [ { "name": "adhocore/jwt", @@ -1897,24 +1897,26 @@ }, { "name": "utopia-php/cache", - "version": "0.6.1", + "version": "dev-feat-redis-sync", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "9889235a6d3da6cbb1f435201529da4d27c30e79" + "reference": "99e7085eb229d0f0159a4f2107ea5ea123f7b32b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/9889235a6d3da6cbb1f435201529da4d27c30e79", - "reference": "9889235a6d3da6cbb1f435201529da4d27c30e79", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/99e7085eb229d0f0159a4f2107ea5ea123f7b32b", + "reference": "99e7085eb229d0f0159a4f2107ea5ea123f7b32b", "shasum": "" }, "require": { "ext-json": "*", + "ext-memcached": "*", "ext-redis": "*", "php": ">=8.0" }, "require-dev": { + "laravel/pint": "1.2.*", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.13.1" }, @@ -1928,12 +1930,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "A simple cache library to manage application cache storing, loading and purging", "keywords": [ "cache", @@ -1944,9 +1940,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/0.6.1" + "source": "https://github.com/utopia-php/cache/tree/feat-redis-sync" }, - "time": "2022-08-10T08:12:46+00:00" + "time": "2022-10-18T06:58:42+00:00" }, { "name": "utopia-php/cli", @@ -3413,25 +3409,30 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.6.1", + "version": "1.6.2", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "77a32518733312af16a44300404e945338981de3" + "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/77a32518733312af16a44300404e945338981de3", - "reference": "77a32518733312af16a44300404e945338981de3", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/48f445a408c131e38cab1c235aa6d2bb7a0bb20d", + "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d", "shasum": "" }, "require": { - "php": "^7.2 || ^8.0", + "php": "^7.4 || ^8.0", "phpdocumentor/reflection-common": "^2.0" }, "require-dev": { "ext-tokenizer": "*", - "psalm/phar": "^4.8" + "phpstan/extension-installer": "^1.1", + "phpstan/phpstan": "^1.8", + "phpstan/phpstan-phpunit": "^1.1", + "phpunit/phpunit": "^9.5", + "rector/rector": "^0.13.9", + "vimeo/psalm": "^4.25" }, "type": "library", "extra": { @@ -3457,9 +3458,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.1" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.2" }, - "time": "2022-03-15T21:29:03+00:00" + "time": "2022-10-14T12:47:21+00:00" }, { "name": "phpspec/prophecy", @@ -5352,9 +5353,18 @@ "time": "2022-09-28T08:42:51+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/cache", + "version": "dev-feat-redis-sync", + "alias": "0.6.1", + "alias_normalized": "0.6.1.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/cache": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index 55a4028ca6..cb34d788cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -259,6 +259,7 @@ services: entrypoint: worker-syncs-out <<: *x-logging container_name: appwrite-worker-syncs-out + image: appwrite-dev build: context: . networks: @@ -266,8 +267,8 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - - ./vendor/utopia-php/cache:/usr/src/code/vendor/utopia-php/cache depends_on: + - mariadb - redis environment: - _APP_ENV @@ -275,11 +276,17 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS appwrite-worker-syncs-in: entrypoint: worker-syncs-in <<: *x-logging container_name: appwrite-worker-syncs-in + image: appwrite-dev build: context: . networks: @@ -287,7 +294,6 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - - ./vendor/utopia-php/cache:/usr/src/code/vendor/utopia-php/cache depends_on: - redis environment: diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 72ace2a86d..81de8ee939 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -13,6 +13,7 @@ class Delete extends Event protected ?string $datetime = null; protected ?string $dateTime30m = null; protected ?string $dateTime1d = null; + protected ?string $region = null; public function __construct() @@ -92,6 +93,16 @@ class Delete extends Event return $this; } + /** + * Sets cloud region. + * + * @param string $region + */ + public function setRegion($region): void + { + $this->region = $region; + } + /** * Returns the resource for the delete event. * diff --git a/src/Appwrite/Event/SyncOut.php b/src/Appwrite/Event/SyncOut.php index 73946e7e01..bec0c65857 100644 --- a/src/Appwrite/Event/SyncOut.php +++ b/src/Appwrite/Event/SyncOut.php @@ -7,6 +7,7 @@ use Resque; class SyncOut extends Event { protected string $key = ''; + protected string $region = ''; public function __construct() { @@ -36,6 +37,29 @@ class SyncOut extends Event return $this->key; } + /** + * Sets cloud region. + * + * @param string $region + * @return self + */ + public function setRegion(string $region): self + { + $this->host = $region; + + return $this; + } + + /** + * Returns cloud region. + * + * @return string + */ + public function getRegion(): string + { + return $this->region; + } + /** * Executes the event and sends it to the messaging worker. *