From be26e4b729a46621b053515623e9b828c2593bdb Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 18 Oct 2023 19:44:06 +0000 Subject: [PATCH] chore: remove resque library and update health check endpoints --- CONTRIBUTING.md | 2 - app/controllers/api/health.php | 68 +++--- app/init.php | 8 - composer.json | 1 - composer.lock | 232 +++++--------------- docker-compose.yml | 14 -- src/Appwrite/Event/Database.php | 1 - src/Appwrite/Platform/Tasks/Maintenance.php | 2 +- src/Appwrite/Platform/Workers/Databases.php | 1 + tests/resources/docker/docker-compose.yml | 14 -- 10 files changed, 93 insertions(+), 250 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e71fce135c..145ace7876 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -206,7 +206,6 @@ Appwrite's current structure is a combination of both [Monolithic](https://en.wi │ ├── Network │ ├── OpenSSL │ ├── Promises -│ ├── Resque │ ├── Specification │ ├── Task │ ├── Template @@ -251,7 +250,6 @@ Appwrite stack is a combination of a variety of open-source technologies and too - Imagemagick - for manipulating and managing image media files. - Webp - for better compression of images on supporting clients. - SMTP - for sending email messages and alerts. -- Resque - for managing data queues and scheduled tasks over a Redis server. ## Package Managers diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 2b41e2fef8..1051948629 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -14,6 +14,7 @@ use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; +use Utopia\Validator\Text; App::get('/v1/health') ->desc('Get HTTP') @@ -347,10 +348,11 @@ App::get('/v1/health/queue/webhooks') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::WEBHOOK_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/logs') @@ -364,10 +366,11 @@ App::get('/v1/health/queue/logs') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::AUDITS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::AUDITS_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/certificates') @@ -381,10 +384,11 @@ App::get('/v1/health/queue/certificates') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::CERTIFICATES_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/builds') @@ -398,13 +402,14 @@ App::get('/v1/health/queue/builds') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::BUILDS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::BUILDS_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); -App::get('/v1/health/queue/databases') +App::get('/v1/health/queue/databases/:databaseId') ->desc('Get databases queue') ->groups(['api', 'health']) ->label('scope', 'health.read') @@ -415,10 +420,13 @@ App::get('/v1/health/queue/databases') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('databaseId', 'database_db_main', new Text(256), 'Database for which to check the queue size', true) + ->inject('databaseId') + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::DATABASE_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (string $databaseId, Connection $queue, Response $response) { + $client = new Client($databaseId, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/deletes') @@ -432,10 +440,11 @@ App::get('/v1/health/queue/deletes') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::DELETE_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::DELETE_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/mails') @@ -449,10 +458,11 @@ App::get('/v1/health/queue/mails') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::MAILS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::MAILS_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/messaging') @@ -466,10 +476,11 @@ App::get('/v1/health/queue/messaging') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::MESSAGING_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::MESSAGING_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/migrations') @@ -483,10 +494,11 @@ App::get('/v1/health/queue/migrations') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->inject('queue') ->inject('response') - ->action(function (Response $response) { - - $response->dynamic(new Document([ 'size' => Resque::size(Event::MIGRATIONS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE); + ->action(function (Connection $queue, Response $response) { + $client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue); + $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/functions') diff --git a/app/init.php b/app/init.php index df9e6be990..8aec6122fa 100644 --- a/app/init.php +++ b/app/init.php @@ -262,14 +262,6 @@ Config::load('storage-mimes', __DIR__ . '/config/storage/mimes.php'); Config::load('storage-inputs', __DIR__ . '/config/storage/inputs.php'); Config::load('storage-outputs', __DIR__ . '/config/storage/outputs.php'); -$user = App::getEnv('_APP_REDIS_USER', ''); -$pass = App::getEnv('_APP_REDIS_PASS', ''); -if (!empty($user) || !empty($pass)) { - Resque::setBackend('redis://' . $user . ':' . $pass . '@' . App::getEnv('_APP_REDIS_HOST', '') . ':' . App::getEnv('_APP_REDIS_PORT', '')); -} else { - Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '') . ':' . App::getEnv('_APP_REDIS_PORT', '')); -} - /** * New DB Filters */ diff --git a/composer.json b/composer.json index bd5b143f16..0ffbfe6200 100644 --- a/composer.json +++ b/composer.json @@ -68,7 +68,6 @@ "utopia-php/swoole": "0.5.*", "utopia-php/vcs": "0.5.*", "utopia-php/websocket": "0.1.*", - "resque/php-resque": "1.3.6", "matomo/device-detector": "6.1.*", "dragonmantank/cron-expression": "3.3.2", "influxdb/influxdb-php": "1.15.2", diff --git a/composer.lock b/composer.lock index 2751ee2098..e45e4b3d86 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5060c891e467051fedd25f3853cd37b0", + "content-hash": "3a0624bf1df70e602233efa5916aa6ce", "packages": [ { "name": "adhocore/jwt", @@ -339,53 +339,6 @@ ], "time": "2022-07-05T22:32:14+00:00" }, - { - "name": "colinmollenhour/credis", - "version": "v1.15.0", - "source": { - "type": "git", - "url": "https://github.com/colinmollenhour/credis.git", - "reference": "28810439de1d9597b7ba11794ed9479fb6f3de7c" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/28810439de1d9597b7ba11794ed9479fb6f3de7c", - "reference": "28810439de1d9597b7ba11794ed9479fb6f3de7c", - "shasum": "" - }, - "require": { - "php": ">=5.6.0" - }, - "suggest": { - "ext-redis": "Improved performance for communicating with redis" - }, - "type": "library", - "autoload": { - "classmap": [ - "Client.php", - "Cluster.php", - "Sentinel.php", - "Module.php" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Colin Mollenhour", - "email": "colin@mollenhour.com" - } - ], - "description": "Credis is a lightweight interface to the Redis key-value store which wraps the phpredis library when available for better performance.", - "homepage": "https://github.com/colinmollenhour/credis", - "support": { - "issues": "https://github.com/colinmollenhour/credis/issues", - "source": "https://github.com/colinmollenhour/credis/tree/v1.15.0" - }, - "time": "2023-04-18T15:34:23+00:00" - }, { "name": "dragonmantank/cron-expression", "version": "v3.3.2", @@ -1476,56 +1429,6 @@ }, "time": "2023-04-04T09:54:51+00:00" }, - { - "name": "psr/log", - "version": "1.1.4", - "source": { - "type": "git", - "url": "https://github.com/php-fig/log.git", - "reference": "d49695b909c3b7628b6289db5479a1c204601f11" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/php-fig/log/zipball/d49695b909c3b7628b6289db5479a1c204601f11", - "reference": "d49695b909c3b7628b6289db5479a1c204601f11", - "shasum": "" - }, - "require": { - "php": ">=5.3.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.1.x-dev" - } - }, - "autoload": { - "psr-4": { - "Psr\\Log\\": "Psr/Log/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "PHP-FIG", - "homepage": "https://www.php-fig.org/" - } - ], - "description": "Common interface for logging libraries", - "homepage": "https://github.com/php-fig/log", - "keywords": [ - "log", - "psr", - "psr-3" - ], - "support": { - "source": "https://github.com/php-fig/log/tree/1.1.4" - }, - "time": "2021-05-03T11:20:27+00:00" - }, { "name": "ralouphie/getallheaders", "version": "3.0.3", @@ -1570,89 +1473,6 @@ }, "time": "2019-03-08T08:55:37+00:00" }, - { - "name": "resque/php-resque", - "version": "v1.3.6", - "source": { - "type": "git", - "url": "https://github.com/resque/php-resque.git", - "reference": "fe41c04763699b1318d97ed14cc78583e9380161" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/resque/php-resque/zipball/fe41c04763699b1318d97ed14cc78583e9380161", - "reference": "fe41c04763699b1318d97ed14cc78583e9380161", - "shasum": "" - }, - "require": { - "colinmollenhour/credis": "~1.7", - "php": ">=5.6.0", - "psr/log": "~1.0" - }, - "require-dev": { - "phpunit/phpunit": "^5.7" - }, - "suggest": { - "ext-pcntl": "REQUIRED for forking processes on platforms that support it (so anything but Windows).", - "ext-proctitle": "Allows php-resque to rename the title of UNIX processes to show the status of a worker.", - "ext-redis": "Native PHP extension for Redis connectivity. Credis will automatically utilize when available." - }, - "bin": [ - "bin/resque", - "bin/resque-scheduler" - ], - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.0-dev" - } - }, - "autoload": { - "psr-0": { - "Resque": "lib", - "ResqueScheduler": "lib" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Dan Hunsaker", - "email": "danhunsaker+resque@gmail.com", - "role": "Maintainer" - }, - { - "name": "Rajib Ahmed", - "homepage": "https://github.com/rajibahmed", - "role": "Maintainer" - }, - { - "name": "Steve Klabnik", - "email": "steve@steveklabnik.com", - "role": "Maintainer" - }, - { - "name": "Chris Boulton", - "email": "chris@bigcommerce.com", - "role": "Creator" - } - ], - "description": "Redis backed library for creating background jobs and processing them later. Based on resque for Ruby.", - "homepage": "http://www.github.com/resque/php-resque/", - "keywords": [ - "background", - "job", - "redis", - "resque" - ], - "support": { - "issues": "https://github.com/resque/php-resque/issues", - "source": "https://github.com/resque/php-resque/tree/v1.3.6" - }, - "time": "2020-04-16T16:39:50+00:00" - }, { "name": "slickdeals/statsd", "version": "3.1.0", @@ -4605,6 +4425,56 @@ ], "time": "2022-04-01T12:37:26+00:00" }, + { + "name": "psr/log", + "version": "3.0.0", + "source": { + "type": "git", + "url": "https://github.com/php-fig/log.git", + "reference": "fe5ea303b0887d5caefd3d431c3e61ad47037001" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/log/zipball/fe5ea303b0887d5caefd3d431c3e61ad47037001", + "reference": "fe5ea303b0887d5caefd3d431c3e61ad47037001", + "shasum": "" + }, + "require": { + "php": ">=8.0.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Log\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for logging libraries", + "homepage": "https://github.com/php-fig/log", + "keywords": [ + "log", + "psr", + "psr-3" + ], + "support": { + "source": "https://github.com/php-fig/log/tree/3.0.0" + }, + "time": "2021-07-14T16:46:02+00:00" + }, { "name": "sebastian/cli-parser", "version": "1.0.1", diff --git a/docker-compose.yml b/docker-compose.yml index 78feecd17e..728e87cbc6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -872,7 +872,6 @@ services: # MailCatcher - An SMTP server. Catches all system emails and displays them in a nice UI. # RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks # RedisCommander - A nice UI for exploring Redis data - # Resque - A nice UI for exploring Redis pub/sub, view the different queues workloads, pending and failed tasks # Chronograf - A nice UI for exploring InfluxDB data # Webgrind - A nice UI for exploring and debugging code-level stuff @@ -914,19 +913,6 @@ services: # ports: # - "8081:8081" - # resque: - # image: appwrite/resque-web:1.1.0 - # restart: unless-stopped - # networks: - # - appwrite - # ports: - # - "5678:5678" - # environment: - # - RESQUE_WEB_HOST=redis - # - RESQUE_WEB_PORT=6379 - # - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user - # - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password - # chronograf: # image: chronograf:1.6 # container_name: appwrite-chronograf diff --git a/src/Appwrite/Event/Database.php b/src/Appwrite/Event/Database.php index e5a86ef840..442cbe4bbc 100644 --- a/src/Appwrite/Event/Database.php +++ b/src/Appwrite/Event/Database.php @@ -107,7 +107,6 @@ class Database extends Event */ public function trigger(): string|bool { - $this->setQueue($this->getProject()->getAttribute('database')); $client = new Client($this->queue, $this->connection); diff --git a/src/Appwrite/Platform/Tasks/Maintenance.php b/src/Appwrite/Platform/Tasks/Maintenance.php index f4756cd8a8..82a62ffed1 100644 --- a/src/Appwrite/Platform/Tasks/Maintenance.php +++ b/src/Appwrite/Platform/Tasks/Maintenance.php @@ -22,7 +22,7 @@ class Maintenance extends Action public function __construct() { $this - ->desc('Schedules maintenance tasks and publishes them to resque') + ->desc('Schedules maintenance tasks and publishes them to our queues') ->inject('dbForConsole') ->inject('queueForCertificates') ->inject('queueForDeletes') diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 7d0e015270..e0ec75e1d4 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -104,6 +104,7 @@ class Databases extends Action 'attributeId' => $attribute->getId() ]); /** + * TODO @christyjacob4 verify if this is still the case * Fetch attribute from the database, since with Resque float values are loosing informations. */ $attribute = $dbForProject->getDocument('attributes', $attribute->getId()); diff --git a/tests/resources/docker/docker-compose.yml b/tests/resources/docker/docker-compose.yml index bdb9bf49a4..50635018db 100644 --- a/tests/resources/docker/docker-compose.yml +++ b/tests/resources/docker/docker-compose.yml @@ -376,20 +376,6 @@ services: # ports: # - "8081:8081" - # resque: - # image: registry.gitlab.com/appwrite/appwrite/resque-web:v1.0.2 - # restart: unless-stopped - # networks: - # - appwrite - # ports: - # - "5678:5678" - # environment: - # - RESQUE_WEB_HOST=redis - # - RESQUE_WEB_PORT=6379 - # - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user - # - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password - - # webgrind: # image: 'jokkedk/webgrind:latest' # volumes: