From 408ae68e7a224289e89e9c2b741cacbb74516c1c Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 14 Dec 2020 22:09:44 +0530 Subject: [PATCH 01/30] feat: starter for maintenance task --- Dockerfile | 4 +++- app/cli.php | 1 + app/tasks/maintenance.php | 19 +++++++++++++++++++ bin/maintenance | 3 +++ docker-compose.yml | 16 ++++++++++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 app/tasks/maintenance.php create mode 100644 bin/maintenance diff --git a/Dockerfile b/Dockerfile index d41dbc50fd..eccefed79e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -93,7 +93,8 @@ ENV _APP_SERVER=swoole \ _APP_FUNCTIONS_TIMEOUT=900 \ _APP_FUNCTIONS_CONTAINERS=10 \ _APP_SETUP=self-hosted \ - _APP_VERSION=$VERSION + _APP_VERSION=$VERSION \ + _APP_MAINTENANCE_INTERVAL=10 #ENV _APP_SMTP_SECURE '' #ENV _APP_SMTP_USERNAME '' #ENV _APP_SMTP_PASSWORD '' @@ -155,6 +156,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ + chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/schedule && \ diff --git a/app/cli.php b/app/cli.php index 9c99890602..ced2fb8da4 100644 --- a/app/cli.php +++ b/app/cli.php @@ -9,6 +9,7 @@ use Utopia\CLI\Console; $cli = new CLI(); include 'tasks/doctor.php'; +include 'tasks/maintenance.php'; include 'tasks/install.php'; include 'tasks/migrate.php'; include 'tasks/sdks.php'; diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php new file mode 100644 index 0000000000..aa98662fae --- /dev/null +++ b/app/tasks/maintenance.php @@ -0,0 +1,19 @@ +task('maintenance') + ->desc('Schedules maintenance tasks and publishes them to resque') + ->action(function () { + $interval = App::getEnv('_APP_MAINTENANCE_INTERVAL', ''); + + for($i = 0; $i <= 10; ++$i){ + Console::log('Starting the maintenance worker every '.$interval.' seconds'); + sleep($interval); + } + + }); \ No newline at end of file diff --git a/bin/maintenance b/bin/maintenance new file mode 100644 index 0000000000..099551cb32 --- /dev/null +++ b/bin/maintenance @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php maintenance $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 72d791fa98..5cd6941881 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -277,6 +277,22 @@ services: - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD + appwrite-maintenance: + entrypoint: maintenance + container_name: appwrite-maintenance + build: + context: . + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + appwrite-schedule: entrypoint: schedule container_name: appwrite-schedule From 4f997d3770474182ea2f93af95c5464be0a30216 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 15 Dec 2020 02:56:37 +0530 Subject: [PATCH 02/30] feat: publish delete events to resque --- app/tasks/maintenance.php | 32 +- app/workers/deletes.php | 23 ++ composer.json | 2 +- composer.lock | 671 ++++++++++++++++++++++++++++++-------- 4 files changed, 580 insertions(+), 148 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index aa98662fae..d3099f0007 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -2,18 +2,38 @@ global $cli; +require_once __DIR__.'/../init.php'; + +use Appwrite\Database\Database; +use Appwrite\Database\Document; use Utopia\App; use Utopia\CLI\Console; + $cli ->task('maintenance') ->desc('Schedules maintenance tasks and publishes them to resque') ->action(function () { - $interval = App::getEnv('_APP_MAINTENANCE_INTERVAL', ''); + // Convert string to integer + $interval = App::getEnv('_APP_MAINTENANCE_INTERVAL', '') + 0; + //Convert Seconds to microseconds + $interval = $interval * 1000000; + + Console::loop(function() { + $projects = $consoleDB->getCollection([ + 'filters' => [ + '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, + ], + ]); + var_dump("*************** MAINTENANCE WORKER *****************"); + print_r($projects); + + Resque::enqueue('v1-deletes', 'DeletesV1', [ + 'document' => new Document([ + '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, + 'projectIds' => $this->projects + ]), + ]); + }, $interval); - for($i = 0; $i <= 10; ++$i){ - Console::log('Starting the maintenance worker every '.$interval.' seconds'); - sleep($interval); - } - }); \ No newline at end of file diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 14addd2d18..8bc047e5d1 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -39,6 +39,9 @@ class DeletesV1 case Database::SYSTEM_COLLECTION_FUNCTIONS: $this->deleteFunction($document, $projectId); break; + case Database::SYSTEM_COLLECTION_EXECUTIONS: + $this->deleteExecutionLogs($document); + break; case Database::SYSTEM_COLLECTION_USERS: $this->deleteUser($document, $projectId); break; @@ -95,6 +98,26 @@ class DeletesV1 ], $this->getProjectDB($projectId)); } + protected function deleteExecutionLogs(Document $document) + { + var_dump("*********************DELETING EXECUTION LOGS *********************"); + $projectIds = $document->getAttribute('projectIds', []); + print_r($projectIds); + foreach ($projectIds as $projectId) { + if (!($projectDB = $this->getProjectDB($projectId))) { + throw new Exception('Failed to get projectDB for project '.$projectId, 500); + } + + var_dump("********************* DELETING FOR PROJECT *********************"); + var_dump($projectId); + // Delete Executions + $this->deleteByGroup([ + '$collection='.$document->getCollection(), + '$projectId='.$projectId + ], $projectDB); + } + } + protected function deleteFunction(Document $document, $projectId) { $projectDB = $this->getProjectDB($projectId); diff --git a/composer.json b/composer.json index d70246a474..ed64be8ddf 100644 --- a/composer.json +++ b/composer.json @@ -38,7 +38,7 @@ "utopia-php/abuse": "0.2.*", "utopia-php/audit": "0.3.*", "utopia-php/cache": "0.2.*", - "utopia-php/cli": "0.7.2", + "utopia-php/cli": "0.8.0", "utopia-php/config": "0.2.*", "utopia-php/locale": "0.3.*", "utopia-php/registry": "0.2.*", diff --git a/composer.lock b/composer.lock index bdb00f0705..022d6435aa 100644 --- a/composer.lock +++ b/composer.lock @@ -49,6 +49,10 @@ "clamav", "php" ], + "support": { + "issues": "https://github.com/appwrite/php-clamav/issues", + "source": "https://github.com/appwrite/php-clamav/tree/master" + }, "time": "2020-02-29T11:35:01+00:00" }, { @@ -113,6 +117,10 @@ "qrcode", "qrcode-generator" ], + "support": { + "issues": "https://github.com/chillerlan/php-qrcode/issues", + "source": "https://github.com/chillerlan/php-qrcode/tree/4.2.0" + }, "funding": [ { "url": "https://ko-fi.com/codemasher", @@ -168,6 +176,10 @@ "container", "helper" ], + "support": { + "issues": "https://github.com/chillerlan/php-settings-container/issues", + "source": "https://github.com/chillerlan/php-settings-container" + }, "funding": [ { "url": "https://ko-fi.com/codemasher", @@ -214,6 +226,10 @@ ], "description": "Credis is a lightweight interface to the Redis key-value store which wraps the phpredis library when available for better performance.", "homepage": "https://github.com/colinmollenhour/credis", + "support": { + "issues": "https://github.com/colinmollenhour/credis/issues", + "source": "https://github.com/colinmollenhour/credis/tree/v1.12.1" + }, "time": "2020-11-06T16:09:14+00:00" }, { @@ -264,6 +280,10 @@ "statsd", "udp" ], + "support": { + "issues": "https://github.com/domnikl/statsd-php/issues", + "source": "https://github.com/domnikl/statsd-php/tree/master" + }, "time": "2020-01-03T14:24:58+00:00" }, { @@ -312,6 +332,10 @@ "cron", "schedule" ], + "support": { + "issues": "https://github.com/dragonmantank/cron-expression/issues", + "source": "https://github.com/dragonmantank/cron-expression/tree/3.0.1" + }, "funding": [ { "url": "https://github.com/dragonmantank", @@ -398,6 +422,10 @@ "rest", "web service" ], + "support": { + "issues": "https://github.com/guzzle/guzzle/issues", + "source": "https://github.com/guzzle/guzzle/tree/7.2.0" + }, "funding": [ { "url": "https://github.com/GrahamCampbell", @@ -438,6 +466,7 @@ "require-dev": { "symfony/phpunit-bridge": "^4.4 || ^5.1" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -467,6 +496,10 @@ "keywords": [ "promise" ], + "support": { + "issues": "https://github.com/guzzle/promises/issues", + "source": "https://github.com/guzzle/promises/tree/master" + }, "time": "2020-10-19T16:50:15+00:00" }, { @@ -475,12 +508,12 @@ "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "25f7f893f0b52b7b14e244a16679d72b1f0088de" + "reference": "f47ece9e6e8ce74e3be04bef47f46061dc18c095" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/25f7f893f0b52b7b14e244a16679d72b1f0088de", - "reference": "25f7f893f0b52b7b14e244a16679d72b1f0088de", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/f47ece9e6e8ce74e3be04bef47f46061dc18c095", + "reference": "f47ece9e6e8ce74e3be04bef47f46061dc18c095", "shasum": "" }, "require": { @@ -538,7 +571,11 @@ "uri", "url" ], - "time": "2020-10-22T07:42:05+00:00" + "support": { + "issues": "https://github.com/guzzle/psr7/issues", + "source": "https://github.com/guzzle/psr7/tree/1.x" + }, + "time": "2020-12-08T11:45:39+00:00" }, { "name": "influxdb/influxdb-php", @@ -599,6 +636,10 @@ "influxdb library", "time series" ], + "support": { + "issues": "https://github.com/influxdata/influxdb-php/issues", + "source": "https://github.com/influxdata/influxdb-php/tree/1.15.1" + }, "time": "2020-09-18T13:24:03+00:00" }, { @@ -654,20 +695,26 @@ "parser", "useragent" ], + "support": { + "forum": "http://forum.matomo.org/", + "issues": "https://github.com/matomo-org/device-detector/issues", + "source": "https://github.com/matomo-org/piwik", + "wiki": "https://dev.matomo.org/" + }, "time": "2020-08-17T07:37:33+00:00" }, { "name": "mustangostang/spyc", - "version": "dev-master", + "version": "0.6.3", "source": { "type": "git", - "url": "https://github.com/mustangostang/spyc.git", - "reference": "daf9fa4ef675519386b4f556c9d5ab5f9c14055a" + "url": "git@github.com:mustangostang/spyc.git", + "reference": "4627c838b16550b666d15aeae1e5289dd5b77da0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/mustangostang/spyc/zipball/daf9fa4ef675519386b4f556c9d5ab5f9c14055a", - "reference": "daf9fa4ef675519386b4f556c9d5ab5f9c14055a", + "url": "https://api.github.com/repos/mustangostang/spyc/zipball/4627c838b16550b666d15aeae1e5289dd5b77da0", + "reference": "4627c838b16550b666d15aeae1e5289dd5b77da0", "shasum": "" }, "require": { @@ -704,7 +751,7 @@ "yaml", "yml" ], - "time": "2019-12-03T17:11:33+00:00" + "time": "2019-09-10T13:16:29+00:00" }, { "name": "phpmailer/phpmailer", @@ -766,6 +813,10 @@ } ], "description": "PHPMailer is a full-featured email creation and transfer class for PHP", + "support": { + "issues": "https://github.com/PHPMailer/PHPMailer/issues", + "source": "https://github.com/PHPMailer/PHPMailer/tree/v6.1.7" + }, "funding": [ { "url": "https://github.com/synchro", @@ -792,6 +843,7 @@ "php": "^7.0 || ^8.0", "psr/http-message": "^1.0" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -821,6 +873,9 @@ "psr", "psr-18" ], + "support": { + "source": "https://github.com/php-fig/http-client/tree/master" + }, "time": "2020-09-19T09:12:31+00:00" }, { @@ -840,6 +895,7 @@ "require": { "php": ">=5.3.0" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -871,6 +927,9 @@ "request", "response" ], + "support": { + "source": "https://github.com/php-fig/http-message/tree/master" + }, "time": "2019-08-29T13:16:46+00:00" }, { @@ -890,6 +949,7 @@ "require": { "php": ">=5.3.0" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -918,6 +978,9 @@ "psr", "psr-3" ], + "support": { + "source": "https://github.com/php-fig/log/tree/master" + }, "time": "2020-09-18T06:44:51+00:00" }, { @@ -958,6 +1021,10 @@ } ], "description": "A polyfill for getallheaders.", + "support": { + "issues": "https://github.com/ralouphie/getallheaders/issues", + "source": "https://github.com/ralouphie/getallheaders/tree/develop" + }, "time": "2019-03-08T08:55:37+00:00" }, { @@ -1037,6 +1104,10 @@ "redis", "resque" ], + "support": { + "issues": "https://github.com/resque/php-resque/issues", + "source": "https://github.com/resque/php-resque/tree/v1.3.6" + }, "time": "2020-04-16T16:39:50+00:00" }, { @@ -1085,6 +1156,10 @@ "upf", "utopia" ], + "support": { + "issues": "https://github.com/utopia-php/abuse/issues", + "source": "https://github.com/utopia-php/abuse/tree/0.2.2" + }, "time": "2020-10-23T06:51:42+00:00" }, { @@ -1133,6 +1208,10 @@ "upf", "utopia" ], + "support": { + "issues": "https://github.com/utopia-php/audit/issues", + "source": "https://github.com/utopia-php/audit/tree/0.3.2" + }, "time": "2020-10-23T08:09:44+00:00" }, { @@ -1181,6 +1260,10 @@ "upf", "utopia" ], + "support": { + "issues": "https://github.com/utopia-php/cache/issues", + "source": "https://github.com/utopia-php/cache/tree/0.2.3" + }, "time": "2020-10-24T10:11:01+00:00" }, { @@ -1230,6 +1313,10 @@ "upf", "utopia" ], + "support": { + "issues": "https://github.com/utopia-php/cli/issues", + "source": "https://github.com/utopia-php/cli/tree/0.7.2" + }, "time": "2020-10-23T13:34:41+00:00" }, { @@ -1277,6 +1364,10 @@ "upf", "utopia" ], + "support": { + "issues": "https://github.com/utopia-php/config/issues", + "source": "https://github.com/utopia-php/config/tree/0.2.2" + }, "time": "2020-10-24T09:49:09+00:00" }, { @@ -1328,6 +1419,10 @@ "upf", "utopia" ], + "support": { + "issues": "https://github.com/utopia-php/domains/issues", + "source": "https://github.com/utopia-php/domains/tree/0.2.3" + }, "time": "2020-10-23T09:59:51+00:00" }, { @@ -1373,6 +1468,10 @@ "php", "upf" ], + "support": { + "issues": "https://github.com/utopia-php/framework/issues", + "source": "https://github.com/utopia-php/framework/tree/0.9.8" + }, "time": "2020-11-11T20:34:58+00:00" }, { @@ -1420,6 +1519,10 @@ "upf", "utopia" ], + "support": { + "issues": "https://github.com/utopia-php/locale/issues", + "source": "https://github.com/utopia-php/locale/tree/0.3.3" + }, "time": "2020-10-24T08:12:55+00:00" }, { @@ -1469,6 +1572,10 @@ "upf", "utopia" ], + "support": { + "issues": "https://github.com/utopia-php/preloader/issues", + "source": "https://github.com/utopia-php/preloader/tree/0.2.4" + }, "time": "2020-10-24T07:04:59+00:00" }, { @@ -1517,6 +1624,10 @@ "upf", "utopia" ], + "support": { + "issues": "https://github.com/utopia-php/registry/issues", + "source": "https://github.com/utopia-php/registry/tree/0.2.4" + }, "time": "2020-10-24T08:51:37+00:00" }, { @@ -1569,6 +1680,10 @@ "upf", "utopia" ], + "support": { + "issues": "https://github.com/utopia-php/swoole/issues", + "source": "https://github.com/utopia-php/swoole/tree/0.2.0" + }, "time": "2020-10-29T12:42:38+00:00" } ], @@ -1579,12 +1694,12 @@ "source": { "type": "git", "url": "https://github.com/amphp/amp.git", - "reference": "ecdc3c476b3ccff02f8e5d5bcc04f7ccfd18751c" + "reference": "dbb3c28ece24b36efa91be205f6f0b015bddc27c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/amp/zipball/ecdc3c476b3ccff02f8e5d5bcc04f7ccfd18751c", - "reference": "ecdc3c476b3ccff02f8e5d5bcc04f7ccfd18751c", + "url": "https://api.github.com/repos/amphp/amp/zipball/dbb3c28ece24b36efa91be205f6f0b015bddc27c", + "reference": "dbb3c28ece24b36efa91be205f6f0b015bddc27c", "shasum": "" }, "require": { @@ -1599,6 +1714,7 @@ "psalm/phar": "^3.11@dev", "react/promise": "^2" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -1649,13 +1765,18 @@ "non-blocking", "promise" ], + "support": { + "irc": "irc://irc.freenode.org/amphp", + "issues": "https://github.com/amphp/amp/issues", + "source": "https://github.com/amphp/amp/tree/master" + }, "funding": [ { "url": "https://github.com/amphp", "type": "github" } ], - "time": "2020-11-03T16:23:45+00:00" + "time": "2020-11-14T16:44:06+00:00" }, { "name": "amphp/byte-stream", @@ -1683,6 +1804,7 @@ "phpunit/phpunit": "^6 || ^7 || ^8", "psalm/phar": "^3.11.4" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -1721,6 +1843,11 @@ "non-blocking", "stream" ], + "support": { + "irc": "irc://irc.freenode.org/amphp", + "issues": "https://github.com/amphp/byte-stream/issues", + "source": "https://github.com/amphp/byte-stream/tree/master" + }, "funding": [ { "url": "https://github.com/amphp", @@ -1748,6 +1875,7 @@ "require-dev": { "phpunit/phpunit": "^7.0" }, + "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -1773,12 +1901,12 @@ "source": { "type": "git", "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "7413f0b55a051e89485c5cb9f765fe24bb02a7b6" + "reference": "64291c788b9a18272346decf566931e33a317399" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/7413f0b55a051e89485c5cb9f765fe24bb02a7b6", - "reference": "7413f0b55a051e89485c5cb9f765fe24bb02a7b6", + "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/64291c788b9a18272346decf566931e33a317399", + "reference": "64291c788b9a18272346decf566931e33a317399", "shasum": "" }, "require": { @@ -1793,6 +1921,7 @@ "ext-zip": "^1.13", "phpunit/phpunit": "^6.5 || ^7" }, + "default-branch": true, "type": "composer-plugin", "extra": { "class": "PackageVersions\\Installer", @@ -1820,6 +1949,10 @@ } ], "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", + "support": { + "issues": "https://github.com/composer/package-versions-deprecated/issues", + "source": "https://github.com/composer/package-versions-deprecated/tree/master" + }, "funding": [ { "url": "https://packagist.com", @@ -1834,7 +1967,7 @@ "type": "tidelift" } ], - "time": "2020-11-11T10:22:58+00:00" + "time": "2020-11-12T09:39:33+00:00" }, { "name": "composer/semver", @@ -1842,21 +1975,22 @@ "source": { "type": "git", "url": "https://github.com/composer/semver.git", - "reference": "4089fddb67bcf6bf860d91b979e95be303835002" + "reference": "dd61cb4efbd0cff1700b217faf24ce596af4fc4e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/semver/zipball/4089fddb67bcf6bf860d91b979e95be303835002", - "reference": "4089fddb67bcf6bf860d91b979e95be303835002", + "url": "https://api.github.com/repos/composer/semver/zipball/dd61cb4efbd0cff1700b217faf24ce596af4fc4e", + "reference": "dd61cb4efbd0cff1700b217faf24ce596af4fc4e", "shasum": "" }, "require": { "php": "^5.3.2 || ^7.0 || ^8.0" }, "require-dev": { - "phpstan/phpstan": "^0.12.19", + "phpstan/phpstan": "^0.12.54", "symfony/phpunit-bridge": "^4.2 || ^5" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -1896,6 +2030,11 @@ "validation", "versioning" ], + "support": { + "irc": "irc://irc.freenode.org/composer", + "issues": "https://github.com/composer/semver/issues", + "source": "https://github.com/composer/semver/tree/main" + }, "funding": [ { "url": "https://packagist.com", @@ -1910,20 +2049,20 @@ "type": "tidelift" } ], - "time": "2020-10-14T08:51:15+00:00" + "time": "2020-12-10T07:55:43+00:00" }, { "name": "composer/xdebug-handler", - "version": "1.4.4", + "version": "1.4.5", "source": { "type": "git", "url": "https://github.com/composer/xdebug-handler.git", - "reference": "6e076a124f7ee146f2487554a94b6a19a74887ba" + "reference": "f28d44c286812c714741478d968104c5e604a1d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/xdebug-handler/zipball/6e076a124f7ee146f2487554a94b6a19a74887ba", - "reference": "6e076a124f7ee146f2487554a94b6a19a74887ba", + "url": "https://api.github.com/repos/composer/xdebug-handler/zipball/f28d44c286812c714741478d968104c5e604a1d4", + "reference": "f28d44c286812c714741478d968104c5e604a1d4", "shasum": "" }, "require": { @@ -1954,6 +2093,11 @@ "Xdebug", "performance" ], + "support": { + "irc": "irc://irc.freenode.org/composer", + "issues": "https://github.com/composer/xdebug-handler/issues", + "source": "https://github.com/composer/xdebug-handler/tree/1.4.5" + }, "funding": [ { "url": "https://packagist.com", @@ -1968,7 +2112,7 @@ "type": "tidelift" } ], - "time": "2020-10-24T12:39:10+00:00" + "time": "2020-11-13T08:04:11+00:00" }, { "name": "dnoegel/php-xdg-base-dir", @@ -2001,6 +2145,10 @@ "MIT" ], "description": "implementation of xdg base directory specification for php", + "support": { + "issues": "https://github.com/dnoegel/php-xdg-base-dir/issues", + "source": "https://github.com/dnoegel/php-xdg-base-dir/tree/v0.1.1" + }, "time": "2019-12-04T15:06:13+00:00" }, { @@ -2052,6 +2200,10 @@ "constructor", "instantiate" ], + "support": { + "issues": "https://github.com/doctrine/instantiator/issues", + "source": "https://github.com/doctrine/instantiator/tree/1.4.x" + }, "funding": [ { "url": "https://www.doctrine-project.org/sponsorship.html", @@ -2107,6 +2259,10 @@ } ], "description": "A more advanced JSONRPC implementation", + "support": { + "issues": "https://github.com/felixfbecker/php-advanced-json-rpc/issues", + "source": "https://github.com/felixfbecker/php-advanced-json-rpc/tree/master" + }, "time": "2020-03-11T15:21:41+00:00" }, { @@ -2131,6 +2287,7 @@ "squizlabs/php_codesniffer": "^3.1", "vimeo/psalm": "^4.0" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -2159,6 +2316,10 @@ "php", "server" ], + "support": { + "issues": "https://github.com/felixfbecker/php-language-server-protocol/issues", + "source": "https://github.com/felixfbecker/php-language-server-protocol/tree/v1.5.0" + }, "time": "2020-10-23T13:55:30+00:00" }, { @@ -2219,6 +2380,10 @@ "minifier", "minify" ], + "support": { + "issues": "https://github.com/matthiasmullie/minify/issues", + "source": "https://github.com/matthiasmullie/minify/tree/1.3.63" + }, "time": "2020-01-21T20:21:08+00:00" }, { @@ -2268,6 +2433,10 @@ "paths", "relative" ], + "support": { + "issues": "https://github.com/matthiasmullie/path-converter/issues", + "source": "https://github.com/matthiasmullie/path-converter/tree/1.1.3" + }, "time": "2019-02-05T23:41:09+00:00" }, { @@ -2276,12 +2445,12 @@ "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "00aba97fc36feabc8d94667eebd5d43959e60008" + "reference": "776f831124e9c62e1a2c601ecc52e776d8bb7220" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/00aba97fc36feabc8d94667eebd5d43959e60008", - "reference": "00aba97fc36feabc8d94667eebd5d43959e60008", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/776f831124e9c62e1a2c601ecc52e776d8bb7220", + "reference": "776f831124e9c62e1a2c601ecc52e776d8bb7220", "shasum": "" }, "require": { @@ -2295,6 +2464,7 @@ "doctrine/common": "^2.6", "phpunit/phpunit": "^7.1" }, + "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -2316,13 +2486,17 @@ "object", "object graph" ], + "support": { + "issues": "https://github.com/myclabs/DeepCopy/issues", + "source": "https://github.com/myclabs/DeepCopy/tree/1.10.2" + }, "funding": [ { "url": "https://tidelift.com/funding/github/packagist/myclabs/deep-copy", "type": "tidelift" } ], - "time": "2020-10-01T09:35:15+00:00" + "time": "2020-11-13T09:40:50+00:00" }, { "name": "netresearch/jsonmapper", @@ -2368,20 +2542,25 @@ } ], "description": "Map nested JSON structures onto PHP classes", + "support": { + "email": "cweiske@cweiske.de", + "issues": "https://github.com/cweiske/jsonmapper/issues", + "source": "https://github.com/cweiske/jsonmapper/tree/master" + }, "time": "2020-04-16T18:48:43+00:00" }, { "name": "nikic/php-parser", - "version": "v4.10.2", + "version": "v4.10.3", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "658f1be311a230e0907f5dfe0213742aff0596de" + "reference": "dbe56d23de8fcb157bbc0cfb3ad7c7de0cfb0984" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/658f1be311a230e0907f5dfe0213742aff0596de", - "reference": "658f1be311a230e0907f5dfe0213742aff0596de", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/dbe56d23de8fcb157bbc0cfb3ad7c7de0cfb0984", + "reference": "dbe56d23de8fcb157bbc0cfb3ad7c7de0cfb0984", "shasum": "" }, "require": { @@ -2420,7 +2599,11 @@ "parser", "php" ], - "time": "2020-09-26T10:30:38+00:00" + "support": { + "issues": "https://github.com/nikic/PHP-Parser/issues", + "source": "https://github.com/nikic/PHP-Parser/tree/v4.10.3" + }, + "time": "2020-12-03T17:45:45+00:00" }, { "name": "openlss/lib-array2xml", @@ -2469,6 +2652,10 @@ "xml", "xml conversion" ], + "support": { + "issues": "https://github.com/nullivex/lib-array2xml/issues", + "source": "https://github.com/nullivex/lib-array2xml/tree/master" + }, "time": "2019-03-29T20:06:56+00:00" }, { @@ -2492,6 +2679,7 @@ "phar-io/version": "^3.0.1", "php": "^7.2 || ^8.0" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -2525,20 +2713,24 @@ } ], "description": "Component for reading phar.io manifest information from a PHP Archive (PHAR)", + "support": { + "issues": "https://github.com/phar-io/manifest/issues", + "source": "https://github.com/phar-io/manifest/tree/2.0.1" + }, "time": "2020-06-27T14:33:11+00:00" }, { "name": "phar-io/version", - "version": "3.0.2", + "version": "3.0.4", "source": { "type": "git", "url": "https://github.com/phar-io/version.git", - "reference": "c6bb6825def89e0a32220f88337f8ceaf1975fa0" + "reference": "e4782611070e50613683d2b9a57730e9a3ba5451" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phar-io/version/zipball/c6bb6825def89e0a32220f88337f8ceaf1975fa0", - "reference": "c6bb6825def89e0a32220f88337f8ceaf1975fa0", + "url": "https://api.github.com/repos/phar-io/version/zipball/e4782611070e50613683d2b9a57730e9a3ba5451", + "reference": "e4782611070e50613683d2b9a57730e9a3ba5451", "shasum": "" }, "require": { @@ -2572,7 +2764,11 @@ } ], "description": "Library for handling version information and constraints", - "time": "2020-06-27T14:39:04+00:00" + "support": { + "issues": "https://github.com/phar-io/version/issues", + "source": "https://github.com/phar-io/version/tree/3.0.4" + }, + "time": "2020-12-13T23:18:30+00:00" }, { "name": "phpdocumentor/reflection-common", @@ -2591,6 +2787,7 @@ "require": { "php": ">=7.1" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -2621,6 +2818,10 @@ "reflection", "static analysis" ], + "support": { + "issues": "https://github.com/phpDocumentor/ReflectionCommon/issues", + "source": "https://github.com/phpDocumentor/ReflectionCommon/tree/master" + }, "time": "2020-06-19T17:42:03+00:00" }, { @@ -2629,12 +2830,12 @@ "source": { "type": "git", "url": "https://github.com/phpDocumentor/ReflectionDocBlock.git", - "reference": "069a785b2141f5bcf49f3e353548dc1cce6df556" + "reference": "e3324ecbde7319b0bbcf0fd7ca4af19469c38da9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/ReflectionDocBlock/zipball/069a785b2141f5bcf49f3e353548dc1cce6df556", - "reference": "069a785b2141f5bcf49f3e353548dc1cce6df556", + "url": "https://api.github.com/repos/phpDocumentor/ReflectionDocBlock/zipball/e3324ecbde7319b0bbcf0fd7ca4af19469c38da9", + "reference": "e3324ecbde7319b0bbcf0fd7ca4af19469c38da9", "shasum": "" }, "require": { @@ -2647,6 +2848,7 @@ "require-dev": { "mockery/mockery": "~1.3.2" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -2673,7 +2875,11 @@ } ], "description": "With this component, a library can provide support for annotations via DocBlocks or otherwise retrieve information that is embedded in a DocBlock.", - "time": "2020-09-03T19:13:55+00:00" + "support": { + "issues": "https://github.com/phpDocumentor/ReflectionDocBlock/issues", + "source": "https://github.com/phpDocumentor/ReflectionDocBlock/tree/master" + }, + "time": "2020-11-18T14:27:38+00:00" }, { "name": "phpdocumentor/type-resolver", @@ -2696,6 +2902,7 @@ "require-dev": { "ext-tokenizer": "*" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -2718,6 +2925,10 @@ } ], "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", + "support": { + "issues": "https://github.com/phpDocumentor/TypeResolver/issues", + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.4.0" + }, "time": "2020-09-17T18:55:26+00:00" }, { @@ -2781,6 +2992,10 @@ "spy", "stub" ], + "support": { + "issues": "https://github.com/phpspec/prophecy/issues", + "source": "https://github.com/phpspec/prophecy/tree/1.12.1" + }, "time": "2020-09-29T09:10:42+00:00" }, { @@ -2789,12 +3004,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "5d66bde3afba51e21c6eb7d1a3776bd3b88dfafc" + "reference": "fc8fd9194f696432e3887df09098f8bacfe07f16" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/5d66bde3afba51e21c6eb7d1a3776bd3b88dfafc", - "reference": "5d66bde3afba51e21c6eb7d1a3776bd3b88dfafc", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/fc8fd9194f696432e3887df09098f8bacfe07f16", + "reference": "fc8fd9194f696432e3887df09098f8bacfe07f16", "shasum": "" }, "require": { @@ -2808,7 +3023,7 @@ "sebastian/code-unit-reverse-lookup": "^2.0.2", "sebastian/complexity": "^2.0", "sebastian/environment": "^5.1.2", - "sebastian/lines-of-code": "^1.0", + "sebastian/lines-of-code": "^1.0.3", "sebastian/version": "^3.0.1", "theseer/tokenizer": "^1.2.0" }, @@ -2819,6 +3034,7 @@ "ext-pcov": "*", "ext-xdebug": "*" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -2848,13 +3064,17 @@ "testing", "xunit" ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:25:30+00:00" + "time": "2020-12-09T08:37:03+00:00" }, { "name": "phpunit/php-file-iterator", @@ -2862,12 +3082,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-file-iterator.git", - "reference": "854c530d02bcc6b5b96942e6e4cfeead11b35aad" + "reference": "32f330e983b0ad25134c5b1a32a4cf4523178086" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/854c530d02bcc6b5b96942e6e4cfeead11b35aad", - "reference": "854c530d02bcc6b5b96942e6e4cfeead11b35aad", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/32f330e983b0ad25134c5b1a32a4cf4523178086", + "reference": "32f330e983b0ad25134c5b1a32a4cf4523178086", "shasum": "" }, "require": { @@ -2876,6 +3096,7 @@ "require-dev": { "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -2904,13 +3125,17 @@ "filesystem", "iterator" ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-file-iterator/issues", + "source": "https://github.com/sebastianbergmann/php-file-iterator/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:26:59+00:00" + "time": "2020-12-09T08:38:27+00:00" }, { "name": "phpunit/php-invoker", @@ -2918,12 +3143,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-invoker.git", - "reference": "0cd572c9935a3a8373416e733ad1566d7c602ff1" + "reference": "af6fff7777c93ca23726690b75e6b35936619a2a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-invoker/zipball/0cd572c9935a3a8373416e733ad1566d7c602ff1", - "reference": "0cd572c9935a3a8373416e733ad1566d7c602ff1", + "url": "https://api.github.com/repos/sebastianbergmann/php-invoker/zipball/af6fff7777c93ca23726690b75e6b35936619a2a", + "reference": "af6fff7777c93ca23726690b75e6b35936619a2a", "shasum": "" }, "require": { @@ -2936,6 +3161,7 @@ "suggest": { "ext-pcntl": "*" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -2963,13 +3189,17 @@ "keywords": [ "process" ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-invoker/issues", + "source": "https://github.com/sebastianbergmann/php-invoker/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:27:08+00:00" + "time": "2020-12-09T08:38:35+00:00" }, { "name": "phpunit/php-text-template", @@ -2977,12 +3207,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-text-template.git", - "reference": "0d0bdf563575ba6715c5a6754541c80a9aa6dde2" + "reference": "1875733758aac84b2dafea3f27629e0201571e60" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/0d0bdf563575ba6715c5a6754541c80a9aa6dde2", - "reference": "0d0bdf563575ba6715c5a6754541c80a9aa6dde2", + "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/1875733758aac84b2dafea3f27629e0201571e60", + "reference": "1875733758aac84b2dafea3f27629e0201571e60", "shasum": "" }, "require": { @@ -2991,6 +3221,7 @@ "require-dev": { "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3018,13 +3249,17 @@ "keywords": [ "template" ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-text-template/issues", + "source": "https://github.com/sebastianbergmann/php-text-template/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:27:42+00:00" + "time": "2020-12-09T08:39:08+00:00" }, { "name": "phpunit/php-timer", @@ -3032,12 +3267,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-timer.git", - "reference": "16bcf8ca6821c270f22794c02e50b8107cb375d8" + "reference": "e7957e44bf97c25931c802da2bb1cc42ddfdd4eb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/16bcf8ca6821c270f22794c02e50b8107cb375d8", - "reference": "16bcf8ca6821c270f22794c02e50b8107cb375d8", + "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/e7957e44bf97c25931c802da2bb1cc42ddfdd4eb", + "reference": "e7957e44bf97c25931c802da2bb1cc42ddfdd4eb", "shasum": "" }, "require": { @@ -3046,6 +3281,7 @@ "require-dev": { "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3073,13 +3309,17 @@ "keywords": [ "timer" ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-timer/issues", + "source": "https://github.com/sebastianbergmann/php-timer/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:27:16+00:00" + "time": "2020-12-09T08:38:43+00:00" }, { "name": "phpunit/phpunit", @@ -3168,6 +3408,10 @@ "testing", "xunit" ], + "support": { + "issues": "https://github.com/sebastianbergmann/phpunit/issues", + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.4.2" + }, "funding": [ { "url": "https://phpunit.de/donate.html", @@ -3197,6 +3441,7 @@ "require": { "php": ">=7.2.0" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3227,6 +3472,10 @@ "container-interop", "psr" ], + "support": { + "issues": "https://github.com/php-fig/container/issues", + "source": "https://github.com/php-fig/container/tree/master" + }, "time": "2020-10-13T07:07:53+00:00" }, { @@ -3235,12 +3484,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/cli-parser.git", - "reference": "0810c53718aa9ea7e42f2de2865a049ce3f5122f" + "reference": "922f683e943fef277c005e27b3b2ea97299d997e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/0810c53718aa9ea7e42f2de2865a049ce3f5122f", - "reference": "0810c53718aa9ea7e42f2de2865a049ce3f5122f", + "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/922f683e943fef277c005e27b3b2ea97299d997e", + "reference": "922f683e943fef277c005e27b3b2ea97299d997e", "shasum": "" }, "require": { @@ -3249,6 +3498,7 @@ "require-dev": { "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3273,26 +3523,30 @@ ], "description": "Library for parsing CLI options", "homepage": "https://github.com/sebastianbergmann/cli-parser", + "support": { + "issues": "https://github.com/sebastianbergmann/cli-parser/issues", + "source": "https://github.com/sebastianbergmann/cli-parser/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:28:10+00:00" + "time": "2020-12-14T10:23:09+00:00" }, { "name": "sebastian/code-unit", - "version": "dev-master", + "version": "1.0.8", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/code-unit.git", - "reference": "2af9894d3d514c2ee6d6ad03cc23bb4625050649" + "reference": "1fc9f64c0927627ef78ba436c9b17d967e68e120" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/code-unit/zipball/2af9894d3d514c2ee6d6ad03cc23bb4625050649", - "reference": "2af9894d3d514c2ee6d6ad03cc23bb4625050649", + "url": "https://api.github.com/repos/sebastianbergmann/code-unit/zipball/1fc9f64c0927627ef78ba436c9b17d967e68e120", + "reference": "1fc9f64c0927627ef78ba436c9b17d967e68e120", "shasum": "" }, "require": { @@ -3325,13 +3579,17 @@ ], "description": "Collection of value objects that represent the PHP code units", "homepage": "https://github.com/sebastianbergmann/code-unit", + "support": { + "issues": "https://github.com/sebastianbergmann/code-unit/issues", + "source": "https://github.com/sebastianbergmann/code-unit/tree/1.0.8" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:25:39+00:00" + "time": "2020-10-26T13:08:54+00:00" }, { "name": "sebastian/code-unit-reverse-lookup", @@ -3339,12 +3597,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/code-unit-reverse-lookup.git", - "reference": "2f9aa793a37d4f39b5cf5ee3c9a1bd506885d60c" + "reference": "bdf3ed620874ff8e19be0d8b42c475f3ffeefb18" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/code-unit-reverse-lookup/zipball/2f9aa793a37d4f39b5cf5ee3c9a1bd506885d60c", - "reference": "2f9aa793a37d4f39b5cf5ee3c9a1bd506885d60c", + "url": "https://api.github.com/repos/sebastianbergmann/code-unit-reverse-lookup/zipball/bdf3ed620874ff8e19be0d8b42c475f3ffeefb18", + "reference": "bdf3ed620874ff8e19be0d8b42c475f3ffeefb18", "shasum": "" }, "require": { @@ -3353,6 +3611,7 @@ "require-dev": { "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3376,13 +3635,17 @@ ], "description": "Looks up which function or method a line of code belongs to", "homepage": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/", + "support": { + "issues": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/issues", + "source": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:25:48+00:00" + "time": "2020-12-09T08:37:20+00:00" }, { "name": "sebastian/comparator", @@ -3390,12 +3653,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "6d3dc38d8631941ca18c76cac3d7a15838071693" + "reference": "fbecc625ac00815b8f019e932b9cc11d75ec1405" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/6d3dc38d8631941ca18c76cac3d7a15838071693", - "reference": "6d3dc38d8631941ca18c76cac3d7a15838071693", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/fbecc625ac00815b8f019e932b9cc11d75ec1405", + "reference": "fbecc625ac00815b8f019e932b9cc11d75ec1405", "shasum": "" }, "require": { @@ -3406,6 +3669,7 @@ "require-dev": { "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3446,13 +3710,17 @@ "compare", "equality" ], + "support": { + "issues": "https://github.com/sebastianbergmann/comparator/issues", + "source": "https://github.com/sebastianbergmann/comparator/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:25:57+00:00" + "time": "2020-12-09T08:37:28+00:00" }, { "name": "sebastian/complexity", @@ -3460,12 +3728,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/complexity.git", - "reference": "cf04d6dfa8f99e37a677bf67195c43fc30024282" + "reference": "cbec8097a839b5840b2f40eeff3566f5b2346ff0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/cf04d6dfa8f99e37a677bf67195c43fc30024282", - "reference": "cf04d6dfa8f99e37a677bf67195c43fc30024282", + "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/cbec8097a839b5840b2f40eeff3566f5b2346ff0", + "reference": "cbec8097a839b5840b2f40eeff3566f5b2346ff0", "shasum": "" }, "require": { @@ -3475,6 +3743,7 @@ "require-dev": { "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3499,13 +3768,17 @@ ], "description": "Library for calculating the complexity of PHP code units", "homepage": "https://github.com/sebastianbergmann/complexity", + "support": { + "issues": "https://github.com/sebastianbergmann/complexity/issues", + "source": "https://github.com/sebastianbergmann/complexity/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:27:51+00:00" + "time": "2020-12-09T08:39:17+00:00" }, { "name": "sebastian/diff", @@ -3513,12 +3786,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/diff.git", - "reference": "2c921642f8495dbf50fdcb920c425f9ccbb6a843" + "reference": "cd1158ecb8c9f33f321b96a6df746725b861d1ae" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/2c921642f8495dbf50fdcb920c425f9ccbb6a843", - "reference": "2c921642f8495dbf50fdcb920c425f9ccbb6a843", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/cd1158ecb8c9f33f321b96a6df746725b861d1ae", + "reference": "cd1158ecb8c9f33f321b96a6df746725b861d1ae", "shasum": "" }, "require": { @@ -3528,6 +3801,7 @@ "phpunit/phpunit": "^9.3", "symfony/process": "^4.2 || ^5" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3561,13 +3835,17 @@ "unidiff", "unified diff" ], + "support": { + "issues": "https://github.com/sebastianbergmann/diff/issues", + "source": "https://github.com/sebastianbergmann/diff/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:26:06+00:00" + "time": "2020-12-09T08:37:36+00:00" }, { "name": "sebastian/environment", @@ -3575,12 +3853,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/environment.git", - "reference": "6fe240eca4c1230f6442747269ff94234674f30e" + "reference": "d9a92208bc71528923f91a079f30f9cbadb5faf5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/6fe240eca4c1230f6442747269ff94234674f30e", - "reference": "6fe240eca4c1230f6442747269ff94234674f30e", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/d9a92208bc71528923f91a079f30f9cbadb5faf5", + "reference": "d9a92208bc71528923f91a079f30f9cbadb5faf5", "shasum": "" }, "require": { @@ -3592,6 +3870,7 @@ "suggest": { "ext-posix": "*" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3620,13 +3899,17 @@ "environment", "hhvm" ], + "support": { + "issues": "https://github.com/sebastianbergmann/environment/issues", + "source": "https://github.com/sebastianbergmann/environment/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:26:15+00:00" + "time": "2020-12-09T08:37:45+00:00" }, { "name": "sebastian/exporter", @@ -3634,12 +3917,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", - "reference": "04229817d2bb369fa6b25229316e7bbc53baece2" + "reference": "5ac88b179db846dbf7ea631602e5e4594280a531" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/04229817d2bb369fa6b25229316e7bbc53baece2", - "reference": "04229817d2bb369fa6b25229316e7bbc53baece2", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/5ac88b179db846dbf7ea631602e5e4594280a531", + "reference": "5ac88b179db846dbf7ea631602e5e4594280a531", "shasum": "" }, "require": { @@ -3650,6 +3933,7 @@ "ext-mbstring": "*", "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3693,13 +3977,17 @@ "export", "exporter" ], + "support": { + "issues": "https://github.com/sebastianbergmann/exporter/issues", + "source": "https://github.com/sebastianbergmann/exporter/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:26:24+00:00" + "time": "2020-12-09T08:37:53+00:00" }, { "name": "sebastian/global-state", @@ -3707,12 +3995,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "bae2a2a7c5dac8eb4ab11e178fcd88242c1f8b17" + "reference": "dbf516227d7ae4fc1a4a86419b159474d31a9c16" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/bae2a2a7c5dac8eb4ab11e178fcd88242c1f8b17", - "reference": "bae2a2a7c5dac8eb4ab11e178fcd88242c1f8b17", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/dbf516227d7ae4fc1a4a86419b159474d31a9c16", + "reference": "dbf516227d7ae4fc1a4a86419b159474d31a9c16", "shasum": "" }, "require": { @@ -3727,6 +4015,7 @@ "suggest": { "ext-uopz": "*" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3753,13 +4042,17 @@ "keywords": [ "global state" ], + "support": { + "issues": "https://github.com/sebastianbergmann/global-state/issues", + "source": "https://github.com/sebastianbergmann/global-state/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:26:33+00:00" + "time": "2020-12-09T08:38:02+00:00" }, { "name": "sebastian/lines-of-code", @@ -3767,12 +4060,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/lines-of-code.git", - "reference": "62f65495957a0d4b39045a059bbf8d6d74db06f5" + "reference": "ace45fe9f4cd96d6d3d6a210bfaf5bdd3e9e6664" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/62f65495957a0d4b39045a059bbf8d6d74db06f5", - "reference": "62f65495957a0d4b39045a059bbf8d6d74db06f5", + "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/ace45fe9f4cd96d6d3d6a210bfaf5bdd3e9e6664", + "reference": "ace45fe9f4cd96d6d3d6a210bfaf5bdd3e9e6664", "shasum": "" }, "require": { @@ -3782,6 +4075,7 @@ "require-dev": { "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3806,13 +4100,17 @@ ], "description": "Library for counting the lines of code in PHP source code", "homepage": "https://github.com/sebastianbergmann/lines-of-code", + "support": { + "issues": "https://github.com/sebastianbergmann/lines-of-code/issues", + "source": "https://github.com/sebastianbergmann/lines-of-code/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:28:00+00:00" + "time": "2020-12-09T08:39:25+00:00" }, { "name": "sebastian/object-enumerator", @@ -3820,12 +4118,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/object-enumerator.git", - "reference": "bd926d4c9372e89f8eeff4d3e1d46886b3647959" + "reference": "d3cd2a77ac07ae8bb05920b7ed8f75102dbfac36" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/object-enumerator/zipball/bd926d4c9372e89f8eeff4d3e1d46886b3647959", - "reference": "bd926d4c9372e89f8eeff4d3e1d46886b3647959", + "url": "https://api.github.com/repos/sebastianbergmann/object-enumerator/zipball/d3cd2a77ac07ae8bb05920b7ed8f75102dbfac36", + "reference": "d3cd2a77ac07ae8bb05920b7ed8f75102dbfac36", "shasum": "" }, "require": { @@ -3836,6 +4134,7 @@ "require-dev": { "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3859,13 +4158,17 @@ ], "description": "Traverses array structures and object graphs to enumerate all referenced objects", "homepage": "https://github.com/sebastianbergmann/object-enumerator/", + "support": { + "issues": "https://github.com/sebastianbergmann/object-enumerator/issues", + "source": "https://github.com/sebastianbergmann/object-enumerator/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:26:42+00:00" + "time": "2020-12-09T08:38:10+00:00" }, { "name": "sebastian/object-reflector", @@ -3873,12 +4176,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/object-reflector.git", - "reference": "2ae863c8667f2c149b20d13782d11ce23ab995f8" + "reference": "9aa0ca52ec0e20696c154eb139ec2cc5715922a7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/object-reflector/zipball/2ae863c8667f2c149b20d13782d11ce23ab995f8", - "reference": "2ae863c8667f2c149b20d13782d11ce23ab995f8", + "url": "https://api.github.com/repos/sebastianbergmann/object-reflector/zipball/9aa0ca52ec0e20696c154eb139ec2cc5715922a7", + "reference": "9aa0ca52ec0e20696c154eb139ec2cc5715922a7", "shasum": "" }, "require": { @@ -3887,6 +4190,7 @@ "require-dev": { "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3910,13 +4214,17 @@ ], "description": "Allows reflection of object attributes, including inherited and non-public ones", "homepage": "https://github.com/sebastianbergmann/object-reflector/", + "support": { + "issues": "https://github.com/sebastianbergmann/object-reflector/issues", + "source": "https://github.com/sebastianbergmann/object-reflector/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:26:50+00:00" + "time": "2020-12-09T08:38:19+00:00" }, { "name": "sebastian/recursion-context", @@ -3924,12 +4232,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/recursion-context.git", - "reference": "4c073073e68e3d2d5dd7a50ebfc53a912116b498" + "reference": "9ac02731471d22991df2591760cfa1937a6e9512" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/4c073073e68e3d2d5dd7a50ebfc53a912116b498", - "reference": "4c073073e68e3d2d5dd7a50ebfc53a912116b498", + "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/9ac02731471d22991df2591760cfa1937a6e9512", + "reference": "9ac02731471d22991df2591760cfa1937a6e9512", "shasum": "" }, "require": { @@ -3938,6 +4246,7 @@ "require-dev": { "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -3969,13 +4278,17 @@ ], "description": "Provides functionality to recursively process PHP variables", "homepage": "http://www.github.com/sebastianbergmann/recursion-context", + "support": { + "issues": "https://github.com/sebastianbergmann/recursion-context/issues", + "source": "https://github.com/sebastianbergmann/recursion-context/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:27:25+00:00" + "time": "2020-12-09T08:38:51+00:00" }, { "name": "sebastian/resource-operations", @@ -3997,6 +4310,7 @@ "require-dev": { "phpunit/phpunit": "^9.0" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -4020,6 +4334,10 @@ ], "description": "Provides a list of PHP built-in functions that operate on resources", "homepage": "https://www.github.com/sebastianbergmann/resource-operations", + "support": { + "issues": "https://github.com/sebastianbergmann/resource-operations/issues", + "source": "https://github.com/sebastianbergmann/resource-operations/tree/3.0.3" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", @@ -4034,12 +4352,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/type.git", - "reference": "dcac0d6b8489ed9975d01303f0903958ebad0ce4" + "reference": "9c58afbf51a4a1d0c738e4d7490d82fc9ae034a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/dcac0d6b8489ed9975d01303f0903958ebad0ce4", - "reference": "dcac0d6b8489ed9975d01303f0903958ebad0ce4", + "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/9c58afbf51a4a1d0c738e4d7490d82fc9ae034a9", + "reference": "9c58afbf51a4a1d0c738e4d7490d82fc9ae034a9", "shasum": "" }, "require": { @@ -4048,6 +4366,7 @@ "require-dev": { "phpunit/phpunit": "^9.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -4072,13 +4391,17 @@ ], "description": "Collection of value objects that represent the types of the PHP type system", "homepage": "https://github.com/sebastianbergmann/type", + "support": { + "issues": "https://github.com/sebastianbergmann/type/issues", + "source": "https://github.com/sebastianbergmann/type/tree/master" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" } ], - "time": "2020-11-09T10:27:33+00:00" + "time": "2020-12-09T08:39:00+00:00" }, { "name": "sebastian/version", @@ -4097,6 +4420,7 @@ "require": { "php": ">=7.3" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -4121,6 +4445,10 @@ ], "description": "Library that helps with managing the version number of Git-hosted PHP projects", "homepage": "https://github.com/sebastianbergmann/version", + "support": { + "issues": "https://github.com/sebastianbergmann/version/issues", + "source": "https://github.com/sebastianbergmann/version/tree/3.0.2" + }, "funding": [ { "url": "https://github.com/sebastianbergmann", @@ -4161,6 +4489,10 @@ } ], "description": "IDE help files for Swoole.", + "support": { + "issues": "https://github.com/swoole/ide-helper/issues", + "source": "https://github.com/swoole/ide-helper/tree/4.5.5" + }, "time": "2020-10-14T18:05:12+00:00" }, { @@ -4169,12 +4501,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "d1d8b8fd9b605630aa73b1b384e246fee54e8ffa" + "reference": "1ba3caf8ed39a49e39fc8a6cad666724524a5342" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/d1d8b8fd9b605630aa73b1b384e246fee54e8ffa", - "reference": "d1d8b8fd9b605630aa73b1b384e246fee54e8ffa", + "url": "https://api.github.com/repos/symfony/console/zipball/1ba3caf8ed39a49e39fc8a6cad666724524a5342", + "reference": "1ba3caf8ed39a49e39fc8a6cad666724524a5342", "shasum": "" }, "require": { @@ -4210,6 +4542,7 @@ "symfony/lock": "", "symfony/process": "" }, + "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -4241,6 +4574,9 @@ "console", "terminal" ], + "support": { + "source": "https://github.com/symfony/console/tree/5.x" + }, "funding": [ { "url": "https://symfony.com/sponsor", @@ -4255,7 +4591,7 @@ "type": "tidelift" } ], - "time": "2020-11-05T20:05:54+00:00" + "time": "2020-12-12T00:32:42+00:00" }, { "name": "symfony/polyfill-ctype", @@ -4277,6 +4613,7 @@ "suggest": { "ext-ctype": "For best performance" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -4317,6 +4654,9 @@ "polyfill", "portable" ], + "support": { + "source": "https://github.com/symfony/polyfill-ctype/tree/main" + }, "funding": [ { "url": "https://symfony.com/sponsor", @@ -4339,12 +4679,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", - "reference": "ee2f954ea0f9ab61dad0170eddc919ee83bef327" + "reference": "be092746c3ab9f9c62608c82e0f04687f8a879f9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/ee2f954ea0f9ab61dad0170eddc919ee83bef327", - "reference": "ee2f954ea0f9ab61dad0170eddc919ee83bef327", + "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/be092746c3ab9f9c62608c82e0f04687f8a879f9", + "reference": "be092746c3ab9f9c62608c82e0f04687f8a879f9", "shasum": "" }, "require": { @@ -4353,6 +4693,7 @@ "suggest": { "ext-intl": "For best performance" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -4395,6 +4736,9 @@ "portable", "shim" ], + "support": { + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/main" + }, "funding": [ { "url": "https://symfony.com/sponsor", @@ -4409,7 +4753,7 @@ "type": "tidelift" } ], - "time": "2020-10-26T13:35:45+00:00" + "time": "2020-11-13T15:40:22+00:00" }, { "name": "symfony/polyfill-intl-normalizer", @@ -4431,6 +4775,7 @@ "suggest": { "ext-intl": "For best performance" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -4476,6 +4821,9 @@ "portable", "shim" ], + "support": { + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/main" + }, "funding": [ { "url": "https://symfony.com/sponsor", @@ -4512,6 +4860,7 @@ "suggest": { "ext-mbstring": "For best performance" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -4553,6 +4902,9 @@ "portable", "shim" ], + "support": { + "source": "https://github.com/symfony/polyfill-mbstring/tree/main" + }, "funding": [ { "url": "https://symfony.com/sponsor", @@ -4586,6 +4938,7 @@ "require": { "php": ">=7.1" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -4629,6 +4982,9 @@ "portable", "shim" ], + "support": { + "source": "https://github.com/symfony/polyfill-php73/tree/main" + }, "funding": [ { "url": "https://symfony.com/sponsor", @@ -4662,6 +5018,7 @@ "require": { "php": ">=7.1" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -4709,6 +5066,9 @@ "portable", "shim" ], + "support": { + "source": "https://github.com/symfony/polyfill-php80/tree/main" + }, "funding": [ { "url": "https://symfony.com/sponsor", @@ -4746,6 +5106,7 @@ "suggest": { "symfony/service-implementation": "" }, + "default-branch": true, "type": "library", "extra": { "branch-version": "2.3", @@ -4786,6 +5147,9 @@ "interoperability", "standards" ], + "support": { + "source": "https://github.com/symfony/service-contracts/tree/main" + }, "funding": [ { "url": "https://symfony.com/sponsor", @@ -4808,12 +5172,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "40e975edadd4e32cd16f3753b3bad65d9ac48242" + "reference": "5bd67751d2e3f7d6f770c9154b8fbcb2aa05f7ed" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/40e975edadd4e32cd16f3753b3bad65d9ac48242", - "reference": "40e975edadd4e32cd16f3753b3bad65d9ac48242", + "url": "https://api.github.com/repos/symfony/string/zipball/5bd67751d2e3f7d6f770c9154b8fbcb2aa05f7ed", + "reference": "5bd67751d2e3f7d6f770c9154b8fbcb2aa05f7ed", "shasum": "" }, "require": { @@ -4830,6 +5194,7 @@ "symfony/translation-contracts": "^1.1|^2", "symfony/var-exporter": "^4.4|^5.0" }, + "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -4866,6 +5231,9 @@ "utf-8", "utf8" ], + "support": { + "source": "https://github.com/symfony/string/tree/5.x" + }, "funding": [ { "url": "https://symfony.com/sponsor", @@ -4880,7 +5248,7 @@ "type": "tidelift" } ], - "time": "2020-10-24T12:08:07+00:00" + "time": "2020-12-05T07:33:16+00:00" }, { "name": "theseer/tokenizer", @@ -4920,6 +5288,10 @@ } ], "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", + "support": { + "issues": "https://github.com/theseer/tokenizer/issues", + "source": "https://github.com/theseer/tokenizer/tree/master" + }, "funding": [ { "url": "https://github.com/theseer", @@ -4934,12 +5306,12 @@ "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "e133d1fd8c9f22dd9bc43d1b7fc46e0f35692226" + "reference": "c5379903e5640e5f29024e71aa0817f7bd56102b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/e133d1fd8c9f22dd9bc43d1b7fc46e0f35692226", - "reference": "e133d1fd8c9f22dd9bc43d1b7fc46e0f35692226", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/c5379903e5640e5f29024e71aa0817f7bd56102b", + "reference": "c5379903e5640e5f29024e71aa0817f7bd56102b", "shasum": "" }, "require": { @@ -4991,6 +5363,10 @@ "keywords": [ "templating" ], + "support": { + "issues": "https://github.com/twigphp/Twig/issues", + "source": "https://github.com/twigphp/Twig/tree/2.x" + }, "funding": [ { "url": "https://github.com/fabpot", @@ -5001,7 +5377,7 @@ "type": "tidelift" } ], - "time": "2020-11-09T13:33:24+00:00" + "time": "2020-11-27T13:05:37+00:00" }, { "name": "vimeo/psalm", @@ -5102,6 +5478,10 @@ "inspection", "php" ], + "support": { + "issues": "https://github.com/vimeo/psalm/issues", + "source": "https://github.com/vimeo/psalm/tree/4.1.1" + }, "time": "2020-11-02T05:54:12+00:00" }, { @@ -5151,6 +5531,10 @@ "check", "validate" ], + "support": { + "issues": "https://github.com/webmozart/assert/issues", + "source": "https://github.com/webmozart/assert/tree/master" + }, "time": "2020-07-08T17:02:28+00:00" }, { @@ -5175,6 +5559,7 @@ "phpunit/phpunit": "^4.6", "sebastian/version": "^1.0.1" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { @@ -5197,6 +5582,10 @@ } ], "description": "A robust cross-platform utility for normalizing, comparing and modifying file paths.", + "support": { + "issues": "https://github.com/webmozart/path-util/issues", + "source": "https://github.com/webmozart/path-util/tree/master" + }, "time": "2016-08-15T15:31:42+00:00" } ], @@ -5224,5 +5613,5 @@ "platform-overrides": { "php": "7.4" }, - "plugin-api-version": "1.1.0" + "plugin-api-version": "2.0.0" } From f10caa28ba9f387d1d4b3f8dfa37a1642c28d7f7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 15 Dec 2020 19:08:41 +0530 Subject: [PATCH 03/30] feat: added maintenance worker for deleting execution logs --- Dockerfile | 2 +- app/tasks/maintenance.php | 29 +++++++++++++++++++++++++---- app/workers/deletes.php | 7 ++----- composer.lock | 14 +++++++------- 4 files changed, 35 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index eccefed79e..cab447f8a4 100755 --- a/Dockerfile +++ b/Dockerfile @@ -94,7 +94,7 @@ ENV _APP_SERVER=swoole \ _APP_FUNCTIONS_CONTAINERS=10 \ _APP_SETUP=self-hosted \ _APP_VERSION=$VERSION \ - _APP_MAINTENANCE_INTERVAL=10 + _APP_MAINTENANCE_INTERVAL=864000 #ENV _APP_SMTP_SECURE '' #ENV _APP_SMTP_USERNAME '' #ENV _APP_SMTP_PASSWORD '' diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index d3099f0007..ac2501c0da 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -6,10 +6,23 @@ require_once __DIR__.'/../init.php'; use Appwrite\Database\Database; use Appwrite\Database\Document; +use Appwrite\Database\Adapter\MySQL as MySQLAdapter; +use Appwrite\Database\Adapter\Redis as RedisAdapter; +use Appwrite\Database\Validator\Authorization; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Config\Config; +function getConsoleDB() { + global $register; + $consoleDB = new Database(); + $consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register)); + $consoleDB->setNamespace('app_console'); // Main DB + $consoleDB->setMocks(Config::getParam('collections', [])); + return $consoleDB; +} + $cli ->task('maintenance') ->desc('Schedules maintenance tasks and publishes them to resque') @@ -19,19 +32,27 @@ $cli //Convert Seconds to microseconds $interval = $interval * 1000000; - Console::loop(function() { + + $consoleDB = getConsoleDB(); + + Console::loop(function() use ($consoleDB){ + + Authorization::disable(); $projects = $consoleDB->getCollection([ 'filters' => [ '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, ], ]); - var_dump("*************** MAINTENANCE WORKER *****************"); - print_r($projects); + Authorization::reset(); + + $projectIds = array_map (function ($project) { + return $project->getId(); + }, $projects); Resque::enqueue('v1-deletes', 'DeletesV1', [ 'document' => new Document([ '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, - 'projectIds' => $this->projects + 'projectIds' => $projectIds ]), ]); }, $interval); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 8bc047e5d1..18dfc4ab63 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -100,16 +100,13 @@ class DeletesV1 protected function deleteExecutionLogs(Document $document) { - var_dump("*********************DELETING EXECUTION LOGS *********************"); + $projectIds = $document->getAttribute('projectIds', []); - print_r($projectIds); foreach ($projectIds as $projectId) { if (!($projectDB = $this->getProjectDB($projectId))) { throw new Exception('Failed to get projectDB for project '.$projectId, 500); } - - var_dump("********************* DELETING FOR PROJECT *********************"); - var_dump($projectId); + // Delete Executions $this->deleteByGroup([ '$collection='.$document->getCollection(), diff --git a/composer.lock b/composer.lock index 022d6435aa..ad4499c477 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "18cec0afcb76ef57fa38386fe04fd87b", + "content-hash": "58b7669908833b2a32d550d9e38f112f", "packages": [ { "name": "appwrite/php-clamav", @@ -1268,16 +1268,16 @@ }, { "name": "utopia-php/cli", - "version": "0.7.2", + "version": "0.8", "source": { "type": "git", "url": "https://github.com/utopia-php/cli.git", - "reference": "0b19cd33b86deb6aeb26bfdabc18bd2bdf5eba04" + "reference": "090c7ae22b53a175d962e8f9601d36350496153c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cli/zipball/0b19cd33b86deb6aeb26bfdabc18bd2bdf5eba04", - "reference": "0b19cd33b86deb6aeb26bfdabc18bd2bdf5eba04", + "url": "https://api.github.com/repos/utopia-php/cli/zipball/090c7ae22b53a175d962e8f9601d36350496153c", + "reference": "090c7ae22b53a175d962e8f9601d36350496153c", "shasum": "" }, "require": { @@ -1315,9 +1315,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cli/issues", - "source": "https://github.com/utopia-php/cli/tree/0.7.2" + "source": "https://github.com/utopia-php/cli/tree/0.8" }, - "time": "2020-10-23T13:34:41+00:00" + "time": "2020-12-14T06:31:42+00:00" }, { "name": "utopia-php/config", From 96a720cb843e38ee96be4cb90ba0a1751ade11f7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 15 Dec 2020 19:10:35 +0530 Subject: [PATCH 04/30] feat: added maintenance worker for deleting execution logs --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5cd6941881..beb913218b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -287,11 +287,11 @@ services: - appwrite depends_on: - redis - - mariadb environment: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT + - _APP_MAINTENANCE_INTERVAL appwrite-schedule: entrypoint: schedule From 8f9089accc301b238fbf4dc0acfe8725dd53b0d1 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 15 Dec 2020 19:19:51 +0530 Subject: [PATCH 05/30] feat: added maintenance worker for deleting execution logs --- app/tasks/maintenance.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index ac2501c0da..de40c29523 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -32,7 +32,6 @@ $cli //Convert Seconds to microseconds $interval = $interval * 1000000; - $consoleDB = getConsoleDB(); Console::loop(function() use ($consoleDB){ From f395c2e92ce022266e2a2175d9e7e8c18741030b Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Dec 2020 20:45:47 +0530 Subject: [PATCH 06/30] feat: added notifiers for delete abuse and audit logs --- app/tasks/maintenance.php | 32 ++++++++++++++++++++++++++------ app/workers/deletes.php | 35 ++++++++++++++++++++++++++++++++++- composer.json | 4 ++-- 3 files changed, 62 insertions(+), 9 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index de40c29523..a585f79aab 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -13,6 +13,8 @@ use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; +define("DELETE_QUEUE_NAME", "v1-deletes"); +define("DELETE_CLASS_NAME", "DeletesV1"); function getConsoleDB() { global $register; @@ -23,6 +25,26 @@ function getConsoleDB() { return $consoleDB; } +function notifyDeleteExecutionLogs(array $projectIds) +{ + Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ + 'document' => new Document([ + '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, + 'projectIds' => $projectIds + ]), + ]); +} + +function notifyDeleteAbuseLogs(array $projectIds) +{ + +} + +function notifyDeleteAuditLogs(array $projectIds) +{ + +} + $cli ->task('maintenance') ->desc('Schedules maintenance tasks and publishes them to resque') @@ -48,12 +70,10 @@ $cli return $project->getId(); }, $projects); - Resque::enqueue('v1-deletes', 'DeletesV1', [ - 'document' => new Document([ - '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, - 'projectIds' => $projectIds - ]), - ]); + notifyDeleteExecutionLogs($projectIds); + notifyDeleteAbuseLogs($projectIds); + notifyDeleteAuditLogs($projectIds); + }, $interval); }); \ No newline at end of file diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 18dfc4ab63..30a58a4b92 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -14,6 +14,8 @@ use Appwrite\Database\Validator\Authorization; use Appwrite\Storage\Device\Local; use Utopia\CLI\Console; use Utopia\Config\Config; +use Utopia\Audit\Audit; +use Utopia\Audit\Adapters\MySQL as AuditAdapter; class DeletesV1 { @@ -100,7 +102,6 @@ class DeletesV1 protected function deleteExecutionLogs(Document $document) { - $projectIds = $document->getAttribute('projectIds', []); foreach ($projectIds as $projectId) { if (!($projectDB = $this->getProjectDB($projectId))) { @@ -115,6 +116,38 @@ class DeletesV1 } } + protected function deleteAbuseLogs($document) + { + global $register; + $projectIds = $document->getAttribute('projectIds', []); + + foreach ($projectIds as $projectId) { + $adapter = new AuditAdapter($register->get('db')); + $adapter->setNamespace('app_'.$projectId); + $audit = new Audit($adapter); + $status = $audit->deleteLogsOlderThan(); + if (!$status) { + throw new Exception('Failed to delete Audit logs for project'.$projectId, 500); + } + } + } + + protected function deleteAuditLogs($document) + { + global $register; + $projectIds = $document->getAttribute('projectIds', []); + + foreach ($projectIds as $projectId) { + $adapter = new AuditAdapter($register->get('db')); + $adapter->setNamespace('app_'.$projectId); + $audit = new Audit($adapter); + $status = $audit->deleteLogsOlderThan(); + if (!$status) { + throw new Exception('Failed to delete Audit logs for project'.$projectId, 500); + } + } + } + protected function deleteFunction(Document $document, $projectId) { $projectDB = $this->getProjectDB($projectId); diff --git a/composer.json b/composer.json index ed64be8ddf..404cb80850 100644 --- a/composer.json +++ b/composer.json @@ -35,8 +35,8 @@ "appwrite/php-clamav": "1.0.*", "utopia-php/framework": "0.9.8", - "utopia-php/abuse": "0.2.*", - "utopia-php/audit": "0.3.*", + "utopia-php/abuse": "0.3.*", + "utopia-php/audit": "0.4.*", "utopia-php/cache": "0.2.*", "utopia-php/cli": "0.8.0", "utopia-php/config": "0.2.*", From 1a9648a6c59a33506235d2f706f4227386c56ab3 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 17 Dec 2020 20:47:38 +0530 Subject: [PATCH 07/30] feat: added notifiers for delete abuse and audit logs --- app/tasks/maintenance.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index a585f79aab..df6373c3cd 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -37,12 +37,22 @@ function notifyDeleteExecutionLogs(array $projectIds) function notifyDeleteAbuseLogs(array $projectIds) { - + Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ + 'document' => new Document([ + '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, + 'projectIds' => $projectIds + ]), + ]); } function notifyDeleteAuditLogs(array $projectIds) { - + Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ + 'document' => new Document([ + '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, + 'projectIds' => $projectIds + ]), + ]); } $cli From c6a105db7e55ed7fe86ebb9ab7a01ccf7503f0f8 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 18 Dec 2020 16:18:00 +0530 Subject: [PATCH 08/30] patch: review comments --- Dockerfile | 4 +++- app/tasks/maintenance.php | 3 ++- app/workers/deletes.php | 4 ++-- composer.json | 2 +- composer.lock | 34 +++++++++++++++++----------------- 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index cab447f8a4..246e28d3ed 100755 --- a/Dockerfile +++ b/Dockerfile @@ -94,7 +94,9 @@ ENV _APP_SERVER=swoole \ _APP_FUNCTIONS_CONTAINERS=10 \ _APP_SETUP=self-hosted \ _APP_VERSION=$VERSION \ - _APP_MAINTENANCE_INTERVAL=864000 + # The interval at which to carry out maintenance tasks + # 1 day = 86400 seconds + _APP_MAINTENANCE_INTERVAL=86400 #ENV _APP_SMTP_SECURE '' #ENV _APP_SMTP_USERNAME '' #ENV _APP_SMTP_PASSWORD '' diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index df6373c3cd..401095fd5e 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -16,6 +16,7 @@ use Utopia\Config\Config; define("DELETE_QUEUE_NAME", "v1-deletes"); define("DELETE_CLASS_NAME", "DeletesV1"); +// TODO: Think of a better way to access consoleDB function getConsoleDB() { global $register; $consoleDB = new Database(); @@ -59,7 +60,7 @@ $cli ->task('maintenance') ->desc('Schedules maintenance tasks and publishes them to resque') ->action(function () { - // Convert string to integer + // # of days in seconds (1 day = 86400s) $interval = App::getEnv('_APP_MAINTENANCE_INTERVAL', '') + 0; //Convert Seconds to microseconds $interval = $interval * 1000000; diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 30a58a4b92..1851afebed 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -125,7 +125,7 @@ class DeletesV1 $adapter = new AuditAdapter($register->get('db')); $adapter->setNamespace('app_'.$projectId); $audit = new Audit($adapter); - $status = $audit->deleteLogsOlderThan(); + $status = $audit->cleanup(); if (!$status) { throw new Exception('Failed to delete Audit logs for project'.$projectId, 500); } @@ -141,7 +141,7 @@ class DeletesV1 $adapter = new AuditAdapter($register->get('db')); $adapter->setNamespace('app_'.$projectId); $audit = new Audit($adapter); - $status = $audit->deleteLogsOlderThan(); + $status = $audit->cleanup(); if (!$status) { throw new Exception('Failed to delete Audit logs for project'.$projectId, 500); } diff --git a/composer.json b/composer.json index 404cb80850..8628e48e21 100644 --- a/composer.json +++ b/composer.json @@ -36,7 +36,7 @@ "utopia-php/framework": "0.9.8", "utopia-php/abuse": "0.3.*", - "utopia-php/audit": "0.4.*", + "utopia-php/audit": "0.5.*", "utopia-php/cache": "0.2.*", "utopia-php/cli": "0.8.0", "utopia-php/config": "0.2.*", diff --git a/composer.lock b/composer.lock index ad4499c477..a0e20c17a0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "58b7669908833b2a32d550d9e38f112f", + "content-hash": "f366e884df2132577f44576e459e3a4b", "packages": [ { "name": "appwrite/php-clamav", @@ -1112,16 +1112,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.2.2", + "version": "0.3.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "8e65890a6d7afa9f57992f1eca9fe64508f9822e" + "reference": "5047cb311de8d7609e2678fdc405651f43f7e550" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/8e65890a6d7afa9f57992f1eca9fe64508f9822e", - "reference": "8e65890a6d7afa9f57992f1eca9fe64508f9822e", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/5047cb311de8d7609e2678fdc405651f43f7e550", + "reference": "5047cb311de8d7609e2678fdc405651f43f7e550", "shasum": "" }, "require": { @@ -1158,22 +1158,22 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.2.2" + "source": "https://github.com/utopia-php/abuse/tree/0.3.0" }, - "time": "2020-10-23T06:51:42+00:00" + "time": "2020-12-17T18:39:44+00:00" }, { "name": "utopia-php/audit", - "version": "0.3.2", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "544ecff78788d11f60992a721f102cafc22ab084" + "reference": "5000fdbe755bbfb1e8bc26c61b90e28c40e5744a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/544ecff78788d11f60992a721f102cafc22ab084", - "reference": "544ecff78788d11f60992a721f102cafc22ab084", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/5000fdbe755bbfb1e8bc26c61b90e28c40e5744a", + "reference": "5000fdbe755bbfb1e8bc26c61b90e28c40e5744a", "shasum": "" }, "require": { @@ -1210,9 +1210,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.3.2" + "source": "https://github.com/utopia-php/audit/tree/0.5.0" }, - "time": "2020-10-23T08:09:44+00:00" + "time": "2020-12-17T18:41:34+00:00" }, { "name": "utopia-php/cache", @@ -4501,12 +4501,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "1ba3caf8ed39a49e39fc8a6cad666724524a5342" + "reference": "15c96194f32e1b1aa30d1b302c71c5f83fd4dea9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/1ba3caf8ed39a49e39fc8a6cad666724524a5342", - "reference": "1ba3caf8ed39a49e39fc8a6cad666724524a5342", + "url": "https://api.github.com/repos/symfony/console/zipball/15c96194f32e1b1aa30d1b302c71c5f83fd4dea9", + "reference": "15c96194f32e1b1aa30d1b302c71c5f83fd4dea9", "shasum": "" }, "require": { @@ -4591,7 +4591,7 @@ "type": "tidelift" } ], - "time": "2020-12-12T00:32:42+00:00" + "time": "2020-12-18T08:03:24+00:00" }, { "name": "symfony/polyfill-ctype", From e3a6c899c36cb4ac188e7156c42255293f0db7b0 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 18 Dec 2020 16:38:58 +0530 Subject: [PATCH 09/30] patch: delete based on collection --- app/controllers/api/database.php | 1 + app/controllers/api/functions.php | 1 + app/controllers/api/projects.php | 5 ++++- app/controllers/api/users.php | 1 + app/controllers/general.php | 2 +- app/tasks/maintenance.php | 12 ++++++------ app/workers/deletes.php | 6 +++--- 7 files changed, 17 insertions(+), 11 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 681845957a..d8cf236157 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -253,6 +253,7 @@ App::delete('/v1/database/collections/:collectionId') $deletes ->setParam('document', $collection) + ->setParam('collection', Database::SYSTEM_COLLECTION_COLLECTIONS) ; $events diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 9eed692fc5..62f5a66596 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -366,6 +366,7 @@ App::delete('/v1/functions/:functionId') $deletes ->setParam('document', $function->getArrayCopy()) + ->setParam('collection', Database::SYSTEM_COLLECTION_FUNCTIONS) ; $response->noContent(); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 698516667b..1d3474b0e4 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -429,7 +429,10 @@ App::delete('/v1/projects/:projectId') throw new Exception('Project not found', 404); } - $deletes->setParam('document', $project->getArrayCopy()); + $deletes + ->setParam('document', $project->getArrayCopy()) + ->setParam('collection', Database::SYSTEM_COLLECTION_PROJECTS) + ; foreach (['keys', 'webhooks', 'tasks', 'platforms', 'domains'] as $key) { // Delete all children (keys, webhooks, tasks [stop tasks?], platforms) $list = $project->getAttribute('webhooks', []); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 25508226cb..4fca950fd9 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -517,6 +517,7 @@ App::delete('/v1/users/:userId') $deletes ->setParam('document', $user) + ->setParam('collection', Database::SYSTEM_COLLECTION_USERS) ; $events diff --git a/app/controllers/general.php b/app/controllers/general.php index 972cbf4110..79e9dc97d2 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -283,7 +283,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits $audits->trigger(); } - if (!empty($deletes->getParam('document'))) { + if (!empty($deletes->getParam('collection'))) { $deletes->trigger(); } diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 401095fd5e..f8c6cca599 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -29,30 +29,30 @@ function getConsoleDB() { function notifyDeleteExecutionLogs(array $projectIds) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ + 'collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, 'document' => new Document([ - '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, 'projectIds' => $projectIds - ]), + ]) ]); } function notifyDeleteAbuseLogs(array $projectIds) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ + 'collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, 'document' => new Document([ - '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, 'projectIds' => $projectIds - ]), + ]) ]); } function notifyDeleteAuditLogs(array $projectIds) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ + 'collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, 'document' => new Document([ - '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, 'projectIds' => $projectIds - ]), + ]) ]); } diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 1851afebed..7ea5e1657a 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -29,12 +29,12 @@ class DeletesV1 public function perform() { - $projectId = $this->args['projectId']; $document = $this->args['document']; - $document = new Document($document); + $projectId = $this->args['projectId']; + $collection = $this->args['collection']; - switch (strval($document->getCollection())) { + switch (strval($collection)) { case Database::SYSTEM_COLLECTION_PROJECTS: $this->deleteProject($document); break; From 4a47c593801cb37694053b54181a65a09df5e239 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 18 Dec 2020 16:43:00 +0530 Subject: [PATCH 10/30] patch: trigger only if both collection and document are present --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 79e9dc97d2..38f46009d2 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -283,7 +283,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits $audits->trigger(); } - if (!empty($deletes->getParam('collection'))) { + if (!empty($deletes->getParam('collection')) && !empty($deletes->getParam('document'))) { $deletes->trigger(); } From 1ecbf8bc5d90eb98c80d04b77361b44fd53c2d6f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 18 Dec 2020 17:58:08 +0530 Subject: [PATCH 11/30] patch: refactor delete worker invocation --- app/controllers/api/database.php | 2 +- app/controllers/api/functions.php | 2 +- app/controllers/api/projects.php | 2 +- app/controllers/api/users.php | 2 +- app/tasks/maintenance.php | 18 +++++++---- app/workers/deletes.php | 54 ++++++++++++++++++++----------- 6 files changed, 51 insertions(+), 29 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index d8cf236157..6962035882 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -253,7 +253,7 @@ App::delete('/v1/database/collections/:collectionId') $deletes ->setParam('document', $collection) - ->setParam('collection', Database::SYSTEM_COLLECTION_COLLECTIONS) + ->setParam('type', DeletesV1::TYPE_DOCUMENT) ; $events diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 62f5a66596..06daa4b33a 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -366,7 +366,7 @@ App::delete('/v1/functions/:functionId') $deletes ->setParam('document', $function->getArrayCopy()) - ->setParam('collection', Database::SYSTEM_COLLECTION_FUNCTIONS) + ->setParam('type', DeletesV1::TYPE_DOCUMENT) ; $response->noContent(); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 1d3474b0e4..0315c211ba 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -431,7 +431,7 @@ App::delete('/v1/projects/:projectId') $deletes ->setParam('document', $project->getArrayCopy()) - ->setParam('collection', Database::SYSTEM_COLLECTION_PROJECTS) + ->setParam('type', DeletesV1::TYPE_DOCUMENT) ; foreach (['keys', 'webhooks', 'tasks', 'platforms', 'domains'] as $key) { // Delete all children (keys, webhooks, tasks [stop tasks?], platforms) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 4fca950fd9..d639d50b83 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -517,7 +517,7 @@ App::delete('/v1/users/:userId') $deletes ->setParam('document', $user) - ->setParam('collection', Database::SYSTEM_COLLECTION_USERS) + ->setParam('type', DeletesV1::TYPE_DOCUMENT) ; $events diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index f8c6cca599..d826c4dd50 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -9,6 +9,7 @@ use Appwrite\Database\Document; use Appwrite\Database\Adapter\MySQL as MySQLAdapter; use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Database\Validator\Authorization; +use Swoole\FastCGI\Record\Data; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -29,29 +30,32 @@ function getConsoleDB() { function notifyDeleteExecutionLogs(array $projectIds) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ - 'collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, + 'type' => DeletesV1::TYPE_DOCUMENT, 'document' => new Document([ + '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, 'projectIds' => $projectIds ]) ]); } -function notifyDeleteAbuseLogs(array $projectIds) +function notifyDeleteAbuseLogs(array $projectIds, int $timestamp) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ - 'collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, + 'type' => DeletesV1::TYPE_ABUSE, 'document' => new Document([ - 'projectIds' => $projectIds + 'projectIds' => $projectIds, + 'timestamp' => $timestamp, ]) ]); } -function notifyDeleteAuditLogs(array $projectIds) +function notifyDeleteAuditLogs(array $projectIds, int $timestamp) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ - 'collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, + 'type' => DeletesV1::TYPE_AUDIT, 'document' => new Document([ - 'projectIds' => $projectIds + 'projectIds' => $projectIds, + 'timestamp' => $timestamp ]) ]); } diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 7ea5e1657a..b0e9b6deb8 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -19,6 +19,12 @@ use Utopia\Audit\Adapters\MySQL as AuditAdapter; class DeletesV1 { + + // Deletion Types + const TYPE_DOCUMENT = 'document'; + const TYPE_AUDIT = 'audit'; + const TYPE_ABUSE = 'abuse'; + public $args = []; protected $consoleDB = null; @@ -32,28 +38,39 @@ class DeletesV1 $document = $this->args['document']; $document = new Document($document); $projectId = $this->args['projectId']; - $collection = $this->args['collection']; + $type = $this->args['type']; - switch (strval($collection)) { - case Database::SYSTEM_COLLECTION_PROJECTS: - $this->deleteProject($document); + switch (strval($type)) { + case DeletesV1::TYPE_DOCUMENT: + switch (strval($document->getCollection())) { + case Database::SYSTEM_COLLECTION_PROJECTS: + $this->deleteProject($document); + break; + case Database::SYSTEM_COLLECTION_FUNCTIONS: + $this->deleteFunction($document, $projectId); + break; + case Database::SYSTEM_COLLECTION_EXECUTIONS: + $this->deleteExecutionLogs($document); + break; + case Database::SYSTEM_COLLECTION_USERS: + $this->deleteUser($document, $projectId); + break; + case Database::SYSTEM_COLLECTION_COLLECTIONS: + $this->deleteDocuments($document, $projectId); + break; + default: + Console::error('No lazy delete operation available for document of type: '.$document->getCollection()); + break; + } break; - case Database::SYSTEM_COLLECTION_FUNCTIONS: - $this->deleteFunction($document, $projectId); + case DeletesV1::TYPE_AUDIT: + $this->deleteAuditLogs($document); break; - case Database::SYSTEM_COLLECTION_EXECUTIONS: - $this->deleteExecutionLogs($document); + + case DeletesV1::TYPE_ABUSE: + $this->deleteAbuseLogs($document); break; - case Database::SYSTEM_COLLECTION_USERS: - $this->deleteUser($document, $projectId); - break; - case Database::SYSTEM_COLLECTION_COLLECTIONS: - $this->deleteDocuments($document, $projectId); - break; - default: - Console::error('No lazy delete operation available for document of type: '.$document->getCollection()); - break; - } + } } public function tearDown(): void @@ -136,6 +153,7 @@ class DeletesV1 { global $register; $projectIds = $document->getAttribute('projectIds', []); + $timestamp = $document->getAttribute('time', 0); foreach ($projectIds as $projectId) { $adapter = new AuditAdapter($register->get('db')); From b21e474081e3a8e663fe1e9bdf6c3159712ebb79 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 18 Dec 2020 17:58:36 +0530 Subject: [PATCH 12/30] patch: trigger only if both type and document are present --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 38f46009d2..9bc13632d2 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -283,7 +283,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits $audits->trigger(); } - if (!empty($deletes->getParam('collection')) && !empty($deletes->getParam('document'))) { + if (!empty($deletes->getParam('type')) && !empty($deletes->getParam('document'))) { $deletes->trigger(); } From 62885aa13ac748d11daefc7e0ded3344e4a7530f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 18 Dec 2020 19:35:15 +0530 Subject: [PATCH 13/30] patch: deletes worker refactoring --- app/controllers/api/database.php | 2 +- app/controllers/api/functions.php | 2 +- app/controllers/api/projects.php | 2 +- app/controllers/api/users.php | 2 +- app/init.php | 6 +++++- app/tasks/maintenance.php | 24 ++++++++++++------------ app/workers/deletes.php | 28 +++++++++++++++++----------- 7 files changed, 38 insertions(+), 28 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 6962035882..7e9450830e 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -253,7 +253,7 @@ App::delete('/v1/database/collections/:collectionId') $deletes ->setParam('document', $collection) - ->setParam('type', DeletesV1::TYPE_DOCUMENT) + ->setParam('type', DELETE_TYPE_DOCUMENT) ; $events diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 06daa4b33a..a664d0ebb7 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -366,7 +366,7 @@ App::delete('/v1/functions/:functionId') $deletes ->setParam('document', $function->getArrayCopy()) - ->setParam('type', DeletesV1::TYPE_DOCUMENT) + ->setParam('type', DELETE_TYPE_DOCUMENT) ; $response->noContent(); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 0315c211ba..3d9b33f012 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -431,7 +431,7 @@ App::delete('/v1/projects/:projectId') $deletes ->setParam('document', $project->getArrayCopy()) - ->setParam('type', DeletesV1::TYPE_DOCUMENT) + ->setParam('type', DELETE_TYPE_DOCUMENT) ; foreach (['keys', 'webhooks', 'tasks', 'platforms', 'domains'] as $key) { // Delete all children (keys, webhooks, tasks [stop tasks?], platforms) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index d639d50b83..cbddcac56c 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -517,7 +517,7 @@ App::delete('/v1/users/:userId') $deletes ->setParam('document', $user) - ->setParam('type', DeletesV1::TYPE_DOCUMENT) + ->setParam('type', DELETE_TYPE_DOCUMENT) ; $events diff --git a/app/init.php b/app/init.php index acaf6277d9..1db02ead44 100644 --- a/app/init.php +++ b/app/init.php @@ -52,7 +52,11 @@ const APP_SOCIAL_INSTAGRAM = 'https://www.instagram.com/appwrite.io'; const APP_SOCIAL_GITHUB = 'https://github.com/appwrite'; const APP_SOCIAL_DISCORD = 'https://appwrite.io/discord'; const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; -const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; +const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; +// Deletion Types +const DELETE_TYPE_DOCUMENT = 'document'; +const DELETE_TYPE_AUDIT = 'audit'; +const DELETE_TYPE_ABUSE = 'abuse'; $register = new Registry(); diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index d826c4dd50..31171a1d1b 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -30,7 +30,7 @@ function getConsoleDB() { function notifyDeleteExecutionLogs(array $projectIds) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ - 'type' => DeletesV1::TYPE_DOCUMENT, + 'type' => DELETE_TYPE_DOCUMENT, 'document' => new Document([ '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, 'projectIds' => $projectIds @@ -38,24 +38,24 @@ function notifyDeleteExecutionLogs(array $projectIds) ]); } -function notifyDeleteAbuseLogs(array $projectIds, int $timestamp) +function notifyDeleteAbuseLogs(array $projectIds, int $interval) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ - 'type' => DeletesV1::TYPE_ABUSE, + 'type' => DELETE_TYPE_ABUSE, 'document' => new Document([ 'projectIds' => $projectIds, - 'timestamp' => $timestamp, + 'timestamp' => time() - $interval, ]) ]); } -function notifyDeleteAuditLogs(array $projectIds, int $timestamp) +function notifyDeleteAuditLogs(array $projectIds, int $interval) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ - 'type' => DeletesV1::TYPE_AUDIT, + 'type' => DELETE_TYPE_AUDIT, 'document' => new Document([ 'projectIds' => $projectIds, - 'timestamp' => $timestamp + 'timestamp' => time() - $interval ]) ]); } @@ -67,11 +67,11 @@ $cli // # of days in seconds (1 day = 86400s) $interval = App::getEnv('_APP_MAINTENANCE_INTERVAL', '') + 0; //Convert Seconds to microseconds - $interval = $interval * 1000000; + $intervalMicroseconds = $interval * 1000000; $consoleDB = getConsoleDB(); - Console::loop(function() use ($consoleDB){ + Console::loop(function() use ($consoleDB, $interval){ Authorization::disable(); $projects = $consoleDB->getCollection([ @@ -86,9 +86,9 @@ $cli }, $projects); notifyDeleteExecutionLogs($projectIds); - notifyDeleteAbuseLogs($projectIds); - notifyDeleteAuditLogs($projectIds); + notifyDeleteAbuseLogs($projectIds, $interval); + notifyDeleteAuditLogs($projectIds, $interval); - }, $interval); + }, $intervalMicroseconds); }); \ No newline at end of file diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b0e9b6deb8..7d6752e29b 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -20,11 +20,6 @@ use Utopia\Audit\Adapters\MySQL as AuditAdapter; class DeletesV1 { - // Deletion Types - const TYPE_DOCUMENT = 'document'; - const TYPE_AUDIT = 'audit'; - const TYPE_ABUSE = 'abuse'; - public $args = []; protected $consoleDB = null; @@ -38,10 +33,11 @@ class DeletesV1 $document = $this->args['document']; $document = new Document($document); $projectId = $this->args['projectId']; + $type = $this->args['type']; switch (strval($type)) { - case DeletesV1::TYPE_DOCUMENT: + case DELETE_TYPE_DOCUMENT: switch (strval($document->getCollection())) { case Database::SYSTEM_COLLECTION_PROJECTS: $this->deleteProject($document); @@ -63,11 +59,11 @@ class DeletesV1 break; } break; - case DeletesV1::TYPE_AUDIT: + case DELETE_TYPE_AUDIT: $this->deleteAuditLogs($document); break; - case DeletesV1::TYPE_ABUSE: + case DELETE_TYPE_ABUSE: $this->deleteAbuseLogs($document); break; } @@ -137,29 +133,39 @@ class DeletesV1 { global $register; $projectIds = $document->getAttribute('projectIds', []); + $timestamp = $document->getAttribute('timestamp', 0); + + if($timestamp == 0) { + throw new Exception('Failed to delete abuse logs. No timestamp provided'); + } foreach ($projectIds as $projectId) { $adapter = new AuditAdapter($register->get('db')); $adapter->setNamespace('app_'.$projectId); $audit = new Audit($adapter); - $status = $audit->cleanup(); + $status = $audit->cleanup($timestamp); if (!$status) { throw new Exception('Failed to delete Audit logs for project'.$projectId, 500); } } + } protected function deleteAuditLogs($document) { global $register; $projectIds = $document->getAttribute('projectIds', []); - $timestamp = $document->getAttribute('time', 0); + $timestamp = $document->getAttribute('timestamp', 0); + + if($timestamp == 0) { + throw new Exception('Failed to delete audit logs. No timestamp provided'); + } foreach ($projectIds as $projectId) { $adapter = new AuditAdapter($register->get('db')); $adapter->setNamespace('app_'.$projectId); $audit = new Audit($adapter); - $status = $audit->cleanup(); + $status = $audit->cleanup($timestamp); if (!$status) { throw new Exception('Failed to delete Audit logs for project'.$projectId, 500); } From 9a05fedb6a6f6e5a1ff17ef128a3a0de9ddd143d Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 18 Dec 2020 19:38:28 +0530 Subject: [PATCH 14/30] patch: deletes worker for abuse logs --- app/workers/deletes.php | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 7d6752e29b..9b673159d6 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -12,6 +12,8 @@ use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Database\Document; use Appwrite\Database\Validator\Authorization; use Appwrite\Storage\Device\Local; +use Utopia\Abuse\Abuse; +use Utopia\Abuse\Adapters\TimeLimit; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Audit\Audit; @@ -140,12 +142,15 @@ class DeletesV1 } foreach ($projectIds as $projectId) { - $adapter = new AuditAdapter($register->get('db')); - $adapter->setNamespace('app_'.$projectId); - $audit = new Audit($adapter); - $status = $audit->cleanup($timestamp); + $timeLimit = new TimeLimit("", 0, 0, function () use ($register) { + return $register->get('db'); + }); + $timeLimit->setNamespace('app_'.$projectId); + $abuse = new Abuse($timeLimit); + + $status = $abuse->cleanup($timestamp); if (!$status) { - throw new Exception('Failed to delete Audit logs for project'.$projectId, 500); + throw new Exception('Failed to delete Abuse logs for project '.$projectId, 500); } } From 4049f6470d4591a17a755d9ddec9938e0b22a979 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sat, 19 Dec 2020 13:38:03 +0530 Subject: [PATCH 15/30] patch: added logs for maintenance task --- app/tasks/maintenance.php | 3 ++- app/workers/deletes.php | 17 +++++++++++------ app/workers/mails.php | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 31171a1d1b..bf2159eb09 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -9,7 +9,6 @@ use Appwrite\Database\Document; use Appwrite\Database\Adapter\MySQL as MySQLAdapter; use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Database\Validator\Authorization; -use Swoole\FastCGI\Record\Data; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -85,6 +84,8 @@ $cli return $project->getId(); }, $projects); + Console::info("[ MAINTENANCE ] Notifying deletes workers every {$interval} seconds"); + notifyDeleteExecutionLogs($projectIds); notifyDeleteAbuseLogs($projectIds, $interval); notifyDeleteAuditLogs($projectIds, $interval); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 9b673159d6..eccbf6d0c3 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -1,11 +1,5 @@ deleteAbuseLogs($document); break; + + default: + Console::error('No delete operation for type: '.$type); + break; + } } diff --git a/app/workers/mails.php b/app/workers/mails.php index a7f6f6c235..5431ee9225 100644 --- a/app/workers/mails.php +++ b/app/workers/mails.php @@ -7,7 +7,7 @@ require_once __DIR__.'/../init.php'; \cli_set_process_title('Mails V1 Worker'); -echo APP_NAME.' mails worker v1 has started'."\n"; +Console::success(APP_NAME.' mails worker v1 has started'."\n"); class MailsV1 { From 2085c16f914fb461d8b33a67bcbc7b0fa16ff0bc Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 21 Dec 2020 20:16:31 +0530 Subject: [PATCH 16/30] patch: added logs for maintenance task --- .env | 1 + Dockerfile | 4 +- app/tasks/maintenance.php | 8 +-- app/workers/deletes.php | 8 ++- composer.lock | 138 +++++++++++++++++++------------------- docker-compose.yml | 8 ++- 6 files changed, 87 insertions(+), 80 deletions(-) diff --git a/.env b/.env index 9764cfe894..2f87526391 100644 --- a/.env +++ b/.env @@ -27,3 +27,4 @@ _APP_SMTP_PASSWORD= _APP_STORAGE_LIMIT=10000000 _APP_FUNCTIONS_TIMEOUT=900 _APP_FUNCTIONS_CONTAINERS=10 +_APP_MAINTENANCE_INTERVAL=60 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 246e28d3ed..70112f973d 100755 --- a/Dockerfile +++ b/Dockerfile @@ -94,9 +94,7 @@ ENV _APP_SERVER=swoole \ _APP_FUNCTIONS_CONTAINERS=10 \ _APP_SETUP=self-hosted \ _APP_VERSION=$VERSION \ - # The interval at which to carry out maintenance tasks - # 1 day = 86400 seconds - _APP_MAINTENANCE_INTERVAL=86400 + _APP_MAINTENANCE_INTERVAL=60 #ENV _APP_SMTP_SECURE '' #ENV _APP_SMTP_USERNAME '' #ENV _APP_SMTP_PASSWORD '' diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index bf2159eb09..09a62928b6 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -84,11 +84,11 @@ $cli return $project->getId(); }, $projects); - Console::info("[ MAINTENANCE ] Notifying deletes workers every {$interval} seconds"); - - notifyDeleteExecutionLogs($projectIds); + Console::info("[ MAINTENANCE TASK ] Notifying deletes workers every {$interval} seconds"); + + // notifyDeleteExecutionLogs($projectIds); notifyDeleteAbuseLogs($projectIds, $interval); - notifyDeleteAuditLogs($projectIds, $interval); + // notifyDeleteAuditLogs($projectIds, $interval); }, $intervalMicroseconds); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index eccbf6d0c3..f7fd9439cb 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -146,10 +146,12 @@ class DeletesV1 throw new Exception('Failed to delete abuse logs. No timestamp provided'); } + $timeLimit = new TimeLimit("", 0, 1, function () use ($register) { + return $register->get('db'); + }); + foreach ($projectIds as $projectId) { - $timeLimit = new TimeLimit("", 0, 0, function () use ($register) { - return $register->get('db'); - }); + Console::success("Deleting abuse logs for Project: ", $projectId); $timeLimit->setNamespace('app_'.$projectId); $abuse = new Abuse($timeLimit); diff --git a/composer.lock b/composer.lock index a0e20c17a0..e070af3683 100644 --- a/composer.lock +++ b/composer.lock @@ -3004,12 +3004,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "fc8fd9194f696432e3887df09098f8bacfe07f16" + "reference": "3952570aeea0f38f8523a4bbb647a6d45797220c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/fc8fd9194f696432e3887df09098f8bacfe07f16", - "reference": "fc8fd9194f696432e3887df09098f8bacfe07f16", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/3952570aeea0f38f8523a4bbb647a6d45797220c", + "reference": "3952570aeea0f38f8523a4bbb647a6d45797220c", "shasum": "" }, "require": { @@ -3074,7 +3074,7 @@ "type": "github" } ], - "time": "2020-12-09T08:37:03+00:00" + "time": "2020-12-18T12:30:12+00:00" }, { "name": "phpunit/php-file-iterator", @@ -3082,12 +3082,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-file-iterator.git", - "reference": "32f330e983b0ad25134c5b1a32a4cf4523178086" + "reference": "20954647176fc684ea80b3801d73eab6d3734b58" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/32f330e983b0ad25134c5b1a32a4cf4523178086", - "reference": "32f330e983b0ad25134c5b1a32a4cf4523178086", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/20954647176fc684ea80b3801d73eab6d3734b58", + "reference": "20954647176fc684ea80b3801d73eab6d3734b58", "shasum": "" }, "require": { @@ -3135,7 +3135,7 @@ "type": "github" } ], - "time": "2020-12-09T08:38:27+00:00" + "time": "2020-12-18T12:31:37+00:00" }, { "name": "phpunit/php-invoker", @@ -3207,12 +3207,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-text-template.git", - "reference": "1875733758aac84b2dafea3f27629e0201571e60" + "reference": "1bf5e9937d86cfe12c90018ccb081dfb0ceee770" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/1875733758aac84b2dafea3f27629e0201571e60", - "reference": "1875733758aac84b2dafea3f27629e0201571e60", + "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/1bf5e9937d86cfe12c90018ccb081dfb0ceee770", + "reference": "1bf5e9937d86cfe12c90018ccb081dfb0ceee770", "shasum": "" }, "require": { @@ -3259,7 +3259,7 @@ "type": "github" } ], - "time": "2020-12-09T08:39:08+00:00" + "time": "2020-12-18T12:32:19+00:00" }, { "name": "phpunit/php-timer", @@ -3267,12 +3267,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-timer.git", - "reference": "e7957e44bf97c25931c802da2bb1cc42ddfdd4eb" + "reference": "fe5620838118e9e4cef2ed462923e1f59016e8f4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/e7957e44bf97c25931c802da2bb1cc42ddfdd4eb", - "reference": "e7957e44bf97c25931c802da2bb1cc42ddfdd4eb", + "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/fe5620838118e9e4cef2ed462923e1f59016e8f4", + "reference": "fe5620838118e9e4cef2ed462923e1f59016e8f4", "shasum": "" }, "require": { @@ -3319,7 +3319,7 @@ "type": "github" } ], - "time": "2020-12-09T08:38:43+00:00" + "time": "2020-12-18T12:31:54+00:00" }, { "name": "phpunit/phpunit", @@ -3484,12 +3484,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/cli-parser.git", - "reference": "922f683e943fef277c005e27b3b2ea97299d997e" + "reference": "b887289b80dd8394719d40e6e7fcef74203db570" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/922f683e943fef277c005e27b3b2ea97299d997e", - "reference": "922f683e943fef277c005e27b3b2ea97299d997e", + "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/b887289b80dd8394719d40e6e7fcef74203db570", + "reference": "b887289b80dd8394719d40e6e7fcef74203db570", "shasum": "" }, "require": { @@ -3533,7 +3533,7 @@ "type": "github" } ], - "time": "2020-12-14T10:23:09+00:00" + "time": "2020-12-18T12:32:47+00:00" }, { "name": "sebastian/code-unit", @@ -3597,12 +3597,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/code-unit-reverse-lookup.git", - "reference": "bdf3ed620874ff8e19be0d8b42c475f3ffeefb18" + "reference": "2e42a717208897d00b868a7086776ac89f676d0f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/code-unit-reverse-lookup/zipball/bdf3ed620874ff8e19be0d8b42c475f3ffeefb18", - "reference": "bdf3ed620874ff8e19be0d8b42c475f3ffeefb18", + "url": "https://api.github.com/repos/sebastianbergmann/code-unit-reverse-lookup/zipball/2e42a717208897d00b868a7086776ac89f676d0f", + "reference": "2e42a717208897d00b868a7086776ac89f676d0f", "shasum": "" }, "require": { @@ -3645,7 +3645,7 @@ "type": "github" } ], - "time": "2020-12-09T08:37:20+00:00" + "time": "2020-12-18T12:30:29+00:00" }, { "name": "sebastian/comparator", @@ -3653,12 +3653,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "fbecc625ac00815b8f019e932b9cc11d75ec1405" + "reference": "f8a802b2820f3ed7aa72d684cc7342edef07ee90" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/fbecc625ac00815b8f019e932b9cc11d75ec1405", - "reference": "fbecc625ac00815b8f019e932b9cc11d75ec1405", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/f8a802b2820f3ed7aa72d684cc7342edef07ee90", + "reference": "f8a802b2820f3ed7aa72d684cc7342edef07ee90", "shasum": "" }, "require": { @@ -3720,7 +3720,7 @@ "type": "github" } ], - "time": "2020-12-09T08:37:28+00:00" + "time": "2020-12-18T12:30:37+00:00" }, { "name": "sebastian/complexity", @@ -3728,12 +3728,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/complexity.git", - "reference": "cbec8097a839b5840b2f40eeff3566f5b2346ff0" + "reference": "431f6c8278fadf1f978a367fb5b50d3fe79257a4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/cbec8097a839b5840b2f40eeff3566f5b2346ff0", - "reference": "cbec8097a839b5840b2f40eeff3566f5b2346ff0", + "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/431f6c8278fadf1f978a367fb5b50d3fe79257a4", + "reference": "431f6c8278fadf1f978a367fb5b50d3fe79257a4", "shasum": "" }, "require": { @@ -3778,7 +3778,7 @@ "type": "github" } ], - "time": "2020-12-09T08:39:17+00:00" + "time": "2020-12-18T12:32:27+00:00" }, { "name": "sebastian/diff", @@ -3786,12 +3786,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/diff.git", - "reference": "cd1158ecb8c9f33f321b96a6df746725b861d1ae" + "reference": "ee2368af73650a4bb1af6bf9e958344001d7e68f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/cd1158ecb8c9f33f321b96a6df746725b861d1ae", - "reference": "cd1158ecb8c9f33f321b96a6df746725b861d1ae", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/ee2368af73650a4bb1af6bf9e958344001d7e68f", + "reference": "ee2368af73650a4bb1af6bf9e958344001d7e68f", "shasum": "" }, "require": { @@ -3845,7 +3845,7 @@ "type": "github" } ], - "time": "2020-12-09T08:37:36+00:00" + "time": "2020-12-18T12:30:46+00:00" }, { "name": "sebastian/environment", @@ -3853,12 +3853,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/environment.git", - "reference": "d9a92208bc71528923f91a079f30f9cbadb5faf5" + "reference": "c6910747e97d6838d06dc72e2ea9f60b24f574b9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/d9a92208bc71528923f91a079f30f9cbadb5faf5", - "reference": "d9a92208bc71528923f91a079f30f9cbadb5faf5", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/c6910747e97d6838d06dc72e2ea9f60b24f574b9", + "reference": "c6910747e97d6838d06dc72e2ea9f60b24f574b9", "shasum": "" }, "require": { @@ -3909,7 +3909,7 @@ "type": "github" } ], - "time": "2020-12-09T08:37:45+00:00" + "time": "2020-12-18T12:30:54+00:00" }, { "name": "sebastian/exporter", @@ -3917,12 +3917,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", - "reference": "5ac88b179db846dbf7ea631602e5e4594280a531" + "reference": "25bad5473a55028cff0d58a55812ecfd52344abd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/5ac88b179db846dbf7ea631602e5e4594280a531", - "reference": "5ac88b179db846dbf7ea631602e5e4594280a531", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/25bad5473a55028cff0d58a55812ecfd52344abd", + "reference": "25bad5473a55028cff0d58a55812ecfd52344abd", "shasum": "" }, "require": { @@ -3987,7 +3987,7 @@ "type": "github" } ], - "time": "2020-12-09T08:37:53+00:00" + "time": "2020-12-18T12:31:04+00:00" }, { "name": "sebastian/global-state", @@ -3995,12 +3995,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "dbf516227d7ae4fc1a4a86419b159474d31a9c16" + "reference": "27db5bebcc77aa943d8e30e0346f231eaeb286ce" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/dbf516227d7ae4fc1a4a86419b159474d31a9c16", - "reference": "dbf516227d7ae4fc1a4a86419b159474d31a9c16", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/27db5bebcc77aa943d8e30e0346f231eaeb286ce", + "reference": "27db5bebcc77aa943d8e30e0346f231eaeb286ce", "shasum": "" }, "require": { @@ -4052,7 +4052,7 @@ "type": "github" } ], - "time": "2020-12-09T08:38:02+00:00" + "time": "2020-12-18T12:31:12+00:00" }, { "name": "sebastian/lines-of-code", @@ -4060,12 +4060,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/lines-of-code.git", - "reference": "ace45fe9f4cd96d6d3d6a210bfaf5bdd3e9e6664" + "reference": "d998f7a3b6abedf9d2ed93c7b3f7ffc6cf0f6acb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/ace45fe9f4cd96d6d3d6a210bfaf5bdd3e9e6664", - "reference": "ace45fe9f4cd96d6d3d6a210bfaf5bdd3e9e6664", + "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/d998f7a3b6abedf9d2ed93c7b3f7ffc6cf0f6acb", + "reference": "d998f7a3b6abedf9d2ed93c7b3f7ffc6cf0f6acb", "shasum": "" }, "require": { @@ -4110,7 +4110,7 @@ "type": "github" } ], - "time": "2020-12-09T08:39:25+00:00" + "time": "2020-12-18T12:32:36+00:00" }, { "name": "sebastian/object-enumerator", @@ -4118,12 +4118,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/object-enumerator.git", - "reference": "d3cd2a77ac07ae8bb05920b7ed8f75102dbfac36" + "reference": "31a6d04d6b49c0b6a1499eaac1a395a766a4176b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/object-enumerator/zipball/d3cd2a77ac07ae8bb05920b7ed8f75102dbfac36", - "reference": "d3cd2a77ac07ae8bb05920b7ed8f75102dbfac36", + "url": "https://api.github.com/repos/sebastianbergmann/object-enumerator/zipball/31a6d04d6b49c0b6a1499eaac1a395a766a4176b", + "reference": "31a6d04d6b49c0b6a1499eaac1a395a766a4176b", "shasum": "" }, "require": { @@ -4168,7 +4168,7 @@ "type": "github" } ], - "time": "2020-12-09T08:38:10+00:00" + "time": "2020-12-18T12:31:20+00:00" }, { "name": "sebastian/object-reflector", @@ -4176,12 +4176,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/object-reflector.git", - "reference": "9aa0ca52ec0e20696c154eb139ec2cc5715922a7" + "reference": "b111d3533226d210f133d7b7e45783765794067f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/object-reflector/zipball/9aa0ca52ec0e20696c154eb139ec2cc5715922a7", - "reference": "9aa0ca52ec0e20696c154eb139ec2cc5715922a7", + "url": "https://api.github.com/repos/sebastianbergmann/object-reflector/zipball/b111d3533226d210f133d7b7e45783765794067f", + "reference": "b111d3533226d210f133d7b7e45783765794067f", "shasum": "" }, "require": { @@ -4224,7 +4224,7 @@ "type": "github" } ], - "time": "2020-12-09T08:38:19+00:00" + "time": "2020-12-18T12:31:28+00:00" }, { "name": "sebastian/recursion-context", @@ -4232,12 +4232,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/recursion-context.git", - "reference": "9ac02731471d22991df2591760cfa1937a6e9512" + "reference": "d1e4fef0e0cf6c2becafc1f53169339a6e426fb8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/9ac02731471d22991df2591760cfa1937a6e9512", - "reference": "9ac02731471d22991df2591760cfa1937a6e9512", + "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/d1e4fef0e0cf6c2becafc1f53169339a6e426fb8", + "reference": "d1e4fef0e0cf6c2becafc1f53169339a6e426fb8", "shasum": "" }, "require": { @@ -4288,7 +4288,7 @@ "type": "github" } ], - "time": "2020-12-09T08:38:51+00:00" + "time": "2020-12-18T12:32:02+00:00" }, { "name": "sebastian/resource-operations", @@ -4352,12 +4352,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/type.git", - "reference": "9c58afbf51a4a1d0c738e4d7490d82fc9ae034a9" + "reference": "a2baeeb40f81ad4b7348975332c6825b8f759871" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/9c58afbf51a4a1d0c738e4d7490d82fc9ae034a9", - "reference": "9c58afbf51a4a1d0c738e4d7490d82fc9ae034a9", + "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/a2baeeb40f81ad4b7348975332c6825b8f759871", + "reference": "a2baeeb40f81ad4b7348975332c6825b8f759871", "shasum": "" }, "require": { @@ -4401,7 +4401,7 @@ "type": "github" } ], - "time": "2020-12-09T08:39:00+00:00" + "time": "2020-12-18T12:32:10+00:00" }, { "name": "sebastian/version", @@ -5232,7 +5232,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/5.x" + "source": "https://github.com/symfony/string/tree/v5.2.1" }, "funding": [ { diff --git a/docker-compose.yml b/docker-compose.yml index beb913218b..83bbd5e76e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,7 +55,7 @@ services: - ./psalm.xml:/usr/src/code/psalm.xml - ./tests:/usr/src/code/tests - ./app:/usr/src/code/app - # - ./vendor:/usr/src/code/vendor + - ./vendor:/usr/src/code/vendor - ./docs:/usr/src/code/docs - ./public:/usr/src/code/public - ./src:/usr/src/code/src @@ -191,6 +191,7 @@ services: volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw + - ./vendor:/usr/src/code/vendor environment: - _APP_ENV - _APP_REDIS_HOST @@ -292,6 +293,11 @@ services: - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_MAINTENANCE_INTERVAL + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS appwrite-schedule: entrypoint: schedule From b40b58634232b33dff97223ca3c85176a6a47f99 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 21 Dec 2020 23:45:52 +0530 Subject: [PATCH 17/30] patch: review comments and some fixes --- app/controllers/api/database.php | 2 +- app/controllers/api/functions.php | 2 +- app/controllers/api/projects.php | 2 +- app/controllers/api/users.php | 2 +- app/init.php | 1 + app/tasks/maintenance.php | 39 ++++++----------------- app/workers/deletes.php | 41 +++++++++++++++++------- composer.json | 4 +-- composer.lock | 52 +++++++++++++++---------------- docker-compose.yml | 3 +- 10 files changed, 75 insertions(+), 73 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 7e9450830e..0a8dec4421 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -252,8 +252,8 @@ App::delete('/v1/database/collections/:collectionId') } $deletes - ->setParam('document', $collection) ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $collection) ; $events diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index a664d0ebb7..476db04724 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -365,8 +365,8 @@ App::delete('/v1/functions/:functionId') } $deletes - ->setParam('document', $function->getArrayCopy()) ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $function->getArrayCopy()) ; $response->noContent(); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 3d9b33f012..f8aa16936f 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -430,8 +430,8 @@ App::delete('/v1/projects/:projectId') } $deletes - ->setParam('document', $project->getArrayCopy()) ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $project->getArrayCopy()) ; foreach (['keys', 'webhooks', 'tasks', 'platforms', 'domains'] as $key) { // Delete all children (keys, webhooks, tasks [stop tasks?], platforms) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index cbddcac56c..90ff6cd1b9 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -516,8 +516,8 @@ App::delete('/v1/users/:userId') } $deletes - ->setParam('document', $user) ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $user) ; $events diff --git a/app/init.php b/app/init.php index 1db02ead44..df8e9b0328 100644 --- a/app/init.php +++ b/app/init.php @@ -55,6 +55,7 @@ const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; // Deletion Types const DELETE_TYPE_DOCUMENT = 'document'; +const DELETE_TYPE_EXECUTION_LOGS = 'execution_logs'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 09a62928b6..ced04c47c3 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -26,36 +26,29 @@ function getConsoleDB() { return $consoleDB; } -function notifyDeleteExecutionLogs(array $projectIds) +function notifyDeleteExecutionLogs() { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ - 'type' => DELETE_TYPE_DOCUMENT, + 'type' => DELETE_TYPE_EXECUTION_LOGS, 'document' => new Document([ - '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS, - 'projectIds' => $projectIds + '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS ]) ]); } -function notifyDeleteAbuseLogs(array $projectIds, int $interval) +function notifyDeleteAbuseLogs(int $interval) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ 'type' => DELETE_TYPE_ABUSE, - 'document' => new Document([ - 'projectIds' => $projectIds, - 'timestamp' => time() - $interval, - ]) + 'timestamp' => time() - $interval ]); } -function notifyDeleteAuditLogs(array $projectIds, int $interval) +function notifyDeleteAuditLogs(int $interval) { Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ 'type' => DELETE_TYPE_AUDIT, - 'document' => new Document([ - 'projectIds' => $projectIds, - 'timestamp' => time() - $interval - ]) + 'timestamp' => time() - $interval ]); } @@ -72,23 +65,11 @@ $cli Console::loop(function() use ($consoleDB, $interval){ - Authorization::disable(); - $projects = $consoleDB->getCollection([ - 'filters' => [ - '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, - ], - ]); - Authorization::reset(); - - $projectIds = array_map (function ($project) { - return $project->getId(); - }, $projects); - Console::info("[ MAINTENANCE TASK ] Notifying deletes workers every {$interval} seconds"); - // notifyDeleteExecutionLogs($projectIds); - notifyDeleteAbuseLogs($projectIds, $interval); - // notifyDeleteAuditLogs($projectIds, $interval); + notifyDeleteExecutionLogs(); + notifyDeleteAbuseLogs($interval); + notifyDeleteAuditLogs($interval); }, $intervalMicroseconds); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f7fd9439cb..3052439b4e 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -47,9 +47,6 @@ class DeletesV1 case Database::SYSTEM_COLLECTION_FUNCTIONS: $this->deleteFunction($document, $projectId); break; - case Database::SYSTEM_COLLECTION_EXECUTIONS: - $this->deleteExecutionLogs($document); - break; case Database::SYSTEM_COLLECTION_USERS: $this->deleteUser($document, $projectId); break; @@ -61,6 +58,11 @@ class DeletesV1 break; } break; + + case DELETE_TYPE_EXECUTION_LOGS: + $this->deleteExecutionLogs($document); + break; + case DELETE_TYPE_AUDIT: $this->deleteAuditLogs($document); break; @@ -109,7 +111,7 @@ class DeletesV1 foreach ($tokens as $token) { if (!$this->getProjectDB($projectId)->deleteDocument($token->getId())) { - throw new Exception('Failed to remove token from DB', 500); + throw new Exception('Failed to remove token from DB'); } } @@ -122,10 +124,10 @@ class DeletesV1 protected function deleteExecutionLogs(Document $document) { - $projectIds = $document->getAttribute('projectIds', []); + $projectIds = $this->getProjectIds(); foreach ($projectIds as $projectId) { if (!($projectDB = $this->getProjectDB($projectId))) { - throw new Exception('Failed to get projectDB for project '.$projectId, 500); + throw new Exception('Failed to get projectDB for project '.$projectId); } // Delete Executions @@ -139,7 +141,7 @@ class DeletesV1 protected function deleteAbuseLogs($document) { global $register; - $projectIds = $document->getAttribute('projectIds', []); + $projectIds = $this->getProjectIds(); $timestamp = $document->getAttribute('timestamp', 0); if($timestamp == 0) { @@ -151,13 +153,13 @@ class DeletesV1 }); foreach ($projectIds as $projectId) { - Console::success("Deleting abuse logs for Project: ", $projectId); + Console::success("Deleting abuse logs for Project: ".$projectId); $timeLimit->setNamespace('app_'.$projectId); $abuse = new Abuse($timeLimit); $status = $abuse->cleanup($timestamp); if (!$status) { - throw new Exception('Failed to delete Abuse logs for project '.$projectId, 500); + throw new Exception('Failed to delete Abuse logs for project '.$projectId); } } @@ -166,7 +168,7 @@ class DeletesV1 protected function deleteAuditLogs($document) { global $register; - $projectIds = $document->getAttribute('projectIds', []); + $projectIds = $this->getProjectIds(); $timestamp = $document->getAttribute('timestamp', 0); if($timestamp == 0) { @@ -179,7 +181,7 @@ class DeletesV1 $audit = new Audit($adapter); $status = $audit->cleanup($timestamp); if (!$status) { - throw new Exception('Failed to delete Audit logs for project'.$projectId, 500); + throw new Exception('Failed to delete Audit logs for project'.$projectId); } } } @@ -231,6 +233,23 @@ class DeletesV1 Authorization::reset(); } + protected function getProjectIds(): array + { + Authorization::disable(); + $projects = $this->getConsoleDB()->getCollection([ + 'filters' => [ + '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, + ], + ]); + Authorization::reset(); + + $projectIds = array_map (function ($project) { + return $project->getId(); + }, $projects); + + return $projectIds; + } + protected function deleteByGroup(array $filters, Database $database, callable $callback = null) { $count = 0; diff --git a/composer.json b/composer.json index 8628e48e21..d691919a50 100644 --- a/composer.json +++ b/composer.json @@ -35,8 +35,8 @@ "appwrite/php-clamav": "1.0.*", "utopia-php/framework": "0.9.8", - "utopia-php/abuse": "0.3.*", - "utopia-php/audit": "0.5.*", + "utopia-php/abuse": "0.3.1", + "utopia-php/audit": "0.5.1", "utopia-php/cache": "0.2.*", "utopia-php/cli": "0.8.0", "utopia-php/config": "0.2.*", diff --git a/composer.lock b/composer.lock index e070af3683..d260828c0a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f366e884df2132577f44576e459e3a4b", + "content-hash": "3e96a4fbfbff083f2ca166fcfa6fbbbd", "packages": [ { "name": "appwrite/php-clamav", @@ -1112,16 +1112,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.3.0", + "version": "0.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "5047cb311de8d7609e2678fdc405651f43f7e550" + "reference": "23c2eb533bca8f3ef5548ae265398fa7d4d39a1c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/5047cb311de8d7609e2678fdc405651f43f7e550", - "reference": "5047cb311de8d7609e2678fdc405651f43f7e550", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/23c2eb533bca8f3ef5548ae265398fa7d4d39a1c", + "reference": "23c2eb533bca8f3ef5548ae265398fa7d4d39a1c", "shasum": "" }, "require": { @@ -1158,22 +1158,22 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.3.0" + "source": "https://github.com/utopia-php/abuse/tree/0.3.1" }, - "time": "2020-12-17T18:39:44+00:00" + "time": "2020-12-21T17:28:03+00:00" }, { "name": "utopia-php/audit", - "version": "0.5.0", + "version": "0.5.1", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "5000fdbe755bbfb1e8bc26c61b90e28c40e5744a" + "reference": "154a850170a58667a15e4b65fbabb6cd0b709dd9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/5000fdbe755bbfb1e8bc26c61b90e28c40e5744a", - "reference": "5000fdbe755bbfb1e8bc26c61b90e28c40e5744a", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/154a850170a58667a15e4b65fbabb6cd0b709dd9", + "reference": "154a850170a58667a15e4b65fbabb6cd0b709dd9", "shasum": "" }, "require": { @@ -1210,9 +1210,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.5.0" + "source": "https://github.com/utopia-php/audit/tree/0.5.1" }, - "time": "2020-12-17T18:41:34+00:00" + "time": "2020-12-21T17:28:53+00:00" }, { "name": "utopia-php/cache", @@ -2551,16 +2551,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.10.3", + "version": "v4.10.4", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "dbe56d23de8fcb157bbc0cfb3ad7c7de0cfb0984" + "reference": "c6d052fc58cb876152f89f532b95a8d7907e7f0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/dbe56d23de8fcb157bbc0cfb3ad7c7de0cfb0984", - "reference": "dbe56d23de8fcb157bbc0cfb3ad7c7de0cfb0984", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/c6d052fc58cb876152f89f532b95a8d7907e7f0e", + "reference": "c6d052fc58cb876152f89f532b95a8d7907e7f0e", "shasum": "" }, "require": { @@ -2601,9 +2601,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.10.3" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.10.4" }, - "time": "2020-12-03T17:45:45+00:00" + "time": "2020-12-20T10:01:03+00:00" }, { "name": "openlss/lib-array2xml", @@ -2933,16 +2933,16 @@ }, { "name": "phpspec/prophecy", - "version": "1.12.1", + "version": "1.12.2", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "8ce87516be71aae9b956f81906aaf0338e0d8a2d" + "reference": "245710e971a030f42e08f4912863805570f23d39" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/8ce87516be71aae9b956f81906aaf0338e0d8a2d", - "reference": "8ce87516be71aae9b956f81906aaf0338e0d8a2d", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/245710e971a030f42e08f4912863805570f23d39", + "reference": "245710e971a030f42e08f4912863805570f23d39", "shasum": "" }, "require": { @@ -2954,7 +2954,7 @@ }, "require-dev": { "phpspec/phpspec": "^6.0", - "phpunit/phpunit": "^8.0 || ^9.0 <9.3" + "phpunit/phpunit": "^8.0 || ^9.0" }, "type": "library", "extra": { @@ -2994,9 +2994,9 @@ ], "support": { "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/1.12.1" + "source": "https://github.com/phpspec/prophecy/tree/1.12.2" }, - "time": "2020-09-29T09:10:42+00:00" + "time": "2020-12-19T10:15:11+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/docker-compose.yml b/docker-compose.yml index 83bbd5e76e..b43f1d003c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -191,7 +191,8 @@ services: volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw - - ./vendor:/usr/src/code/vendor + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src environment: - _APP_ENV - _APP_REDIS_HOST From fab01abe33f41fdc8ac116b4382d42e5c294c47e Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Dec 2020 18:21:46 +0530 Subject: [PATCH 18/30] patch: added classname and queue names as constants in event --- app/init.php | 8 ++++---- app/tasks/maintenance.php | 3 --- src/Appwrite/Event/Event.php | 13 +++++++++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/app/init.php b/app/init.php index df8e9b0328..db56d6fe93 100644 --- a/app/init.php +++ b/app/init.php @@ -303,19 +303,19 @@ App::setResource('events', function($register) { }, ['register']); App::setResource('audits', function($register) { - return new Event('v1-audits', 'AuditsV1'); + return new Event(Event::AUDITS_QUEUE_NAME, Event::AUDITS_CLASS_NAME); }, ['register']); App::setResource('usage', function($register) { - return new Event('v1-usage', 'UsageV1'); + return new Event(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME); }, ['register']); App::setResource('mails', function($register) { - return new Event('v1-mails', 'MailsV1'); + return new Event(Event::MAILS_QUEUE_NAME, Event::MAILS_CLASS_NAME); }, ['register']); App::setResource('deletes', function($register) { - return new Event('v1-deletes', 'DeletesV1'); + return new Event(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME); }, ['register']); // Test Mock diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index ced04c47c3..a37de737e7 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -13,9 +13,6 @@ use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; -define("DELETE_QUEUE_NAME", "v1-deletes"); -define("DELETE_CLASS_NAME", "DeletesV1"); - // TODO: Think of a better way to access consoleDB function getConsoleDB() { global $register; diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index bc712d7244..2ce929f922 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -6,6 +6,19 @@ use Resque; class Event { + + const DELETE_QUEUE_NAME = 'v1-deletes'; + const DELETE_CLASS_NAME = 'DeletesV1'; + + const AUDITS_QUEUE_NAME = 'v1-audits'; + const AUDITS_CLASS_NAME = 'AuditsV1'; + + const USAGE_QUEUE_NAME = 'v1-usage'; + const USAGE_CLASS_NAME = 'UsageV1'; + + const MAILS_QUEUE_NAME = 'v1-mails'; + const MAILS_CLASS_NAME = 'MailsV1'; + /** * @var string */ From 0a363cd78a22f4af3772f58d678866e3e8dbe13e Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Dec 2020 18:23:12 +0530 Subject: [PATCH 19/30] patch: added classname and queue names as constants in event --- app/controllers/api/health.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 8b146167da..e49084357f 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -164,7 +164,7 @@ App::get('/v1/health/queue/usage') ->action(function ($response) { /** @var Appwrite\Utopia\Response $response */ - $response->json(['size' => Resque::size('v1-usage')]); + $response->json(['size' => Resque::size(Event::USAGE_QUEUE_NAME)]); }, ['response']); App::get('/v1/health/queue/certificates') From 3898656f59807cdcfa5ad4a017996296801d80fc Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Dec 2020 18:27:22 +0530 Subject: [PATCH 20/30] patch: updated env and docker files --- .env | 2 +- Dockerfile | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 2f87526391..12b1311ff7 100644 --- a/.env +++ b/.env @@ -27,4 +27,4 @@ _APP_SMTP_PASSWORD= _APP_STORAGE_LIMIT=10000000 _APP_FUNCTIONS_TIMEOUT=900 _APP_FUNCTIONS_CONTAINERS=10 -_APP_MAINTENANCE_INTERVAL=60 \ No newline at end of file +_APP_MAINTENANCE_INTERVAL=86400 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 70112f973d..2ddb366e3a 100755 --- a/Dockerfile +++ b/Dockerfile @@ -94,7 +94,8 @@ ENV _APP_SERVER=swoole \ _APP_FUNCTIONS_CONTAINERS=10 \ _APP_SETUP=self-hosted \ _APP_VERSION=$VERSION \ - _APP_MAINTENANCE_INTERVAL=60 + # 1 Day = 86400 s + _APP_MAINTENANCE_INTERVAL=86400 #ENV _APP_SMTP_SECURE '' #ENV _APP_SMTP_USERNAME '' #ENV _APP_SMTP_PASSWORD '' From e7f600175791b6c4d0fb22139f3fa6d9aa3d0081 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Dec 2020 18:30:01 +0530 Subject: [PATCH 21/30] patch: updated composer and docker-compose --- composer.json | 4 ++-- docker-compose.yml | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/composer.json b/composer.json index d691919a50..8628e48e21 100644 --- a/composer.json +++ b/composer.json @@ -35,8 +35,8 @@ "appwrite/php-clamav": "1.0.*", "utopia-php/framework": "0.9.8", - "utopia-php/abuse": "0.3.1", - "utopia-php/audit": "0.5.1", + "utopia-php/abuse": "0.3.*", + "utopia-php/audit": "0.5.*", "utopia-php/cache": "0.2.*", "utopia-php/cli": "0.8.0", "utopia-php/config": "0.2.*", diff --git a/docker-compose.yml b/docker-compose.yml index b43f1d003c..a15034c3d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -191,8 +191,6 @@ services: volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src environment: - _APP_ENV - _APP_REDIS_HOST From c714ada6ff50caf660efac39fca40d38bcf39ab3 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Dec 2020 19:34:09 +0530 Subject: [PATCH 22/30] patch: added delete for project Ids by group --- .env | 2 +- app/tasks/maintenance.php | 14 ++---- app/workers/deletes.php | 97 ++++++++++++++++++++++----------------- 3 files changed, 62 insertions(+), 51 deletions(-) diff --git a/.env b/.env index 12b1311ff7..2f87526391 100644 --- a/.env +++ b/.env @@ -27,4 +27,4 @@ _APP_SMTP_PASSWORD= _APP_STORAGE_LIMIT=10000000 _APP_FUNCTIONS_TIMEOUT=900 _APP_FUNCTIONS_CONTAINERS=10 -_APP_MAINTENANCE_INTERVAL=86400 \ No newline at end of file +_APP_MAINTENANCE_INTERVAL=60 \ No newline at end of file diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index a37de737e7..d4b2352348 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -9,6 +9,7 @@ use Appwrite\Database\Document; use Appwrite\Database\Adapter\MySQL as MySQLAdapter; use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Database\Validator\Authorization; +use Appwrite\Event\Event; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -25,17 +26,14 @@ function getConsoleDB() { function notifyDeleteExecutionLogs() { - Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ - 'type' => DELETE_TYPE_EXECUTION_LOGS, - 'document' => new Document([ - '$collection' => Database::SYSTEM_COLLECTION_EXECUTIONS - ]) + Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ + 'type' => DELETE_TYPE_EXECUTION_LOGS ]); } function notifyDeleteAbuseLogs(int $interval) { - Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ + Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ 'type' => DELETE_TYPE_ABUSE, 'timestamp' => time() - $interval ]); @@ -43,7 +41,7 @@ function notifyDeleteAbuseLogs(int $interval) function notifyDeleteAuditLogs(int $interval) { - Resque::enqueue(DELETE_QUEUE_NAME, DELETE_CLASS_NAME, [ + Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ 'type' => DELETE_TYPE_AUDIT, 'timestamp' => time() - $interval ]); @@ -61,9 +59,7 @@ $cli $consoleDB = getConsoleDB(); Console::loop(function() use ($consoleDB, $interval){ - Console::info("[ MAINTENANCE TASK ] Notifying deletes workers every {$interval} seconds"); - notifyDeleteExecutionLogs(); notifyDeleteAbuseLogs($interval); notifyDeleteAuditLogs($interval); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 3052439b4e..19d3e83b7d 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -32,14 +32,13 @@ class DeletesV1 public function perform() { - $document = $this->args['document']; - $document = new Document($document); - $projectId = $this->args['projectId']; - + $projectId = $this->args['projectId']; $type = $this->args['type']; switch (strval($type)) { case DELETE_TYPE_DOCUMENT: + $document = $this->args['document']; + $document = new Document($document); switch (strval($document->getCollection())) { case Database::SYSTEM_COLLECTION_PROJECTS: $this->deleteProject($document); @@ -60,15 +59,15 @@ class DeletesV1 break; case DELETE_TYPE_EXECUTION_LOGS: - $this->deleteExecutionLogs($document); + $this->deleteExecutionLogs(); break; case DELETE_TYPE_AUDIT: - $this->deleteAuditLogs($document); + $this->deleteAuditLogs($this->args['timestamp']); break; case DELETE_TYPE_ABUSE: - $this->deleteAbuseLogs($document); + $this->deleteAbuseLogs($this->args['timestamp']); break; default: @@ -122,38 +121,33 @@ class DeletesV1 ], $this->getProjectDB($projectId)); } - protected function deleteExecutionLogs(Document $document) + protected function deleteExecutionLogs() { - $projectIds = $this->getProjectIds(); - foreach ($projectIds as $projectId) { + $this->deleteForProjectIds(function($projectId) { if (!($projectDB = $this->getProjectDB($projectId))) { throw new Exception('Failed to get projectDB for project '.$projectId); } // Delete Executions $this->deleteByGroup([ - '$collection='.$document->getCollection(), + '$collection='.Database::SYSTEM_COLLECTION_EXECUTIONS, '$projectId='.$projectId ], $projectDB); - } + }); } - protected function deleteAbuseLogs($document) + protected function deleteAbuseLogs($timestamp) { global $register; - $projectIds = $this->getProjectIds(); - $timestamp = $document->getAttribute('timestamp', 0); - if($timestamp == 0) { - throw new Exception('Failed to delete abuse logs. No timestamp provided'); + throw new Exception('Failed to delete audit logs. No timestamp provided'); } $timeLimit = new TimeLimit("", 0, 1, function () use ($register) { return $register->get('db'); }); - - foreach ($projectIds as $projectId) { - Console::success("Deleting abuse logs for Project: ".$projectId); + + $this->deleteForProjectIds(function($projectId) use ($timeLimit, $timestamp){ $timeLimit->setNamespace('app_'.$projectId); $abuse = new Abuse($timeLimit); @@ -161,21 +155,16 @@ class DeletesV1 if (!$status) { throw new Exception('Failed to delete Abuse logs for project '.$projectId); } - } - + }); } - protected function deleteAuditLogs($document) + protected function deleteAuditLogs($timestamp) { global $register; - $projectIds = $this->getProjectIds(); - $timestamp = $document->getAttribute('timestamp', 0); - if($timestamp == 0) { throw new Exception('Failed to delete audit logs. No timestamp provided'); } - - foreach ($projectIds as $projectId) { + $this->deleteForProjectIds(function($projectId) use ($register, $timestamp){ $adapter = new AuditAdapter($register->get('db')); $adapter->setNamespace('app_'.$projectId); $audit = new Audit($adapter); @@ -183,7 +172,7 @@ class DeletesV1 if (!$status) { throw new Exception('Failed to delete Audit logs for project'.$projectId); } - } + }); } protected function deleteFunction(Document $document, $projectId) @@ -233,21 +222,47 @@ class DeletesV1 Authorization::reset(); } - protected function getProjectIds(): array + protected function deleteForProjectIds(callable $callback) { - Authorization::disable(); - $projects = $this->getConsoleDB()->getCollection([ - 'filters' => [ - '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, - ], - ]); - Authorization::reset(); + $count = 0; + $chunk = 0; + $limit = 50; + $projects = []; + $sum = $limit; - $projectIds = array_map (function ($project) { - return $project->getId(); - }, $projects); + $executionStart = \microtime(true); + + while($sum === $limit) { + $chunk++; - return $projectIds; + Authorization::disable(); + $projects = $this->getConsoleDB()->getCollection([ + 'limit' => $limit, + 'offset' => 0, + 'orderField' => '$id', + 'orderType' => 'ASC', + 'orderCast' => 'string', + 'filters' => [ + '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, + ], + ]); + Authorization::reset(); + + $projectIds = array_map (function ($project) { + return $project->getId(); + }, $projects); + + $sum = count($projects); + + Console::info('Executing delete function for chunk #'.$chunk.'. Found '.$sum.' projects'); + foreach ($projectIds as $projectId) { + $callback($projectId); + $count++; + } + } + + $executionEnd = \microtime(true); + Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds"); } protected function deleteByGroup(array $filters, Database $database, callable $callback = null) From d58e1fdbcf7cae17e037affeff28949ce3e7971f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Dec 2020 19:39:34 +0530 Subject: [PATCH 23/30] patch: restore environment var --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 2f87526391..12b1311ff7 100644 --- a/.env +++ b/.env @@ -27,4 +27,4 @@ _APP_SMTP_PASSWORD= _APP_STORAGE_LIMIT=10000000 _APP_FUNCTIONS_TIMEOUT=900 _APP_FUNCTIONS_CONTAINERS=10 -_APP_MAINTENANCE_INTERVAL=60 \ No newline at end of file +_APP_MAINTENANCE_INTERVAL=86400 \ No newline at end of file From 92907fca0793c8f5b11a976cb22e9985b885a365 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 22 Dec 2020 19:49:29 +0530 Subject: [PATCH 24/30] patch: update docker compose files for prod --- app/config/variables.php | 6 ++++++ app/views/install/compose.phtml | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/app/config/variables.php b/app/config/variables.php index 3da5a21e25..ae49a968a2 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -151,4 +151,10 @@ return [ 'required' => false, 'question' => '', ], + [ + 'name' => '_APP_MAINTENANCE_INTERVAL', + 'default' => '86400', + 'required' => false, + 'question' => '', + ], ]; \ No newline at end of file diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index a97cd7882f..bf5512b41d 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -251,6 +251,27 @@ services: - _APP_SMTP_SECURE - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD + + appwrite-maintenance: + entrypoint: maintenance + container_name: appwrite-maintenance + build: + context: . + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + environment: + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_MAINTENANCE_INTERVAL + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS appwrite-schedule: image: appwrite/appwrite: From 1fe931410d3599c0d31d83bbae6cd6b0cca74bdb Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 24 Dec 2020 19:46:31 +0530 Subject: [PATCH 25/30] feat: address review comments --- app/tasks/maintenance.php | 2 -- app/workers/deletes.php | 4 ++-- docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index d4b2352348..5c9b6f8967 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -5,10 +5,8 @@ global $cli; require_once __DIR__.'/../init.php'; use Appwrite\Database\Database; -use Appwrite\Database\Document; use Appwrite\Database\Adapter\MySQL as MySQLAdapter; use Appwrite\Database\Adapter\Redis as RedisAdapter; -use Appwrite\Database\Validator\Authorization; use Appwrite\Event\Event; use Utopia\App; use Utopia\CLI\Console; diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 19d3e83b7d..b1091a6d8e 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -238,7 +238,7 @@ class DeletesV1 Authorization::disable(); $projects = $this->getConsoleDB()->getCollection([ 'limit' => $limit, - 'offset' => 0, + 'offset' => $count, 'orderField' => '$id', 'orderType' => 'ASC', 'orderCast' => 'string', @@ -282,7 +282,7 @@ class DeletesV1 $results = $database->getCollection([ 'limit' => $limit, - 'offset' => 0, + 'offset' => $count, 'orderField' => '$id', 'orderType' => 'ASC', 'orderCast' => 'string', diff --git a/docker-compose.yml b/docker-compose.yml index a15034c3d7..bbda6daf41 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,7 +55,7 @@ services: - ./psalm.xml:/usr/src/code/psalm.xml - ./tests:/usr/src/code/tests - ./app:/usr/src/code/app - - ./vendor:/usr/src/code/vendor + # - ./vendor:/usr/src/code/vendor - ./docs:/usr/src/code/docs - ./public:/usr/src/code/public - ./src:/usr/src/code/src From ec21e74813491d9d73b269ff3a74c076fb3294cf Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 24 Dec 2020 19:49:09 +0530 Subject: [PATCH 26/30] feat: address review comments --- app/workers/deletes.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b1091a6d8e..991bca58f2 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -130,8 +130,7 @@ class DeletesV1 // Delete Executions $this->deleteByGroup([ - '$collection='.Database::SYSTEM_COLLECTION_EXECUTIONS, - '$projectId='.$projectId + '$collection='.Database::SYSTEM_COLLECTION_EXECUTIONS ], $projectDB); }); } From e088a9e648506d76ec57341aa50b017efc62eb5d Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 27 Dec 2020 23:18:14 +0530 Subject: [PATCH 27/30] patch: review comments --- app/init.php | 2 +- app/workers/deletes.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index db56d6fe93..3307d8a9a6 100644 --- a/app/init.php +++ b/app/init.php @@ -55,7 +55,7 @@ const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; // Deletion Types const DELETE_TYPE_DOCUMENT = 'document'; -const DELETE_TYPE_EXECUTION_LOGS = 'execution_logs'; +const DELETE_TYPE_EXECUTION = 'execution'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 991bca58f2..2e6d216523 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -58,7 +58,7 @@ class DeletesV1 } break; - case DELETE_TYPE_EXECUTION_LOGS: + case DELETE_TYPE_EXECUTION: $this->deleteExecutionLogs(); break; From 355a516365b0f4ec677a69f4804d996763e8b7d7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 27 Dec 2020 23:18:41 +0530 Subject: [PATCH 28/30] patch: review comments --- app/tasks/maintenance.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 5c9b6f8967..9486b0def5 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -25,7 +25,7 @@ function getConsoleDB() { function notifyDeleteExecutionLogs() { Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ - 'type' => DELETE_TYPE_EXECUTION_LOGS + 'type' => DELETE_TYPE_EXECUTION ]); } From dce89663b08d87a96a8ab63c5ce56211508c0cee Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 27 Dec 2020 23:27:35 +0530 Subject: [PATCH 29/30] patch: review comments --- app/controllers/api/health.php | 11 ++++++----- app/init.php | 2 +- app/tasks/maintenance.php | 2 +- app/workers/deletes.php | 2 +- src/Appwrite/Event/Event.php | 13 +++++++++++++ 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index e49084357f..0131969eae 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -5,6 +5,7 @@ use Utopia\Exception; use Appwrite\Storage\Device\Local; use Appwrite\Storage\Storage; use Appwrite\ClamAV\Network; +use Appwrite\Event\Event; App::get('/v1/health') ->desc('Get HTTP') @@ -122,7 +123,7 @@ App::get('/v1/health/queue/webhooks') ->action(function ($response) { /** @var Appwrite\Utopia\Response $response */ - $response->json(['size' => Resque::size('v1-webhooks')]); + $response->json(['size' => Resque::size(Event::WEBHOOK_QUEUE_NAME)]); }, ['response']); App::get('/v1/health/queue/tasks') @@ -136,7 +137,7 @@ App::get('/v1/health/queue/tasks') ->action(function ($response) { /** @var Appwrite\Utopia\Response $response */ - $response->json(['size' => Resque::size('v1-tasks')]); + $response->json(['size' => Resque::size(Event::TASK_QUEUE_NAME)]); }, ['response']); App::get('/v1/health/queue/logs') @@ -150,7 +151,7 @@ App::get('/v1/health/queue/logs') ->action(function ($response) { /** @var Appwrite\Utopia\Response $response */ - $response->json(['size' => Resque::size('v1-audit')]); + $response->json(['size' => Resque::size(Event::AUDITS_QUEUE_NAME)]); }, ['response']); App::get('/v1/health/queue/usage') @@ -178,7 +179,7 @@ App::get('/v1/health/queue/certificates') ->action(function ($response) { /** @var Appwrite\Utopia\Response $response */ - $response->json(['size' => Resque::size('v1-certificates')]); + $response->json(['size' => Resque::size(Event::CERTIFICATES_QUEUE_NAME)]); }, ['response']); App::get('/v1/health/queue/functions') @@ -192,7 +193,7 @@ App::get('/v1/health/queue/functions') ->action(function ($response) { /** @var Appwrite\Utopia\Response $response */ - $response->json(['size' => Resque::size('v1-functions')]); + $response->json(['size' => Resque::size(Event::FUNCTIONS_QUEUE_NAME)]); }, ['response']); App::get('/v1/health/storage/local') diff --git a/app/init.php b/app/init.php index 3307d8a9a6..9ac7876e05 100644 --- a/app/init.php +++ b/app/init.php @@ -55,7 +55,7 @@ const APP_SOCIAL_DEV = 'https://dev.to/appwrite'; const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite'; // Deletion Types const DELETE_TYPE_DOCUMENT = 'document'; -const DELETE_TYPE_EXECUTION = 'execution'; +const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 9486b0def5..0d711ecb6b 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -25,7 +25,7 @@ function getConsoleDB() { function notifyDeleteExecutionLogs() { Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ - 'type' => DELETE_TYPE_EXECUTION + 'type' => DELETE_TYPE_EXECUTIONS ]); } diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 2e6d216523..3a7d7b52d7 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -58,7 +58,7 @@ class DeletesV1 } break; - case DELETE_TYPE_EXECUTION: + case DELETE_TYPE_EXECUTIONS: $this->deleteExecutionLogs(); break; diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 2ce929f922..b2c69337a2 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -18,6 +18,19 @@ class Event const MAILS_QUEUE_NAME = 'v1-mails'; const MAILS_CLASS_NAME = 'MailsV1'; + + const FUNCTIONS_QUEUE_NAME = 'v1-functions'; + const FUNCTIONS_CLASS_NAME = 'FunctionsV1'; + + const WEBHOOK_QUEUE_NAME = 'v1-webhooks'; + const WEBHOOK_CLASS_NAME = 'WebhooksV1'; + + const TASK_QUEUE_NAME = 'v1-tasks'; + const TASK_CLASS_NAME = 'TasksV1'; + + const CERTIFICATES_QUEUE_NAME = 'v1-certificates'; + const CERTIFICATES_CLASS_NAME = 'CertificatesV1'; + /** * @var string From e28c4feb978b8288e2dd911fd918dae8c2cf98a1 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 30 Dec 2020 00:50:08 +0530 Subject: [PATCH 30/30] feat: review comments --- app/tasks/maintenance.php | 2 +- app/workers/deletes.php | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 0d711ecb6b..f8786bcac8 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -50,7 +50,7 @@ $cli ->desc('Schedules maintenance tasks and publishes them to resque') ->action(function () { // # of days in seconds (1 day = 86400s) - $interval = App::getEnv('_APP_MAINTENANCE_INTERVAL', '') + 0; + $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); //Convert Seconds to microseconds $intervalMicroseconds = $interval * 1000000; diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 3a7d7b52d7..36d68f0f9f 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -238,7 +238,6 @@ class DeletesV1 $projects = $this->getConsoleDB()->getCollection([ 'limit' => $limit, 'offset' => $count, - 'orderField' => '$id', 'orderType' => 'ASC', 'orderCast' => 'string', 'filters' => [