From 4dfaa5e69ace071f2917a8417103ac8ddcd496b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 5 Dec 2022 14:54:13 +0000 Subject: [PATCH 01/24] Fix mimetype size --- app/config/collections.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/collections.php b/app/config/collections.php index bdc14d9105..e872953568 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -3256,7 +3256,7 @@ $collections = [ '$id' => ID::custom('mimeType'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 127, // https://tools.ietf.org/html/rfc4288#section-4.2 + 'size' => 255, // https://tools.ietf.org/html/rfc4288#section-4.2 'signed' => true, 'required' => false, 'default' => null, From ea58cb4aff4dab9936042ab1d79ac88de8c4cbff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 5 Dec 2022 14:57:24 +0000 Subject: [PATCH 02/24] Update changelog --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 0ba56af243..e9b8ed761f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ - Fix invited account verified status [#4776](https://github.com/appwrite/appwrite/pull/4776) - Get default region from environment on project create [#4780](https://github.com/appwrite/appwrite/pull/4780) +- Fix max mimetype size [#4814](https://github.com/appwrite/appwrite/pull/4814) # Version 1.1.2 ## Changes From 0a177713f06c54a4107f0b90ac3a1ce5d6de4485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 7 Dec 2022 11:01:58 +0000 Subject: [PATCH 03/24] Implement migration for mimeType --- src/Appwrite/Migration/Migration.php | 1 + src/Appwrite/Migration/Version/V17.php | 79 ++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 src/Appwrite/Migration/Version/V17.php diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index bef6e5e603..b764641279 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -48,6 +48,7 @@ abstract class Migration '1.1.0' => 'V16', '1.1.1' => 'V16', '1.1.2' => 'V16', + '1.2.0' => 'V17', ]; /** diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php new file mode 100644 index 0000000000..b04aa2a7c3 --- /dev/null +++ b/src/Appwrite/Migration/Version/V17.php @@ -0,0 +1,79 @@ + null, + fn () => [] + ); + } + + Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); + + Console::info('Migrating Collections'); + $this->migrateCollections(); + + // Console::info('Migrating Documents'); + // $this->forEachDocument([$this, 'fixDocument']); + } + + /** + * Migrate all Collections. + * + * @return void + */ + protected function migrateCollections(): void + { + foreach ($this->collections as $collection) { + $id = $collection['$id']; + + Console::log("Migrating Collection \"{$id}\""); + + $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); + + switch ($id) { + case 'files': + try { + /** + * Update 'mimeType' attribute size (127->255) + */ + $this->projectDB->updateAttribute($id, 'mimeType', Database::VAR_STRING, 255, true, false); + } catch (\Throwable $th) { + Console::warning("'mimeType' from {$id}: {$th->getMessage()}"); + } + + break; + + default: + break; + } + + usleep(50000); + } + } + + /** + * Fix run on each document + * + * @param \Utopia\Database\Document $document + * @return \Utopia\Database\Document + */ + protected function fixDocument(Document $document) + { + return $document; + } +} From fe27670e1f255587b32c3cceb47d5feb50e91b0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 12 Dec 2022 13:39:43 +0100 Subject: [PATCH 04/24] New attribute + migration --- app/config/collections.php | 11 ++++ src/Appwrite/Migration/Migration.php | 1 + src/Appwrite/Migration/Version/V17.php | 76 ++++++++++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 src/Appwrite/Migration/Version/V17.php diff --git a/app/config/collections.php b/app/config/collections.php index d23f452b08..a4ccc6eeb4 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2627,6 +2627,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('outputSize'), + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('stderr'), 'type' => Database::VAR_STRING, diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index d98746520a..a0c237fd5f 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -48,6 +48,7 @@ abstract class Migration '1.1.0' => 'V16', '1.1.1' => 'V16', '1.1.2' => 'V16', + '1.2.0' => 'V17', ]; /** diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php new file mode 100644 index 0000000000..c7e73fa1b5 --- /dev/null +++ b/src/Appwrite/Migration/Version/V17.php @@ -0,0 +1,76 @@ + null, + fn () => [] + ); + } + + Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); + + Console::info('Migrating Collections'); + $this->migrateCollections(); + + // Console::info('Migrating Documents'); + // $this->forEachDocument([$this, 'fixDocument']); + } + + /** + * Migrate all Collections. + * + * @return void + */ + protected function migrateCollections(): void + { + foreach ($this->collections as $collection) { + $id = $collection['$id']; + + Console::log("Migrating Collection \"{$id}\""); + + $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); + + switch ($id) { + case 'builds': + try { + /** + * Create 'region' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'outputSize'); + } catch (\Throwable $th) { + Console::warning("'region' from {$id}: {$th->getMessage()}"); + } + default: + break; + } + + usleep(50000); + } + } + + /** + * Fix run on each document + * + * @param \Utopia\Database\Document $document + * @return \Utopia\Database\Document + */ + protected function fixDocument(Document $document) + { + return $document; + } +} \ No newline at end of file From b513c2fbad7f388ea280db298477e8a0a0942dc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 12 Dec 2022 13:40:04 +0100 Subject: [PATCH 05/24] Store build output size to DB --- app/workers/builds.php | 1 + docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/workers/builds.php b/app/workers/builds.php index d26f07ab75..eb9d2e6762 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -194,6 +194,7 @@ class BuildsV1 extends Worker $build->setAttribute('duration', \intval($response['duration'])); $build->setAttribute('status', $response['status']); $build->setAttribute('outputPath', $response['outputPath']); + $build->setAttribute('outputSize', $response['outputSize']); $build->setAttribute('stderr', $response['stderr']); $build->setAttribute('stdout', $response['stdout']); diff --git a/docker-compose.yml b/docker-compose.yml index 6c040df4cc..8ccb90c656 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -666,7 +666,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.1.6 + image: myexc networks: - appwrite - runtimes From d79eea0f4bbe05d4520af2e1faaa811def8e0dba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 12 Dec 2022 13:42:35 +0100 Subject: [PATCH 06/24] Send build size to stats --- src/Appwrite/Usage/Stats.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index e6e0056664..38fdc123cf 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -184,6 +184,7 @@ class Stats $functionBuild = $this->params['builds.{scope}.compute'] ?? 0; $functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms + $functionBuildSize = ($this->params['buildSize'] ?? 0); // bytes $functionBuildStatus = $this->params['buildStatus'] ?? ''; $functionCompute = $functionExecutionTime + $functionBuildTime; $functionTags = $tags . ',functionId=' . $functionId; @@ -207,6 +208,7 @@ class Stats if ($functionBuild >= 1) { $this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus); $this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime); + $this->statsd->count('builds.{scope}.storage.size' . $functionTags, $functionBuildSize); } if ($functionBuild + $functionExecution >= 1) { $this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute); From 2f53260197bb4a69fd13ac9db944d5c48e6c68b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 12 Dec 2022 13:45:07 +0100 Subject: [PATCH 07/24] Update CHANGES.md --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index ea67861809..800aaa7014 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ - Increase Traefik TCP + file limits [#4673](https://github.com/appwrite/appwrite/pull/4673) - Fix invited account verified status [#4776](https://github.com/appwrite/appwrite/pull/4776) - Get default region from environment on project create [#4780](https://github.com/appwrite/appwrite/pull/4780) +- Store build output file size [#4844](https://github.com/appwrite/appwrite/pull/4844) # Version 1.1.2 ## Changes From de0de5f2254c8cfd961113c9e933fceab03bad84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 12 Dec 2022 13:45:21 +0100 Subject: [PATCH 08/24] Linter fix --- src/Appwrite/Migration/Version/V17.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index c7e73fa1b5..2897826117 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -73,4 +73,4 @@ class V17 extends Migration { return $document; } -} \ No newline at end of file +} From d702f3581fb9bdb75bae43269c6f8bc860e6fffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 12 Dec 2022 13:52:22 +0100 Subject: [PATCH 09/24] Fix bug --- src/Appwrite/Migration/Version/V17.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index 2897826117..d00df0a61e 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -55,6 +55,7 @@ class V17 extends Migration } catch (\Throwable $th) { Console::warning("'region' from {$id}: {$th->getMessage()}"); } + break; default: break; } From 746390e30acc3780276d5608aa1c07c4fcbd2156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 13 Dec 2022 08:10:51 +0100 Subject: [PATCH 10/24] Fix missing param --- app/workers/builds.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/workers/builds.php b/app/workers/builds.php index eb9d2e6762..c173b99fe5 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -261,6 +261,7 @@ class BuildsV1 extends Worker ->setParam('builds.{scope}.compute', 1) ->setParam('buildStatus', $build->getAttribute('status', '')) ->setParam('buildTime', $build->getAttribute('duration')) + ->setParam('buildSize', $build->getAttribute('outputSize')) ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) ->submit(); From 6279784009cb602eb5f86c3900d7caa7cf5ebe39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 14 Dec 2022 09:44:40 +0100 Subject: [PATCH 11/24] Fix expired session, schedule deletes --- .env | 4 ++-- app/workers/deletes.php | 51 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/.env b/.env index f042b1686d..fee6c6653d 100644 --- a/.env +++ b/.env @@ -24,7 +24,7 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_CONNECTIONS_MAX=251 +_APP_CONNECTIONS_MAX=3100 _APP_POOL_CLIENTS=14 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite @@ -64,7 +64,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_MAINTENANCE_RETENTION_SCHEDULES=86400 +_APP_MAINTENANCE_RETENTION_SCHEDULES=600 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_USAGE_STATS=enabled _APP_USAGE_AGGREGATION_INTERVAL=30 diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b8efa38ae6..458b341a45 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -134,8 +134,7 @@ class DeletesV1 extends Worker */ protected function deleteSchedules(string $datetime): void { - - $this->deleteByGroup( + $this->listByGroup( 'schedules', [ Query::equal('region', [App::getEnv('_APP_REGION', 'default')]), @@ -145,7 +144,6 @@ class DeletesV1 extends Worker ], $this->getConsoleDB(), function (Document $document) { - Console::info('Querying schedule for function ' . $document->getAttribute('resourceId')); $project = $this->getConsoleDB()->getDocument('projects', $document->getAttribute('projectId')); if ($project->isEmpty()) { @@ -358,8 +356,15 @@ class DeletesV1 extends Worker protected function deleteExpiredSessions(): void { - $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $consoleDB = $this->getConsoleDB(); + + $this->deleteForProjectIds(function (Document $project) use ($consoleDB) { $dbForProject = $this->getProjectDB($project); + + $project = $consoleDB->getDocument('projects', $project->getId()); + $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration); + // Delete Sessions $this->deleteByGroup('sessions', [ Query::lessThan('$createdAt', $expired) @@ -569,6 +574,7 @@ class DeletesV1 extends Worker */ protected function deleteForProjectIds(callable $callback): void { + // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document $count = 0; $chunk = 0; $limit = 50; @@ -632,6 +638,43 @@ class DeletesV1 extends Worker Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } + /** + * @param string $collection collectionID + * @param Query[] $queries + * @param Database $database + * @param callable $callback + */ + protected function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void + { + $count = 0; + $chunk = 0; + $limit = 50; + $results = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + + $sum = count($results); + + foreach ($results as $document) { + if (is_callable($callback)) { + $callback($document); + } + + $count++; + } + } + + $executionEnd = \microtime(true); + + Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); + } + /** * @param Document $document certificates document */ From 41a6a6dbf79273a7143b410b0dc69d78a2cd60a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 14 Dec 2022 09:51:28 +0100 Subject: [PATCH 12/24] Add patch CLI task --- bin/create-missing-schedules | 3 + src/Appwrite/Platform/Services/Tasks.php | 2 + .../Platform/Tasks/CreateMissingSchedules.php | 97 +++++++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 bin/create-missing-schedules create mode 100644 src/Appwrite/Platform/Tasks/CreateMissingSchedules.php diff --git a/bin/create-missing-schedules b/bin/create-missing-schedules new file mode 100644 index 0000000000..41750b163c --- /dev/null +++ b/bin/create-missing-schedules @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php create-missing-schedules $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 2968a66b95..e6919f2cc7 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -8,6 +8,7 @@ use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\Schedule; +use Appwrite\Platform\Tasks\CreateMissingSchedules; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; @@ -29,6 +30,7 @@ class Tasks extends Service ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) + ->addAction(CreateMissingSchedules::getName(), new CreateMissingSchedules()) ->addAction(Schedule::getName(), new Schedule()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/CreateMissingSchedules.php b/src/Appwrite/Platform/Tasks/CreateMissingSchedules.php new file mode 100644 index 0000000000..75e1d5b637 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/CreateMissingSchedules.php @@ -0,0 +1,97 @@ +desc('Ensure every function has a schedule') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB)); + } + + /** + * Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule. + */ + public function action(Database $dbForConsole, callable $getProjectDB): void + { + Authorization::disable(); + Authorization::setDefaultStatus(false); + + Console::title('CreateMissingSchedules V1'); + Console::success(APP_NAME . ' CreateMissingSchedules v1 has started'); + + $limit = 100; + $projectCursor = null; + while (true) { + $projectsQueries = [Query::limit($limit)]; + if ($projectCursor !== null) { + $projectsQueries[] = Query::cursorAfter($projectCursor); + } + $projects = $dbForConsole->find('projects', $projectsQueries); + + if (count($projects) === 0) { + break; + } + + foreach ($projects as $project) { + Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")"); + $dbForProject = $getProjectDB($project); + $functionCursor = null; + + while (true) { + $functionsQueries = [Query::limit($limit)]; + if ($functionCursor !== null) { + $functionsQueries[] = Query::cursorAfter($functionCursor); + } + $functions = $dbForProject->find('functions', $functionsQueries); + if (count($functions) === 0) { + break; + } + + foreach ($functions as $function) { + $scheduleId = $function->getAttribute('scheduleId'); + $schedule = $dbForConsole->getDocument('schedules', $scheduleId); + + if ($schedule->isEmpty()) { + $functionId = $function->getId(); + $schedule = $dbForConsole->createDocument('schedules', new Document([ + '$id' => ID::custom($scheduleId), + 'region' => $project->getAttribute('region', 'default'), + 'resourceType' => 'function', + 'resourceId' => $functionId, + 'resourceUpdatedAt' => DateTime::now(), + 'projectId' => $project->getId(), + 'schedule' => $function->getAttribute('schedule'), + 'active' => !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')), + ])); + + Console::success('Recreated schedule for function ' . $functionId); + } + } + + $functionCursor = $functions[array_key_last($functions)]; + } + } + + $projectCursor = $projects[array_key_last($projects)]; + } + } +} From 11b3abebd1f57f4f32f9f644b6b07d84d28111fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 14 Dec 2022 09:58:45 +0100 Subject: [PATCH 13/24] Remove tcp setup --- docker-compose.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6c040df4cc..7620039b63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,15 +33,6 @@ services: - 8080:80 - 443:443 - 9500:8080 - ulimits: - nofile: - soft: 655350 - hard: 655350 - sysctls: - - net.core.somaxconn=1024 - - net.ipv4.tcp_rmem=1024 4096 16384 - - net.ipv4.tcp_wmem=1024 4096 16384 - - net.ipv4.ip_local_port_range=1025 65535 volumes: - /var/run/docker.sock:/var/run/docker.sock - appwrite-config:/storage/config:ro From 3d07600a48b1d0d709a571703409221268711f0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 14 Dec 2022 10:44:46 +0100 Subject: [PATCH 14/24] PR review changes --- bin/create-missing-schedules | 3 --- bin/patch-create-missing-schedules | 3 +++ src/Appwrite/Platform/Services/Tasks.php | 4 ++-- ...ssingSchedules.php => PatchCreateMissingSchedules.php} | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) delete mode 100644 bin/create-missing-schedules create mode 100644 bin/patch-create-missing-schedules rename src/Appwrite/Platform/Tasks/{CreateMissingSchedules.php => PatchCreateMissingSchedules.php} (93%) diff --git a/bin/create-missing-schedules b/bin/create-missing-schedules deleted file mode 100644 index 41750b163c..0000000000 --- a/bin/create-missing-schedules +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php create-missing-schedules $@ \ No newline at end of file diff --git a/bin/patch-create-missing-schedules b/bin/patch-create-missing-schedules new file mode 100644 index 0000000000..e38d3e9a6f --- /dev/null +++ b/bin/patch-create-missing-schedules @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php patch-create-missing-schedules $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index e6919f2cc7..2e15cd015c 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -8,7 +8,7 @@ use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\Schedule; -use Appwrite\Platform\Tasks\CreateMissingSchedules; +use Appwrite\Platform\Tasks\PatchCreateMissingSchedules; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; @@ -30,7 +30,7 @@ class Tasks extends Service ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) - ->addAction(CreateMissingSchedules::getName(), new CreateMissingSchedules()) + ->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules()) ->addAction(Schedule::getName(), new Schedule()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/CreateMissingSchedules.php b/src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php similarity index 93% rename from src/Appwrite/Platform/Tasks/CreateMissingSchedules.php rename to src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php index 75e1d5b637..32b9886347 100644 --- a/src/Appwrite/Platform/Tasks/CreateMissingSchedules.php +++ b/src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php @@ -11,11 +11,11 @@ use Utopia\Database\Database; use Utopia\Database\ID; use Utopia\Database\Validator\Authorization; -class CreateMissingSchedules extends Action +class PatchCreateMissingSchedules extends Action { public static function getName(): string { - return 'create-missing-schedules'; + return 'patch-create-missing-schedules'; } public function __construct() @@ -35,8 +35,8 @@ class CreateMissingSchedules extends Action Authorization::disable(); Authorization::setDefaultStatus(false); - Console::title('CreateMissingSchedules V1'); - Console::success(APP_NAME . ' CreateMissingSchedules v1 has started'); + Console::title('PatchCreateMissingSchedules V1'); + Console::success(APP_NAME . ' PatchCreateMissingSchedules v1 has started'); $limit = 100; $projectCursor = null; From 3af214172cc8417289052494949baa671ee7cf47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 14 Dec 2022 11:10:16 +0100 Subject: [PATCH 15/24] Rollback unwanted changes --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index fee6c6653d..51e6cc3705 100644 --- a/.env +++ b/.env @@ -64,7 +64,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_MAINTENANCE_RETENTION_SCHEDULES=600 +_APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_USAGE_STATS=enabled _APP_USAGE_AGGREGATION_INTERVAL=30 From 072d896bb94249461958e9b9cd786aaf8edaa824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 14 Dec 2022 12:37:01 +0100 Subject: [PATCH 16/24] Add missing chmod for patch task --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 7aa447d279..54aa327769 100755 --- a/Dockerfile +++ b/Dockerfile @@ -303,6 +303,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ + chmod +x /usr/local/bin/patch-create-missing-schedules && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/usage && \ From 4ee8fd5629e01547e042cc449cde130860bbd30a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 16 Dec 2022 10:39:53 +0100 Subject: [PATCH 17/24] Add cache clean --- src/Appwrite/Migration/Version/V17.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index b04aa2a7c3..66a02662d1 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -52,6 +52,7 @@ class V17 extends Migration * Update 'mimeType' attribute size (127->255) */ $this->projectDB->updateAttribute($id, 'mimeType', Database::VAR_STRING, 255, true, false); + $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'mimeType' from {$id}: {$th->getMessage()}"); } From c5da386d6721c639c73f87918a98b67a121c3e4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 18 Dec 2022 08:20:50 +0100 Subject: [PATCH 18/24] PR review changes --- app/config/collections.php | 4 ++-- app/workers/builds.php | 6 +++--- docker-compose.yml | 2 +- src/Appwrite/Migration/Version/V17.php | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index a4ccc6eeb4..7d4e224ec6 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2617,7 +2617,7 @@ $collections = [ 'filters' => [], ], [ - '$id' => ID::custom('outputPath'), + '$id' => ID::custom('path'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 2048, @@ -2628,7 +2628,7 @@ $collections = [ 'filters' => [], ], [ - '$id' => ID::custom('outputSize'), + '$id' => ID::custom('size'), 'type' => Database::VAR_INTEGER, 'format' => '', 'size' => 0, diff --git a/app/workers/builds.php b/app/workers/builds.php index c173b99fe5..975af6532f 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -193,8 +193,8 @@ class BuildsV1 extends Worker $build->setAttribute('endTime', DateTime::format($endTime)); $build->setAttribute('duration', \intval($response['duration'])); $build->setAttribute('status', $response['status']); - $build->setAttribute('outputPath', $response['outputPath']); - $build->setAttribute('outputSize', $response['outputSize']); + $build->setAttribute('path', $response['path']); + $build->setAttribute('size', $response['size']); $build->setAttribute('stderr', $response['stderr']); $build->setAttribute('stdout', $response['stdout']); @@ -261,7 +261,7 @@ class BuildsV1 extends Worker ->setParam('builds.{scope}.compute', 1) ->setParam('buildStatus', $build->getAttribute('status', '')) ->setParam('buildTime', $build->getAttribute('duration')) - ->setParam('buildSize', $build->getAttribute('outputSize')) + ->setParam('buildSize', $build->getAttribute('size')) ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) ->submit(); diff --git a/docker-compose.yml b/docker-compose.yml index 8ccb90c656..a24e739c89 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -666,7 +666,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: myexc + image: meldiron/appwrite-executor:latest networks: - appwrite - runtimes diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index d00df0a61e..695d31bc8f 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -49,11 +49,11 @@ class V17 extends Migration case 'builds': try { /** - * Create 'region' attribute + * Create 'size' attribute */ - $this->createAttributeFromCollection($this->projectDB, $id, 'outputSize'); + $this->createAttributeFromCollection($this->projectDB, $id, 'size'); } catch (\Throwable $th) { - Console::warning("'region' from {$id}: {$th->getMessage()}"); + Console::warning("'size' from {$id}: {$th->getMessage()}"); } break; default: From 8f363eae7aa511aacdf3ccbced966b98fb0c18a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 18 Dec 2022 08:35:16 +0100 Subject: [PATCH 19/24] Fix bugs --- app/config/collections.php | 11 ----------- app/controllers/api/functions.php | 2 +- app/workers/builds.php | 12 +++++------- app/workers/deletes.php | 12 ++++++------ app/workers/functions.php | 2 +- src/Appwrite/Migration/Version/V17.php | 10 ++++++++++ 6 files changed, 23 insertions(+), 26 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 4d8d734109..a8c7a46b18 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2561,17 +2561,6 @@ $collections = [ 'array' => false, 'filters' => ['datetime'], ], - [ - '$id' => ID::custom('endTime'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], [ '$id' => ID::custom('duration'), 'type' => Database::VAR_INTEGER, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 6fe7c67e69..4c84795712 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1197,7 +1197,7 @@ App::post('/v1/functions/:functionId/executions') variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $build->getAttribute('outputPath', ''), + source: $build->getAttribute('path', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), ); diff --git a/app/workers/builds.php b/app/workers/builds.php index 975af6532f..94c984caf8 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -99,13 +99,13 @@ class BuildsV1 extends Worker 'startTime' => $startTime, 'deploymentId' => $deployment->getId(), 'status' => 'processing', - 'outputPath' => '', + 'path' => '', + 'size' => 0, 'runtime' => $function->getAttribute('runtime'), 'source' => $deployment->getAttribute('path'), 'sourceType' => $device, 'stdout' => '', 'stderr' => '', - 'endTime' => null, 'duration' => 0 ])); $deployment->setAttribute('buildId', $buildId); @@ -186,11 +186,8 @@ class BuildsV1 extends Worker ] ); - $endTime = new \DateTime(); - $endTime->setTimestamp($response['endTimeUnix']); - /** Update the build document */ - $build->setAttribute('endTime', DateTime::format($endTime)); + $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime']))); $build->setAttribute('duration', \intval($response['duration'])); $build->setAttribute('status', $response['status']); $build->setAttribute('path', $response['path']); @@ -225,12 +222,13 @@ class BuildsV1 extends Worker } catch (\Throwable $th) { $endTime = DateTime::now(); $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime)); - $build->setAttribute('endTime', $endTime); + $build->setAttribute('duration', $interval->format('%s') + 0); $build->setAttribute('status', 'failed'); $build->setAttribute('stderr', $th->getMessage()); Console::error($th->getMessage()); } finally { + \var_dump($build); $build = $dbForProject->updateDocument('builds', $buildId, $build); /** diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 458b341a45..6bf29cc3ff 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -486,10 +486,10 @@ class DeletesV1 extends Worker $this->deleteByGroup('builds', [ Query::equal('deploymentId', [$deploymentId]) ], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) { - if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); + if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('path', '')); } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); + Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); } }); } @@ -535,10 +535,10 @@ class DeletesV1 extends Worker $this->deleteByGroup('builds', [ Query::equal('deploymentId', [$deploymentId]) ], $dbForProject, function (Document $document) use ($storageBuilds) { - if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); + if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('path', '')); } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); + Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); } }); diff --git a/app/workers/functions.php b/app/workers/functions.php index 31e64a2bb4..2e8dccc7aa 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -133,7 +133,7 @@ Server::setResource('execute', function () { variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $build->getAttribute('outputPath', ''), + source: $build->getAttribute('path', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), ); diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index e719980635..d0f7372a9a 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -66,6 +66,16 @@ class V17 extends Migration } catch (\Throwable $th) { Console::warning("'size' from {$id}: {$th->getMessage()}"); } + + try { + /** + * Delete 'endTime' attribute (use startTime+duration if needed) + */ + $this->projectDB->deleteAttribute($id, 'startTime'); + $this->createAttributeFromCollection($this->projectDB, $id, 'startTime'); + } catch (\Throwable $th) { + Console::warning("'startTime' from {$id}: {$th->getMessage()}"); + } break; default: break; From 1a33ffa86756ec250ee046c9e89e7aa9f9451f0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 18 Dec 2022 08:37:53 +0100 Subject: [PATCH 20/24] Linter fix --- src/Appwrite/Migration/Version/V17.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index d0f7372a9a..cef4ae0716 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -66,7 +66,7 @@ class V17 extends Migration } catch (\Throwable $th) { Console::warning("'size' from {$id}: {$th->getMessage()}"); } - + try { /** * Delete 'endTime' attribute (use startTime+duration if needed) From fdd9a1caf364b17c700689f4d0110626921bccc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 18 Dec 2022 08:40:56 +0100 Subject: [PATCH 21/24] Add size to build response --- src/Appwrite/Utopia/Response/Model/Build.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Utopia/Response/Model/Build.php b/src/Appwrite/Utopia/Response/Model/Build.php index b76f0ee083..4d7a6cd27c 100644 --- a/src/Appwrite/Utopia/Response/Model/Build.php +++ b/src/Appwrite/Utopia/Response/Model/Build.php @@ -51,18 +51,18 @@ class Build extends Model 'default' => '', 'example' => self::TYPE_DATETIME_EXAMPLE, ]) - ->addRule('endTime', [ - 'type' => self::TYPE_DATETIME, - 'description' => 'The time the build was finished in ISO 8601 format.', - 'default' => '', - 'example' => self::TYPE_DATETIME_EXAMPLE, - ]) ->addRule('duration', [ 'type' => self::TYPE_INTEGER, 'description' => 'The build duration in seconds.', 'default' => 0, 'example' => 0, ]) + ->addRule('size', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The code size in bytes.', + 'default' => 0, + 'example' => 128, + ]) ; } From ff7393985533a0cc5d7934ddf7e9ab320485bcfb Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 18 Dec 2022 15:23:18 +0530 Subject: [PATCH 22/24] feat: update console and build args --- app/console | 2 +- docker-compose.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/console b/app/console index c1ac63889c..43891a526e 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit c1ac63889c0a97b280599355e526669bb207880a +Subproject commit 43891a526e061454617cbb13def3c4901d99a7f1 diff --git a/docker-compose.yml b/docker-compose.yml index 7620039b63..feac2c89d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,6 +53,7 @@ services: DEBUG: false TESTING: true VERSION: dev + VITE_CONSOLE_MODE: self-hosted ports: - 9501:80 networks: From b014eda788c63040092b7538c91bbea9b983dd67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 20 Dec 2022 12:48:11 +0100 Subject: [PATCH 23/24] PR review changes --- docker-compose.yml | 2 +- src/Appwrite/Migration/Version/V17.php | 27 +++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8f6d5257e3..448a2ba2aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -657,7 +657,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: meldiron/appwrite-executor:latest + image: openruntimes/executor:0.2.0 networks: - appwrite - runtimes diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index cef4ae0716..e6877e7225 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -63,6 +63,7 @@ class V17 extends Migration * Create 'size' attribute */ $this->createAttributeFromCollection($this->projectDB, $id, 'size'); + $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { Console::warning("'size' from {$id}: {$th->getMessage()}"); } @@ -71,10 +72,30 @@ class V17 extends Migration /** * Delete 'endTime' attribute (use startTime+duration if needed) */ - $this->projectDB->deleteAttribute($id, 'startTime'); - $this->createAttributeFromCollection($this->projectDB, $id, 'startTime'); + $this->projectDB->deleteAttribute($id, 'endTime'); + $this->projectDB->deleteCachedCollection($id); } catch (\Throwable $th) { - Console::warning("'startTime' from {$id}: {$th->getMessage()}"); + Console::warning("'endTime' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Rename 'outputPath' to 'path' + */ + $this->projectDB->renameAttribute($id, 'outputPath', 'path'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'path' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create 'size' + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'size'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'size' from {$id}: {$th->getMessage()}"); } break; default: From e3737af3fec5cde9c4c15d88d6e94956826d6f40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 20 Dec 2022 12:49:52 +0100 Subject: [PATCH 24/24] Update app/workers/builds.php Co-authored-by: Christy Jacob --- app/workers/builds.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/workers/builds.php b/app/workers/builds.php index 94c984caf8..081537fdfb 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -228,7 +228,6 @@ class BuildsV1 extends Worker $build->setAttribute('stderr', $th->getMessage()); Console::error($th->getMessage()); } finally { - \var_dump($build); $build = $dbForProject->updateDocument('builds', $buildId, $build); /**