diff --git a/.env b/.env index 6897e4e2b3..6c2af3fbe6 100644 --- a/.env +++ b/.env @@ -24,7 +24,7 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_CONNECTIONS_MAX=251 +_APP_CONNECTIONS_MAX=3100 _APP_POOL_CLIENTS=14 _APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite _APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite diff --git a/CHANGES.md b/CHANGES.md index ea67861809..6a51a738e3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ - Increase Traefik TCP + file limits [#4673](https://github.com/appwrite/appwrite/pull/4673) - Fix invited account verified status [#4776](https://github.com/appwrite/appwrite/pull/4776) - Get default region from environment on project create [#4780](https://github.com/appwrite/appwrite/pull/4780) +- Store build output file size [#4844](https://github.com/appwrite/appwrite/pull/4844) +- Fix max mimetype size [#4814](https://github.com/appwrite/appwrite/pull/4814) # Version 1.1.2 ## Changes diff --git a/Dockerfile b/Dockerfile index 0c9c6ef2c1..15c7b1552a 100755 --- a/Dockerfile +++ b/Dockerfile @@ -299,6 +299,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ + chmod +x /usr/local/bin/patch-create-missing-schedules && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/install && \ diff --git a/app/config/collections.php b/app/config/collections.php index 0af568dd82..a56effa9df 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2562,17 +2562,6 @@ $collections = [ 'array' => false, 'filters' => ['datetime'], ], - [ - '$id' => ID::custom('endTime'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], [ '$id' => ID::custom('duration'), 'type' => Database::VAR_INTEGER, @@ -2618,7 +2607,7 @@ $collections = [ 'filters' => [], ], [ - '$id' => ID::custom('outputPath'), + '$id' => ID::custom('path'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 2048, @@ -2628,6 +2617,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('size'), + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('stderr'), 'type' => Database::VAR_STRING, @@ -3372,7 +3372,7 @@ $collections = [ '$id' => ID::custom('mimeType'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 127, // https://tools.ietf.org/html/rfc4288#section-4.2 + 'size' => 255, // https://tools.ietf.org/html/rfc4288#section-4.2 'signed' => true, 'required' => false, 'default' => null, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 2c3ca19d3c..751480a6e5 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -983,7 +983,7 @@ App::post('/v1/functions/:functionId/executions') variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $build->getAttribute('outputPath', ''), + source: $build->getAttribute('path', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), ); diff --git a/app/workers/builds.php b/app/workers/builds.php index ff8bafae58..54e8b581f8 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -56,11 +56,6 @@ class BuildsV1 extends Worker } } - /** - * @throws \Utopia\Database\Exception\Authorization - * @throws Throwable - * @throws \Utopia\Database\Exception\Structure - */ protected function buildDeployment(Document $project, Document $function, Document $deployment) { global $register; @@ -103,13 +98,13 @@ class BuildsV1 extends Worker 'startTime' => $startTime, 'deploymentId' => $deployment->getId(), 'status' => 'processing', - 'outputPath' => '', + 'path' => '', + 'size' => 0, 'runtime' => $function->getAttribute('runtime'), 'source' => $deployment->getAttribute('path'), 'sourceType' => $device, 'stdout' => '', 'stderr' => '', - 'endTime' => null, 'duration' => 0 ])); $deployment->setAttribute('buildId', $buildId); @@ -174,8 +169,8 @@ class BuildsV1 extends Worker try { $response = $this->executor->createRuntime( - deploymentId: $deployment->getId(), projectId: $project->getId(), + deploymentId: $deployment->getId(), source: $source, image: $runtime['image'], remove: true, @@ -190,14 +185,12 @@ class BuildsV1 extends Worker ] ); - $endTime = new \DateTime(); - $endTime->setTimestamp($response['endTimeUnix']); - /** Update the build document */ - $build->setAttribute('endTime', DateTime::format($endTime)); + $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime']))); $build->setAttribute('duration', \intval($response['duration'])); $build->setAttribute('status', $response['status']); - $build->setAttribute('outputPath', $response['outputPath']); + $build->setAttribute('path', $response['path']); + $build->setAttribute('size', $response['size']); $build->setAttribute('stderr', $response['stderr']); $build->setAttribute('stdout', $response['stdout']); @@ -228,7 +221,7 @@ class BuildsV1 extends Worker } catch (\Throwable $th) { $endTime = DateTime::now(); $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime)); - $build->setAttribute('endTime', $endTime); + $build->setAttribute('duration', $interval->format('%s') + 0); $build->setAttribute('status', 'failed'); $build->setAttribute('stderr', $th->getMessage()); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b8efa38ae6..6bf29cc3ff 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -134,8 +134,7 @@ class DeletesV1 extends Worker */ protected function deleteSchedules(string $datetime): void { - - $this->deleteByGroup( + $this->listByGroup( 'schedules', [ Query::equal('region', [App::getEnv('_APP_REGION', 'default')]), @@ -145,7 +144,6 @@ class DeletesV1 extends Worker ], $this->getConsoleDB(), function (Document $document) { - Console::info('Querying schedule for function ' . $document->getAttribute('resourceId')); $project = $this->getConsoleDB()->getDocument('projects', $document->getAttribute('projectId')); if ($project->isEmpty()) { @@ -358,8 +356,15 @@ class DeletesV1 extends Worker protected function deleteExpiredSessions(): void { - $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $consoleDB = $this->getConsoleDB(); + + $this->deleteForProjectIds(function (Document $project) use ($consoleDB) { $dbForProject = $this->getProjectDB($project); + + $project = $consoleDB->getDocument('projects', $project->getId()); + $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration); + // Delete Sessions $this->deleteByGroup('sessions', [ Query::lessThan('$createdAt', $expired) @@ -481,10 +486,10 @@ class DeletesV1 extends Worker $this->deleteByGroup('builds', [ Query::equal('deploymentId', [$deploymentId]) ], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) { - if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); + if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('path', '')); } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); + Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); } }); } @@ -530,10 +535,10 @@ class DeletesV1 extends Worker $this->deleteByGroup('builds', [ Query::equal('deploymentId', [$deploymentId]) ], $dbForProject, function (Document $document) use ($storageBuilds) { - if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); + if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('path', '')); } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); + Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); } }); @@ -569,6 +574,7 @@ class DeletesV1 extends Worker */ protected function deleteForProjectIds(callable $callback): void { + // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document $count = 0; $chunk = 0; $limit = 50; @@ -632,6 +638,43 @@ class DeletesV1 extends Worker Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); } + /** + * @param string $collection collectionID + * @param Query[] $queries + * @param Database $database + * @param callable $callback + */ + protected function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void + { + $count = 0; + $chunk = 0; + $limit = 50; + $results = []; + $sum = $limit; + + $executionStart = \microtime(true); + + while ($sum === $limit) { + $chunk++; + + $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + + $sum = count($results); + + foreach ($results as $document) { + if (is_callable($callback)) { + $callback($document); + } + + $count++; + } + } + + $executionEnd = \microtime(true); + + Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds"); + } + /** * @param Document $document certificates document */ diff --git a/app/workers/functions.php b/app/workers/functions.php index cb9817d73e..63e1b5582e 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -141,7 +141,7 @@ Server::setResource('execute', function () { variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $build->getAttribute('outputPath', ''), + source: $build->getAttribute('path', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), ); diff --git a/bin/patch-create-missing-schedules b/bin/patch-create-missing-schedules new file mode 100644 index 0000000000..e38d3e9a6f --- /dev/null +++ b/bin/patch-create-missing-schedules @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php patch-create-missing-schedules $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index b9c1235d52..c6b02784a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,15 +33,6 @@ services: - 8080:80 - 443:443 - 9500:8080 - ulimits: - nofile: - soft: 655350 - hard: 655350 - sysctls: - - net.core.somaxconn=1024 - - net.ipv4.tcp_rmem=1024 4096 16384 - - net.ipv4.tcp_wmem=1024 4096 16384 - - net.ipv4.ip_local_port_range=1025 65535 volumes: - /var/run/docker.sock:/var/run/docker.sock - appwrite-config:/storage/config:ro @@ -62,6 +53,7 @@ services: DEBUG: false TESTING: true VERSION: dev + VITE_CONSOLE_MODE: self-hosted ports: - 9501:80 networks: @@ -85,7 +77,7 @@ services: - appwrite-cache:/storage/cache:rw - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw - - openruntimes-functions:/storage/functions:rw + - appwrite-functions:/storage/functions:rw - ./phpunit.xml:/usr/src/code/phpunit.xml - ./tests:/usr/src/code/tests - ./app:/usr/src/code/app @@ -99,7 +91,7 @@ services: - mariadb - redis # - clamav - entrypoint: + entrypoint: - php - -e - app/http.php @@ -107,10 +99,12 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_SERVER_MULTIPROCESS=enabled - _APP_LOCALE - _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_EMAILS - _APP_CONSOLE_WHITELIST_IPS + - _APP_CONSOLE_INVITES - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS @@ -126,6 +120,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -173,7 +168,7 @@ services: container_name: appwrite-realtime image: appwrite-dev restart: unless-stopped - ports: + ports: - 9505:80 labels: - "traefik.enable=true" @@ -201,6 +196,7 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_SERVER_MULTIPROCESS=enabled - _APP_OPTIONS_ABUSE - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST @@ -209,6 +205,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -272,6 +269,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - _APP_REDIS_HOST @@ -292,17 +291,19 @@ services: depends_on: - redis - mariadb - volumes: + volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw - - openruntimes-functions:/storage/functions:rw - - openruntimes-builds:/storage/builds:rw + - appwrite-functions:/storage/functions:rw + - appwrite-builds:/storage/builds:rw - appwrite-certificates:/storage/certificates:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -330,7 +331,7 @@ services: image: appwrite-dev networks: - appwrite - volumes: + volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database @@ -340,6 +341,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -364,7 +367,7 @@ services: image: appwrite-dev networks: - appwrite - volumes: + volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src depends_on: @@ -373,6 +376,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -389,9 +394,9 @@ services: - _APP_CONNECTIONS_DB_PROJECT - _APP_CONNECTIONS_CACHE - _APP_CONNECTIONS_QUEUE + - _APP_CONNECTIONS_STORAGE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - - _APP_CONNECTIONS_STORAGE appwrite-worker-certificates: entrypoint: worker-certificates @@ -403,7 +408,7 @@ services: depends_on: - redis - mariadb - volumes: + volumes: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw - ./app:/usr/src/code/app @@ -411,6 +416,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET @@ -448,6 +455,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_DB_HOST - _APP_DB_PORT @@ -486,6 +495,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_OPENSSL_KEY_V1 - _APP_SYSTEM_EMAIL_NAME - _APP_SYSTEM_EMAIL_ADDRESS @@ -517,6 +528,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -617,6 +630,8 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -637,18 +652,19 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.1.4 + image: openruntimes/executor:0.2.0 networks: - appwrite - - openruntimes-runtimes + - runtimes volumes: - /var/run/docker.sock:/var/run/docker.sock - - openruntimes-builds:/storage/builds:rw - - openruntimes-functions:/storage/functions:rw + - appwrite-builds:/storage/builds:rw + - appwrite-functions:/storage/functions:rw - /tmp:/tmp:rw environment: - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD + - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD @@ -658,7 +674,6 @@ services: - OPR_EXECUTOR_LOGGING_PROVIDER=$_APP_LOGGING_PROVIDER - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG - mariadb: image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p container_name: appwrite-mariadb @@ -679,7 +694,6 @@ services: # smtp: # image: appwrite/smtp:1.2.0 # container_name: appwrite-smtp - # restart: unless-stopped # networks: # - appwrite # environment: @@ -745,15 +759,26 @@ services: image: adminer container_name: appwrite-adminer <<: *x-logging - restart: always ports: - 9506:8080 networks: - appwrite + + # appwrite-volume-sync: + # entrypoint: volume-sync + # <<: *x-logging + # container_name: appwrite-volume-sync + # image: appwrite-dev + # command: + # - --source=/data/src/ --destination=/data/dest/ --interval=10 + # networks: + # - appwrite + # # volumes: # Mount the rsync source and destination directories + # # - /nfs/config:/data/src + # # - /storage/config:/data/dest # redis-commander: # image: rediscommander/redis-commander:latest - # restart: unless-stopped # networks: # - appwrite # environment: @@ -763,7 +788,6 @@ services: # resque: # image: appwrite/resque-web:1.1.0 - # restart: unless-stopped # networks: # - appwrite # ports: @@ -779,7 +803,7 @@ services: # - './debug:/tmp' # ports: # - '3001:80' - + # Dev Tools End ------------------------------------------------------------------------------------------ networks: @@ -787,8 +811,8 @@ networks: name: gateway appwrite: name: appwrite - openruntimes-runtimes: - name: openruntimes-runtimes + runtimes: + name: runtimes volumes: appwrite-mariadb: @@ -799,6 +823,5 @@ volumes: appwrite-functions: appwrite-builds: appwrite-config: - openruntimes-functions: - openruntimes-builds: + diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index d98746520a..a0c237fd5f 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -48,6 +48,7 @@ abstract class Migration '1.1.0' => 'V16', '1.1.1' => 'V16', '1.1.2' => 'V16', + '1.2.0' => 'V17', ]; /** diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php new file mode 100644 index 0000000000..e6877e7225 --- /dev/null +++ b/src/Appwrite/Migration/Version/V17.php @@ -0,0 +1,119 @@ + null, + fn () => [] + ); + } + + Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); + + Console::info('Migrating Collections'); + $this->migrateCollections(); + + // Console::info('Migrating Documents'); + // $this->forEachDocument([$this, 'fixDocument']); + } + + /** + * Migrate all Collections. + * + * @return void + */ + protected function migrateCollections(): void + { + foreach ($this->collections as $collection) { + $id = $collection['$id']; + + Console::log("Migrating Collection \"{$id}\""); + + $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); + + switch ($id) { + case 'files': + try { + /** + * Update 'mimeType' attribute size (127->255) + */ + $this->projectDB->updateAttribute($id, 'mimeType', Database::VAR_STRING, 255, true, false); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'mimeType' from {$id}: {$th->getMessage()}"); + } + break; + case 'builds': + try { + /** + * Create 'size' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'size'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'size' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Delete 'endTime' attribute (use startTime+duration if needed) + */ + $this->projectDB->deleteAttribute($id, 'endTime'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'endTime' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Rename 'outputPath' to 'path' + */ + $this->projectDB->renameAttribute($id, 'outputPath', 'path'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'path' from {$id}: {$th->getMessage()}"); + } + + try { + /** + * Create 'size' + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'size'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'size' from {$id}: {$th->getMessage()}"); + } + break; + default: + break; + } + + usleep(50000); + } + } + + /** + * Fix run on each document + * + * @param \Utopia\Database\Document $document + * @return \Utopia\Database\Document + */ + protected function fixDocument(Document $document) + { + return $document; + } +} diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index f3bf1b9a3f..4fa93a1f6e 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -8,6 +8,7 @@ use Appwrite\Platform\Tasks\Install; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\Schedule; +use Appwrite\Platform\Tasks\PatchCreateMissingSchedules; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; @@ -28,6 +29,7 @@ class Tasks extends Service ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) + ->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules()) ->addAction(Schedule::getName(), new Schedule()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) diff --git a/src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php b/src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php new file mode 100644 index 0000000000..32b9886347 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php @@ -0,0 +1,97 @@ +desc('Ensure every function has a schedule') + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB)); + } + + /** + * Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule. + */ + public function action(Database $dbForConsole, callable $getProjectDB): void + { + Authorization::disable(); + Authorization::setDefaultStatus(false); + + Console::title('PatchCreateMissingSchedules V1'); + Console::success(APP_NAME . ' PatchCreateMissingSchedules v1 has started'); + + $limit = 100; + $projectCursor = null; + while (true) { + $projectsQueries = [Query::limit($limit)]; + if ($projectCursor !== null) { + $projectsQueries[] = Query::cursorAfter($projectCursor); + } + $projects = $dbForConsole->find('projects', $projectsQueries); + + if (count($projects) === 0) { + break; + } + + foreach ($projects as $project) { + Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")"); + $dbForProject = $getProjectDB($project); + $functionCursor = null; + + while (true) { + $functionsQueries = [Query::limit($limit)]; + if ($functionCursor !== null) { + $functionsQueries[] = Query::cursorAfter($functionCursor); + } + $functions = $dbForProject->find('functions', $functionsQueries); + if (count($functions) === 0) { + break; + } + + foreach ($functions as $function) { + $scheduleId = $function->getAttribute('scheduleId'); + $schedule = $dbForConsole->getDocument('schedules', $scheduleId); + + if ($schedule->isEmpty()) { + $functionId = $function->getId(); + $schedule = $dbForConsole->createDocument('schedules', new Document([ + '$id' => ID::custom($scheduleId), + 'region' => $project->getAttribute('region', 'default'), + 'resourceType' => 'function', + 'resourceId' => $functionId, + 'resourceUpdatedAt' => DateTime::now(), + 'projectId' => $project->getId(), + 'schedule' => $function->getAttribute('schedule'), + 'active' => !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')), + ])); + + Console::success('Recreated schedule for function ' . $functionId); + } + } + + $functionCursor = $functions[array_key_last($functions)]; + } + } + + $projectCursor = $projects[array_key_last($projects)]; + } + } +} diff --git a/src/Appwrite/Utopia/Response/Model/Build.php b/src/Appwrite/Utopia/Response/Model/Build.php index b76f0ee083..4d7a6cd27c 100644 --- a/src/Appwrite/Utopia/Response/Model/Build.php +++ b/src/Appwrite/Utopia/Response/Model/Build.php @@ -51,18 +51,18 @@ class Build extends Model 'default' => '', 'example' => self::TYPE_DATETIME_EXAMPLE, ]) - ->addRule('endTime', [ - 'type' => self::TYPE_DATETIME, - 'description' => 'The time the build was finished in ISO 8601 format.', - 'default' => '', - 'example' => self::TYPE_DATETIME_EXAMPLE, - ]) ->addRule('duration', [ 'type' => self::TYPE_INTEGER, 'description' => 'The build duration in seconds.', 'default' => 0, 'example' => 0, ]) + ->addRule('size', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The code size in bytes.', + 'default' => 0, + 'example' => 128, + ]) ; }