diff --git a/app/config/errors.php b/app/config/errors.php index 9e742d3296..cb3ffb53a7 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -519,6 +519,11 @@ return [ 'description' => 'Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 'code' => 404, ], + Exception::FUNCTION_SYNCHRONOUS_TIMEOUT => [ + 'name' => Exception::FUNCTION_SYNCHRONOUS_TIMEOUT, + 'description' => 'Synchronous function execution timed out. Use asynchronous execution instead, or ensure the execution duration doesn\'t exceed 30 seconds.', + 'code' => 408, + ], /** Builds */ Exception::BUILD_NOT_FOUND => [ diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index ed5af1d38a..a88d3815e9 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -9,6 +9,7 @@ use Appwrite\Event\Func; use Appwrite\Event\Usage; use Appwrite\Event\Validator\FunctionEvent; use Appwrite\Extend\Exception; +use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Task\Validator\Cron; use Appwrite\Utopia\Database\Validator\CustomId; @@ -1750,6 +1751,10 @@ App::post('/v1/functions/:functionId/executions') ->setAttribute('responseStatusCode', 500) ->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode()); Console::error($th->getMessage()); + + if ($th instanceof AppwriteException) { + throw $th; + } } finally { $queueForUsage ->addMetric(METRIC_EXECUTIONS, 1) @@ -1757,11 +1762,11 @@ App::post('/v1/functions/:functionId/executions') ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function ; - } - if ($function->getAttribute('logging')) { - /** @var Document $execution */ - $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); + if ($function->getAttribute('logging')) { + /** @var Document $execution */ + $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); + } } $roles = Authorization::getRoles(); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index c51697bf95..5464014b48 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -152,10 +152,10 @@ App::post('/v1/projects') throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project."); } - // TODO: One in 20 projects use shared tables. Temporary until all projects are using shared tables. + // TODO: 1 in 5 projects use shared tables. Temporary until all projects are using shared tables. if ( ( - !\mt_rand(0, 19) + !\mt_rand(0, 4) && System::getEnv('_APP_DATABASE_SHARED_TABLES', 'enabled') === 'enabled' && System::getEnv('_APP_EDITION', 'self-hosted') !== 'self-hosted' ) || diff --git a/app/controllers/general.php b/app/controllers/general.php index 3734b81a3b..2c8a4f68aa 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -289,6 +289,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $execution->setAttribute('logs', $executionResponse['logs']); $execution->setAttribute('errors', $executionResponse['errors']); $execution->setAttribute('duration', $executionResponse['duration']); + } catch (\Throwable $th) { $durationEnd = \microtime(true); @@ -298,6 +299,10 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo ->setAttribute('responseStatusCode', 500) ->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode()); Console::error($th->getMessage()); + + if ($th instanceof AppwriteException) { + throw $th; + } } finally { $queueForUsage ->addMetric(METRIC_EXECUTIONS, 1) @@ -305,11 +310,11 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function ; - } - if ($function->getAttribute('logging')) { - /** @var Document $execution */ - $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); + if ($function->getAttribute('logging')) { + /** @var Document $execution */ + $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); + } } $execution->setAttribute('logs', ''); @@ -725,6 +730,7 @@ App::error() $classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName); $adapter = new $classname($providerConfig); $logger = new Logger($adapter); + $logger->setSample(0.04); $publish = true; } } diff --git a/app/init.php b/app/init.php index fbb3e2ee8d..88e8de0ea2 100644 --- a/app/init.php +++ b/app/init.php @@ -735,6 +735,16 @@ $register->set('logger', function () { throw new Exception(Exception::GENERAL_SERVER_ERROR, "Logging provider not supported. Logging is disabled"); } + // Old Sentry Format conversion. Fallback until the old syntax is completely deprecated. + if (str_contains($providerConfig, ';') && strtolower($providerName) == 'sentry') { + $configChunks = \explode(";", $providerConfig); + + $sentryKey = $configChunks[0]; + $projectId = $configChunks[1]; + + $providerConfig = 'https://' . $sentryKey . '@sentry.io/' . $projectId; + } + $classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName); $adapter = new $classname($providerConfig); return new Logger($adapter); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 53e7e71518..9c0cc31bbf 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -727,11 +727,11 @@ services: openruntimes-executor: container_name: openruntimes-executor - hostname: appwrite-executor + hostname: exc1 <<: *x-logging restart: unless-stopped stop_signal: SIGINT - image: openruntimes/executor:0.4.12 + image: openruntimes/executor:0.5.5 networks: - appwrite - runtimes diff --git a/composer.json b/composer.json index 6bc88afb0f..9505711004 100644 --- a/composer.json +++ b/composer.json @@ -57,7 +57,7 @@ "utopia-php/fetch": "0.2.*", "utopia-php/image": "0.6.*", "utopia-php/locale": "0.4.*", - "utopia-php/logger": "0.3.*", + "utopia-php/logger": "0.5.*", "utopia-php/messaging": "0.10.*", "utopia-php/migration": "0.4.*", "utopia-php/orchestration": "0.9.*", diff --git a/composer.lock b/composer.lock index 0744fa1aec..30d5ed74e0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c9a722b82f10d9bbded155f3f04a924b", + "content-hash": "e3975335737921ddfe4e35d891ea222a", "packages": [ { "name": "adhocore/jwt", @@ -1556,16 +1556,16 @@ }, { "name": "utopia-php/database", - "version": "dev-original-exception", + "version": "0.49.10", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "1a85827705a420b0dbc6c15845c33ba039cde900" + "reference": "216209121bc97a2010f67a39c561fafe1e936bec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/1a85827705a420b0dbc6c15845c33ba039cde900", - "reference": "1a85827705a420b0dbc6c15845c33ba039cde900", + "url": "https://api.github.com/repos/utopia-php/database/zipball/216209121bc97a2010f67a39c561fafe1e936bec", + "reference": "216209121bc97a2010f67a39c561fafe1e936bec", "shasum": "" }, "require": { @@ -1606,9 +1606,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/original-exception" + "source": "https://github.com/utopia-php/database/tree/0.49.10" }, - "time": "2024-05-15T07:29:54+00:00" + "time": "2024-05-20T02:14:20+00:00" }, { "name": "utopia-php/domains", @@ -1902,22 +1902,23 @@ }, { "name": "utopia-php/logger", - "version": "0.3.2", + "version": "0.5.2", "source": { "type": "git", "url": "https://github.com/utopia-php/logger.git", - "reference": "ba763c10688fe2ed715ad2bed3f13d18dfec6253" + "reference": "c6dfdb672e41364c309b0c30dc03bc6d45446dba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/logger/zipball/ba763c10688fe2ed715ad2bed3f13d18dfec6253", - "reference": "ba763c10688fe2ed715ad2bed3f13d18dfec6253", + "url": "https://api.github.com/repos/utopia-php/logger/zipball/c6dfdb672e41364c309b0c30dc03bc6d45446dba", + "reference": "c6dfdb672e41364c309b0c30dc03bc6d45446dba", "shasum": "" }, "require": { "php": ">=8.0" }, "require-dev": { + "laravel/pint": "1.2.*", "phpstan/phpstan": "1.9.x-dev", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.0.1" @@ -1949,9 +1950,9 @@ ], "support": { "issues": "https://github.com/utopia-php/logger/issues", - "source": "https://github.com/utopia-php/logger/tree/0.3.2" + "source": "https://github.com/utopia-php/logger/tree/0.5.2" }, - "time": "2023-11-22T14:45:43+00:00" + "time": "2024-05-17T09:32:59+00:00" }, { "name": "utopia-php/messaging", @@ -2005,22 +2006,21 @@ }, { "name": "utopia-php/migration", - "version": "0.4.1", + "version": "0.4.4", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "ae3cfe93f6d313105d226aeb68806660c806a925" + "reference": "a8a5d392bebf082faf289f4dfe09d9fd76844c33" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/ae3cfe93f6d313105d226aeb68806660c806a925", - "reference": "ae3cfe93f6d313105d226aeb68806660c806a925", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/a8a5d392bebf082faf289f4dfe09d9fd76844c33", + "reference": "a8a5d392bebf082faf289f4dfe09d9fd76844c33", "shasum": "" }, "require": { "appwrite/appwrite": "10.1.0", - "php": "8.*", - "utopia-php/cli": "0.*" + "php": "8.*" }, "require-dev": { "laravel/pint": "1.*", @@ -2047,9 +2047,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.4.1" + "source": "https://github.com/utopia-php/migration/tree/0.4.4" }, - "time": "2024-05-01T13:19:18+00:00" + "time": "2024-05-17T05:25:31+00:00" }, { "name": "utopia-php/mongo", @@ -2823,16 +2823,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.38.2", + "version": "0.38.5", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "51284668529e2b10ed933412a42b603c76cded23" + "reference": "830a46cc8e34ee096a76d4af6f00adf008a7cbf8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/51284668529e2b10ed933412a42b603c76cded23", - "reference": "51284668529e2b10ed933412a42b603c76cded23", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/830a46cc8e34ee096a76d4af6f00adf008a7cbf8", + "reference": "830a46cc8e34ee096a76d4af6f00adf008a7cbf8", "shasum": "" }, "require": { @@ -2868,9 +2868,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.38.2" + "source": "https://github.com/appwrite/sdk-generator/tree/0.38.5" }, - "time": "2024-04-25T07:49:29+00:00" + "time": "2024-05-17T00:59:59+00:00" }, { "name": "doctrine/deprecations", @@ -5503,18 +5503,9 @@ "time": "2023-11-21T18:54:41+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-original-exception", - "alias": "0.49.0", - "alias_normalized": "0.49.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index 82c38ba269..1a38893f3a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -791,10 +791,10 @@ services: openruntimes-executor: container_name: openruntimes-executor - hostname: appwrite-executor + hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.5.1 + image: openruntimes/executor:0.5.5 restart: unless-stopped networks: - appwrite @@ -857,7 +857,7 @@ services: - OPR_PROXY_LOGGING_PROVIDER=$_APP_LOGGING_PROVIDER - OPR_PROXY_LOGGING_CONFIG=$_APP_LOGGING_CONFIG - OPR_PROXY_ALGORITHM=random - - OPR_PROXY_EXECUTORS=appwrite-executor + - OPR_PROXY_EXECUTORS=exc1 - OPR_PROXY_HEALTHCHECK_INTERVAL=10000 - OPR_PROXY_MAX_TIMEOUT=600 - OPR_PROXY_HEALTHCHECK=enabled diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 48409f06b1..d48787bb98 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -153,6 +153,7 @@ class Exception extends \Exception public const FUNCTION_NOT_FOUND = 'function_not_found'; public const FUNCTION_RUNTIME_UNSUPPORTED = 'function_runtime_unsupported'; public const FUNCTION_ENTRYPOINT_MISSING = 'function_entrypoint_missing'; + public const FUNCTION_SYNCHRONOUS_TIMEOUT = 'function_synchronous_timeout'; /** Deployments */ public const DEPLOYMENT_NOT_FOUND = 'deployment_not_found'; diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index b08bfb027a..49b41da495 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -443,7 +443,7 @@ class Deletes extends Action * @param Document $document * @return void * @throws Authorization - * @throws \Utopia\Database\Exception + * @throws DatabaseException * @throws Conflict * @throws Restricted * @throws Structure @@ -471,11 +471,10 @@ class Deletes extends Action * @return void * @throws Exception * @throws Authorization - * @throws \Utopia\Database\Exception + * @throws DatabaseException */ private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void { - $projectId = $document->getId(); $projectInternalId = $document->getInternalId(); try { @@ -486,24 +485,18 @@ class Deletes extends Action } $dbForProject = $getProjectDB($document); - $projectCollectionIds = \array_keys(Config::getParam('collections', [])['projects']); + + $projectCollectionIds = [ + ...\array_keys(Config::getParam('collections', [])['projects']), + Audit::COLLECTION, + TimeLimit::COLLECTION, + ]; + $limit = \count($projectCollectionIds) + 25; while (true) { $collections = $dbForProject->listCollections($limit); - if ($dsn->getHost() === DATABASE_SHARED_TABLES) { - $collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections); - - if ($collectionsIds == $projectCollectionIds) { - break; - } - } else { - if (empty($collections)) { - break; - } - } - foreach ($collections as $collection) { if ($dsn->getHost() !== DATABASE_SHARED_TABLES || !\in_array($collection->getId(), $projectCollectionIds)) { $dbForProject->deleteCollection($collection->getId()); @@ -511,6 +504,16 @@ class Deletes extends Action $this->deleteByGroup($collection->getId(), [], database: $dbForProject); } } + + if ($dsn->getHost() === DATABASE_SHARED_TABLES) { + $collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections); + + if (empty(\array_diff($collectionsIds, $projectCollectionIds))) { + break; + } + } elseif (empty($collections)) { + break; + } } // Delete Platforms @@ -553,6 +556,8 @@ class Deletes extends Action // Delete metadata table if ($dsn->getHost() !== DATABASE_SHARED_TABLES) { $dbForProject->deleteCollection('_metadata'); + } else { + $this->deleteByGroup('_metadata', [], $dbForProject); } // Delete all storage directories @@ -679,9 +684,11 @@ class Deletes extends Action $dbForProject = $getProjectDB($project); $timeLimit = new TimeLimit("", 0, 1, $dbForProject); $abuse = new Abuse($timeLimit); - $status = $abuse->cleanup($abuseRetention); - if (!$status) { - throw new Exception('Failed to delete Abuse logs for project ' . $projectId); + + try { + $abuse->cleanup($abuseRetention); + } catch (DatabaseException $e) { + Console::error('Failed to delete abuse logs for project ' . $projectId . ': ' . $e->getMessage()); } } @@ -697,9 +704,11 @@ class Deletes extends Action $projectId = $project->getId(); $dbForProject = $getProjectDB($project); $audit = new Audit($dbForProject); - $status = $audit->cleanup($auditRetention); - if (!$status) { - throw new Exception('Failed to delete Audit logs for project' . $projectId); + + try { + $audit->cleanup($auditRetention); + } catch (DatabaseException $e) { + Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage()); } } @@ -954,7 +963,12 @@ class Deletes extends Action while ($sum === $limit) { $chunk++; - $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries)); + try { + $results = $database->find($collection, [Query::limit($limit), ...$queries]); + } catch (DatabaseException $e) { + Console::error('Failed to find documents for collection ' . $collection . ': ' . $e->getMessage()); + return; + } $sum = count($results); diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index c270018430..0399302e77 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -25,7 +25,7 @@ use Utopia\Messaging\Adapter\SMS as SMSAdapter; use Utopia\Messaging\Adapter\SMS\Mock; use Utopia\Messaging\Adapter\SMS\Msg91; use Utopia\Messaging\Adapter\SMS\Telesign; -use Utopia\Messaging\Adapter\SMS\Textmagic; +use Utopia\Messaging\Adapter\SMS\TextMagic; use Utopia\Messaging\Adapter\SMS\Twilio; use Utopia\Messaging\Adapter\SMS\Vonage; use Utopia\Messaging\Messages\Email; @@ -459,7 +459,7 @@ class Messaging extends Action return match ($provider->getAttribute('provider')) { 'mock' => new Mock('username', 'password'), 'twilio' => new Twilio($credentials['accountSid'], $credentials['authToken']), - 'textmagic' => new Textmagic($credentials['username'], $credentials['apiKey']), + 'textmagic' => new TextMagic($credentials['username'], $credentials['apiKey']), 'telesign' => new Telesign($credentials['customerId'], $credentials['apiKey']), 'msg91' => new Msg91($credentials['senderId'], $credentials['authKey'], $credentials['templateId']), 'vonage' => new Vonage($credentials['apiKey'], $credentials['apiSecret']), diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index 76c66de231..e9b0ae016e 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -2,6 +2,7 @@ namespace Executor; +use Appwrite\Extend\Exception as AppwriteException; use Exception; use Utopia\System\System; @@ -193,7 +194,6 @@ class Executor 'path' => $path, 'method' => $method, 'headers' => $headers, - 'image' => $image, 'source' => $source, 'entrypoint' => $entrypoint, @@ -311,6 +311,8 @@ class Executor $responseType = $responseHeaders['content-type'] ?? ''; $responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE); + $curlError = curl_errno($ch); + $curlErrorMessage = curl_error($ch); if ($decode) { switch (substr($responseType, 0, strpos($responseType, ';'))) { @@ -327,8 +329,11 @@ class Executor } } - if ((curl_errno($ch)/* || 200 != $responseStatus*/)) { - throw new Exception(curl_error($ch) . ' with status code ' . $responseStatus, $responseStatus); + if ($curlError) { + if ($curlError == CURLE_OPERATION_TIMEDOUT) { + throw new AppwriteException(AppwriteException::FUNCTION_SYNCHRONOUS_TIMEOUT); + } + throw new Exception($curlErrorMessage . ' with status code ' . $responseStatus, $responseStatus); } curl_close($ch);