From 8201fea9efab00598d2fa710cb73ceac1d469ca0 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 6 May 2026 15:50:36 +0100 Subject: [PATCH] Differentiate executor timeouts for builds, sync, and async executions Adds Executor\Exception\Timeout (with timeoutSeconds) and translates it at each call site into BUILD_TIMEOUT, FUNCTION_SYNCHRONOUS_TIMEOUT, or FUNCTION_ASYNCHRONOUS_TIMEOUT instead of always using the misleading sync function error. Build timeouts now append to streamed buildLogs rather than replacing them, and the build worker reports its timeout via Span. --- app/config/errors.php | 10 +++++ app/controllers/general.php | 45 ++++++++++--------- src/Appwrite/Extend/Exception.php | 2 + .../Functions/Http/Executions/Create.php | 43 ++++++++++-------- .../Modules/Functions/Workers/Builds.php | 20 ++++++--- src/Appwrite/Platform/Workers/Functions.php | 41 +++++++++-------- src/Executor/Exception.php | 7 +++ src/Executor/Exception/Timeout.php | 23 ++++++++++ src/Executor/Executor.php | 18 ++++---- 9 files changed, 136 insertions(+), 73 deletions(-) create mode 100644 src/Executor/Exception.php create mode 100644 src/Executor/Exception/Timeout.php diff --git a/app/config/errors.php b/app/config/errors.php index fa112bcb6f..0293be3212 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -623,6 +623,11 @@ return [ 'description' => 'Synchronous function execution timed out. Use asynchronous execution instead, or ensure the execution duration doesn\'t exceed 30 seconds.', 'code' => 408, ], + Exception::FUNCTION_ASYNCHRONOUS_TIMEOUT => [ + 'name' => Exception::FUNCTION_ASYNCHRONOUS_TIMEOUT, + 'description' => 'Asynchronous function execution timed out. Ensure the execution duration doesn\'t exceed the configured function timeout.', + 'code' => 408, + ], Exception::FUNCTION_TEMPLATE_NOT_FOUND => [ 'name' => Exception::FUNCTION_TEMPLATE_NOT_FOUND, 'description' => 'Function Template with the requested ID could not be found.', @@ -687,6 +692,11 @@ return [ 'description' => 'Build with the requested ID failed. Please check the logs for more information.', 'code' => 400, ], + Exception::BUILD_TIMEOUT => [ + 'name' => Exception::BUILD_TIMEOUT, + 'description' => 'Build timed out. Increase the build timeout via the `_APP_COMPUTE_BUILD_TIMEOUT` environment variable, or simplify the build to complete within the limit.', + 'code' => 408, + ], /** Deployments */ Exception::DEPLOYMENT_NOT_FOUND => [ diff --git a/app/controllers/general.php b/app/controllers/general.php index 21bcded22c..bc63d200d7 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -41,6 +41,7 @@ use Appwrite\Utopia\Response\Filters\V23 as ResponseV23; use Appwrite\Utopia\Response\Filters\V24 as ResponseV24; use Appwrite\Utopia\Response\Filters\V25 as ResponseV25; use Appwrite\Utopia\View; +use Executor\Exception\Timeout as ExecutorTimeout; use Executor\Executor; use MaxMind\Db\Reader; use Swoole\Http\Request as SwooleRequest; @@ -581,26 +582,30 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S 'site' => '', }; - $executionResponse = $executor->createExecution( - projectId: $project->getId(), - deploymentId: $deployment->getId(), - body: \strlen($body) > 0 ? $body : null, - variables: $vars, - timeout: $resource->getAttribute('timeout', 30), - image: $runtime['image'], - source: $source, - entrypoint: $entrypoint, - version: $version, - path: $path, - method: $method, - headers: $headers, - runtimeEntrypoint: $runtimeEntrypoint, - cpus: $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT, - memory: $spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, - logging: $resource->getAttribute('logging', true), - requestTimeout: 30, - responseFormat: Executor::RESPONSE_FORMAT_ARRAY_HEADERS - ); + try { + $executionResponse = $executor->createExecution( + projectId: $project->getId(), + deploymentId: $deployment->getId(), + body: \strlen($body) > 0 ? $body : null, + variables: $vars, + timeout: $resource->getAttribute('timeout', 30), + image: $runtime['image'], + source: $source, + entrypoint: $entrypoint, + version: $version, + path: $path, + method: $method, + headers: $headers, + runtimeEntrypoint: $runtimeEntrypoint, + cpus: $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT, + memory: $spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, + logging: $resource->getAttribute('logging', true), + requestTimeout: 30, + responseFormat: Executor::RESPONSE_FORMAT_ARRAY_HEADERS + ); + } catch (ExecutorTimeout $th) { + throw new AppwriteException(AppwriteException::FUNCTION_SYNCHRONOUS_TIMEOUT, previous: $th); + } $headerOverrides = []; diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 6fc3e88635..2cf8e79944 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -178,6 +178,7 @@ class Exception extends \Exception public const string FUNCTION_RUNTIME_UNSUPPORTED = 'function_runtime_unsupported'; public const string FUNCTION_ENTRYPOINT_MISSING = 'function_entrypoint_missing'; public const string FUNCTION_SYNCHRONOUS_TIMEOUT = 'function_synchronous_timeout'; + public const string FUNCTION_ASYNCHRONOUS_TIMEOUT = 'function_asynchronous_timeout'; public const string FUNCTION_TEMPLATE_NOT_FOUND = 'function_template_not_found'; public const string FUNCTION_RUNTIME_NOT_DETECTED = 'function_runtime_not_detected'; public const string FUNCTION_EXECUTE_PERMISSION_MISSING = 'function_execute_permission_missing'; @@ -192,6 +193,7 @@ class Exception extends \Exception public const string BUILD_ALREADY_COMPLETED = 'build_already_completed'; public const string BUILD_CANCELED = 'build_canceled'; public const string BUILD_FAILED = 'build_failed'; + public const string BUILD_TIMEOUT = 'build_timeout'; /** Execution */ public const string EXECUTION_NOT_FOUND = 'execution_not_found'; diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index 9f15cf9d1e..02dd76294e 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -17,6 +17,7 @@ use Appwrite\SDK\Response as SDKResponse; use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response; +use Executor\Exception\Timeout as ExecutorTimeout; use Executor\Executor; use MaxMind\Db\Reader; use Utopia\Auth\Proofs\Token; @@ -417,25 +418,29 @@ class Create extends Base $source = $deployment->getAttribute('buildPath', ''); $extension = str_ends_with($source, '.tar') ? 'tar' : 'tar.gz'; $command = $version === 'v2' ? '' : "cp /tmp/code.$extension /mnt/code/code.$extension && nohup helpers/start.sh \"$command\""; - $executionResponse = $executor->createExecution( - projectId: $project->getId(), - deploymentId: $deployment->getId(), - body: \strlen($body) > 0 ? $body : null, - variables: $vars, - timeout: $function->getAttribute('timeout', 0), - image: $runtime['image'], - source: $source, - entrypoint: $deployment->getAttribute('entrypoint', ''), - version: $version, - path: $path, - method: $method, - headers: $headers, - runtimeEntrypoint: $command, - cpus: $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT, - memory: $spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, - logging: $function->getAttribute('logging', true), - requestTimeout: 30 - ); + try { + $executionResponse = $executor->createExecution( + projectId: $project->getId(), + deploymentId: $deployment->getId(), + body: \strlen($body) > 0 ? $body : null, + variables: $vars, + timeout: $function->getAttribute('timeout', 0), + image: $runtime['image'], + source: $source, + entrypoint: $deployment->getAttribute('entrypoint', ''), + version: $version, + path: $path, + method: $method, + headers: $headers, + runtimeEntrypoint: $command, + cpus: $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT, + memory: $spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, + logging: $function->getAttribute('logging', true), + requestTimeout: 30 + ); + } catch (ExecutorTimeout $th) { + throw new AppwriteException(AppwriteException::FUNCTION_SYNCHRONOUS_TIMEOUT, previous: $th); + } $headersFiltered = []; foreach ($executionResponse['headers'] as $key => $value) { diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 352fb56e28..d285cf3a36 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -14,7 +14,9 @@ use Appwrite\Filter\BranchDomain as BranchDomainFilter; use Appwrite\Usage\Context; use Appwrite\Utopia\Response\Model\Deployment; use Appwrite\Vcs\Comment; +use Appwrite\Extend\Exception as AppwriteException; use Exception; +use Executor\Exception\Timeout as ExecutorTimeout; use Executor\Executor; use Swoole\Coroutine as Co; use Utopia\Cache\Cache; @@ -34,6 +36,7 @@ use Utopia\Detector\Detector\Rendering; use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Span\Span; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; use Utopia\System\System; @@ -183,6 +186,8 @@ class Builds extends Action array $platform, int $timeout ): void { + Span::add('timeout', $timeout); + Console::info('Deployment action started'); $startTime = DateTime::now(); @@ -720,6 +725,9 @@ class Builds extends Action ); Console::log('createRuntime finished'); + } catch (ExecutorTimeout $error) { + Console::warning('createRuntime timed out'); + $err = new AppwriteException(AppwriteException::BUILD_TIMEOUT, previous: $error); } catch (\Throwable $error) { Console::warning('createRuntime failed'); $err = $error; @@ -1147,13 +1155,11 @@ class Builds extends Action $message = \str_replace('{APPWRITE_DETECTION_SEPARATOR_START}', '', $message); $message = \str_replace('{APPWRITE_DETECTION_SEPARATOR_END}', '', $message); - // Combine with previous logs if deployment got past build process - $previousLogs = ''; - if (! is_null($deployment->getAttribute('buildSize', null))) { - $previousLogs = $deployment->getAttribute('buildLogs', ''); - if (! empty($previousLogs)) { - $message = $previousLogs . "\n" . $message; - } + // Append error to whatever build logs were already streamed + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + $previousLogs = $deployment->getAttribute('buildLogs', ''); + if (! empty($previousLogs)) { + $message = $previousLogs . "\n" . $message; } $endTime = DateTime::now(); diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index 8167fb975d..a72b16cc23 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -10,6 +10,7 @@ use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Utopia\Response\Model\Execution; +use Executor\Exception\Timeout as ExecutorTimeout; use Executor\Executor; use Utopia\Bus\Bus; use Utopia\Config\Config; @@ -565,24 +566,28 @@ class Functions extends Action Span::add('trigger', $trigger); Span::current()?->finish(); } - $executionResponse = $executor->createExecution( - projectId: $project->getId(), - deploymentId: $deploymentId, - body: \strlen($body) > 0 ? $body : null, - variables: $vars, - timeout: $function->getAttribute('timeout', 0), - image: $runtime['image'], - source: $source, - entrypoint: $deployment->getAttribute('entrypoint', ''), - version: $version, - path: $path, - method: $method, - headers: $headers, - runtimeEntrypoint: $command, - cpus: $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT, - memory: $spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, - logging: $function->getAttribute('logging', true), - ); + try { + $executionResponse = $executor->createExecution( + projectId: $project->getId(), + deploymentId: $deploymentId, + body: \strlen($body) > 0 ? $body : null, + variables: $vars, + timeout: $function->getAttribute('timeout', 0), + image: $runtime['image'], + source: $source, + entrypoint: $deployment->getAttribute('entrypoint', ''), + version: $version, + path: $path, + method: $method, + headers: $headers, + runtimeEntrypoint: $command, + cpus: $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT, + memory: $spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, + logging: $function->getAttribute('logging', true), + ); + } catch (ExecutorTimeout $th) { + throw new AppwriteException(AppwriteException::FUNCTION_ASYNCHRONOUS_TIMEOUT, previous: $th); + } $status = $executionResponse['statusCode'] >= 500 ? 'failed' : 'completed'; diff --git a/src/Executor/Exception.php b/src/Executor/Exception.php new file mode 100644 index 0000000000..b799d22567 --- /dev/null +++ b/src/Executor/Exception.php @@ -0,0 +1,7 @@ +timeoutSeconds; + } +} diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index eb74867c9c..c570970732 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -2,9 +2,9 @@ namespace Executor; -use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Utopia\Fetch\BodyMultipart; -use Exception; +use Executor\Exception as ExecutorException; +use Executor\Exception\Timeout as ExecutorTimeout; use Utopia\System\System; class Executor @@ -104,7 +104,7 @@ class Executor $status = $response['headers']['status-code']; if ($status >= 400) { $message = \is_string($response['body']) ? $response['body'] : $response['body']['message']; - throw new \Exception($message, $status); + throw new ExecutorException($message, $status); } return $response['body']; @@ -163,7 +163,7 @@ class Executor } if ($status >= 400) { - throw new \Exception($message, $status); + throw new ExecutorException($message, $status); } return $response['body']; @@ -247,7 +247,7 @@ class Executor $status = $response['headers']['status-code']; if ($status >= 400) { $message = \is_string($response['body']) ? $response['body'] : $response['body']['message']; - throw new \Exception($message, $status); + throw new ExecutorException($message, $status); } $headers = $response['body']['headers'] ?? []; @@ -281,7 +281,7 @@ class Executor $status = $response['headers']['status-code']; if ($status >= 400) { $message = \is_string($response['body']) ? $response['body'] : $response['body']['message']; - throw new \Exception($message, $status); + throw new ExecutorException($message, $status); } return $response['body']; @@ -401,7 +401,7 @@ class Executor $json = json_decode($responseBody, true); if ($json === null) { - throw new Exception('Failed to parse response: ' . $responseBody); + throw new ExecutorException('Failed to parse response: ' . $responseBody); } $responseBody = $json; @@ -412,9 +412,9 @@ class Executor if ($curlError) { if ($curlError == CURLE_OPERATION_TIMEDOUT) { - throw new AppwriteException(AppwriteException::FUNCTION_SYNCHRONOUS_TIMEOUT); + throw new ExecutorTimeout('Executor request timed out', $timeout); } - throw new Exception($curlErrorMessage . ' with status code ' . $responseStatus, $responseStatus); + throw new ExecutorException($curlErrorMessage . ' with status code ' . $responseStatus, $responseStatus); } $responseHeaders['status-code'] = $responseStatus;