Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Differentiate executor timeouts for builds, sync, and async executions

Browse Source 
Adds Executor\Exception\Timeout (with timeoutSeconds) and translates it at
each call site into BUILD_TIMEOUT, FUNCTION_SYNCHRONOUS_TIMEOUT, or
FUNCTION_ASYNCHRONOUS_TIMEOUT instead of always using the misleading sync
function error. Build timeouts now append to streamed buildLogs rather
than replacing them, and the build worker reports its timeout via Span.
This commit is contained in:
loks0n
2026-05-06 15:50:36 +01:00
parent bf52b1ba31
commit 8201fea9ef
9 changed files with 136 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/config/errors.php
+10
View File
@@ -623,6 +623,11 @@ return [
'description' => 'Synchronous function execution timed out. Use asynchronous execution instead, or ensure the execution duration doesn\'t exceed 30 seconds.',
'code' => 408,
],
Exception::FUNCTION_ASYNCHRONOUS_TIMEOUT => [
'name' => Exception::FUNCTION_ASYNCHRONOUS_TIMEOUT,
'description' => 'Asynchronous function execution timed out. Ensure the execution duration doesn\'t exceed the configured function timeout.',
'code' => 408,
],
Exception::FUNCTION_TEMPLATE_NOT_FOUND => [
'name' => Exception::FUNCTION_TEMPLATE_NOT_FOUND,
'description' => 'Function Template with the requested ID could not be found.',
@@ -687,6 +692,11 @@ return [
'description' => 'Build with the requested ID failed. Please check the logs for more information.',
'code' => 400,
],
Exception::BUILD_TIMEOUT => [
'name' => Exception::BUILD_TIMEOUT,
'description' => 'Build timed out. Increase the build timeout via the `_APP_COMPUTE_BUILD_TIMEOUT` environment variable, or simplify the build to complete within the limit.',
'code' => 408,
],
/** Deployments */
Exception::DEPLOYMENT_NOT_FOUND => [
app/controllers/general.php
+25 -20
View File
@@ -41,6 +41,7 @@ use Appwrite\Utopia\Response\Filters\V23 as ResponseV23;
use Appwrite\Utopia\Response\Filters\V24 as ResponseV24;
use Appwrite\Utopia\Response\Filters\V25 as ResponseV25;
use Appwrite\Utopia\View;
use Executor\Exception\Timeout as ExecutorTimeout;
use Executor\Executor;
use MaxMind\Db\Reader;
use Swoole\Http\Request as SwooleRequest;
@@ -581,26 +582,30 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S
'site' => '',
};
$executionResponse = $executor->createExecution(
projectId: $project->getId(),
deploymentId: $deployment->getId(),
body: \strlen($body) > 0 ? $body : null,
variables: $vars,
timeout: $resource->getAttribute('timeout', 30),
image: $runtime['image'],
source: $source,
entrypoint: $entrypoint,
version: $version,
path: $path,
method: $method,
headers: $headers,
runtimeEntrypoint: $runtimeEntrypoint,
cpus: $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT,
memory: $spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT,
logging: $resource->getAttribute('logging', true),
requestTimeout: 30,
responseFormat: Executor::RESPONSE_FORMAT_ARRAY_HEADERS
);
try {
$executionResponse = $executor->createExecution(
projectId: $project->getId(),
deploymentId: $deployment->getId(),
body: \strlen($body) > 0 ? $body : null,
variables: $vars,
timeout: $resource->getAttribute('timeout', 30),
image: $runtime['image'],
source: $source,
entrypoint: $entrypoint,
version: $version,
path: $path,
method: $method,
headers: $headers,
runtimeEntrypoint: $runtimeEntrypoint,
cpus: $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT,
memory: $spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT,
logging: $resource->getAttribute('logging', true),
requestTimeout: 30,
responseFormat: Executor::RESPONSE_FORMAT_ARRAY_HEADERS
);
} catch (ExecutorTimeout $th) {
throw new AppwriteException(AppwriteException::FUNCTION_SYNCHRONOUS_TIMEOUT, previous: $th);
}
$headerOverrides = [];
src/Appwrite/Extend/Exception.php
+2
View File
@@ -178,6 +178,7 @@ class Exception extends \Exception
public const string FUNCTION_RUNTIME_UNSUPPORTED = 'function_runtime_unsupported';
public const string FUNCTION_ENTRYPOINT_MISSING = 'function_entrypoint_missing';
public const string FUNCTION_SYNCHRONOUS_TIMEOUT = 'function_synchronous_timeout';
public const string FUNCTION_ASYNCHRONOUS_TIMEOUT = 'function_asynchronous_timeout';
public const string FUNCTION_TEMPLATE_NOT_FOUND = 'function_template_not_found';
public const string FUNCTION_RUNTIME_NOT_DETECTED = 'function_runtime_not_detected';
public const string FUNCTION_EXECUTE_PERMISSION_MISSING = 'function_execute_permission_missing';
@@ -192,6 +193,7 @@ class Exception extends \Exception
public const string BUILD_ALREADY_COMPLETED = 'build_already_completed';
public const string BUILD_CANCELED = 'build_canceled';
public const string BUILD_FAILED = 'build_failed';
public const string BUILD_TIMEOUT = 'build_timeout';
/** Execution */
public const string EXECUTION_NOT_FOUND = 'execution_not_found';
src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php
+24 -19
View File
@@ -17,6 +17,7 @@ use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Usage\Context;
use Appwrite\Utopia\Database\Documents\User;
use Appwrite\Utopia\Response;
use Executor\Exception\Timeout as ExecutorTimeout;
use Executor\Executor;
use MaxMind\Db\Reader;
use Utopia\Auth\Proofs\Token;
@@ -417,25 +418,29 @@ class Create extends Base
$source = $deployment->getAttribute('buildPath', '');
$extension = str_ends_with($source, '.tar') ? 'tar' : 'tar.gz';
$command = $version === 'v2' ? '' : "cp /tmp/code.$extension /mnt/code/code.$extension && nohup helpers/start.sh \"$command\"";
$executionResponse = $executor->createExecution(
projectId: $project->getId(),
deploymentId: $deployment->getId(),
body: \strlen($body) > 0 ? $body : null,
variables: $vars,
timeout: $function->getAttribute('timeout', 0),
image: $runtime['image'],
source: $source,
entrypoint: $deployment->getAttribute('entrypoint', ''),
version: $version,
path: $path,
method: $method,
headers: $headers,
runtimeEntrypoint: $command,
cpus: $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT,
memory: $spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT,
logging: $function->getAttribute('logging', true),
requestTimeout: 30
);
try {
$executionResponse = $executor->createExecution(
projectId: $project->getId(),
deploymentId: $deployment->getId(),
body: \strlen($body) > 0 ? $body : null,
variables: $vars,
timeout: $function->getAttribute('timeout', 0),
image: $runtime['image'],
source: $source,
entrypoint: $deployment->getAttribute('entrypoint', ''),
version: $version,
path: $path,
method: $method,
headers: $headers,
runtimeEntrypoint: $command,
cpus: $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT,
memory: $spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT,
logging: $function->getAttribute('logging', true),
requestTimeout: 30
);
} catch (ExecutorTimeout $th) {
throw new AppwriteException(AppwriteException::FUNCTION_SYNCHRONOUS_TIMEOUT, previous: $th);
}
$headersFiltered = [];
foreach ($executionResponse['headers'] as $key => $value) {
src/Appwrite/Platform/Modules/Functions/Workers/Builds.php
+13 -7
View File
@@ -14,7 +14,9 @@ use Appwrite\Filter\BranchDomain as BranchDomainFilter;
use Appwrite\Usage\Context;
use Appwrite\Utopia\Response\Model\Deployment;
use Appwrite\Vcs\Comment;
use Appwrite\Extend\Exception as AppwriteException;
use Exception;
use Executor\Exception\Timeout as ExecutorTimeout;
use Executor\Executor;
use Swoole\Coroutine as Co;
use Utopia\Cache\Cache;
@@ -34,6 +36,7 @@ use Utopia\Detector\Detector\Rendering;
use Utopia\Logger\Log;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\Span\Span;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\System\System;
@@ -183,6 +186,8 @@ class Builds extends Action
array $platform,
int $timeout
): void {
Span::add('timeout', $timeout);
Console::info('Deployment action started');
$startTime = DateTime::now();
@@ -720,6 +725,9 @@ class Builds extends Action
);
Console::log('createRuntime finished');
} catch (ExecutorTimeout $error) {
Console::warning('createRuntime timed out');
$err = new AppwriteException(AppwriteException::BUILD_TIMEOUT, previous: $error);
} catch (\Throwable $error) {
Console::warning('createRuntime failed');
$err = $error;
@@ -1147,13 +1155,11 @@ class Builds extends Action
$message = \str_replace('{APPWRITE_DETECTION_SEPARATOR_START}', '', $message);
$message = \str_replace('{APPWRITE_DETECTION_SEPARATOR_END}', '', $message);
// Combine with previous logs if deployment got past build process
$previousLogs = '';
if (! is_null($deployment->getAttribute('buildSize', null))) {
$previousLogs = $deployment->getAttribute('buildLogs', '');
if (! empty($previousLogs)) {
$message = $previousLogs . "\n" . $message;
}
// Append error to whatever build logs were already streamed
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$previousLogs = $deployment->getAttribute('buildLogs', '');
if (! empty($previousLogs)) {
$message = $previousLogs . "\n" . $message;
}
$endTime = DateTime::now();
src/Appwrite/Platform/Workers/Functions.php
+23 -18
View File
@@ -10,6 +10,7 @@ use Appwrite\Event\Realtime;
use Appwrite\Event\Webhook;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Utopia\Response\Model\Execution;
use Executor\Exception\Timeout as ExecutorTimeout;
use Executor\Executor;
use Utopia\Bus\Bus;
use Utopia\Config\Config;
@@ -565,24 +566,28 @@ class Functions extends Action
Span::add('trigger', $trigger);
Span::current()?->finish();
}
$executionResponse = $executor->createExecution(
projectId: $project->getId(),
deploymentId: $deploymentId,
body: \strlen($body) > 0 ? $body : null,
variables: $vars,
timeout: $function->getAttribute('timeout', 0),
image: $runtime['image'],
source: $source,
entrypoint: $deployment->getAttribute('entrypoint', ''),
version: $version,
path: $path,
method: $method,
headers: $headers,
runtimeEntrypoint: $command,
cpus: $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT,
memory: $spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT,
logging: $function->getAttribute('logging', true),
);
try {
$executionResponse = $executor->createExecution(
projectId: $project->getId(),
deploymentId: $deploymentId,
body: \strlen($body) > 0 ? $body : null,
variables: $vars,
timeout: $function->getAttribute('timeout', 0),
image: $runtime['image'],
source: $source,
entrypoint: $deployment->getAttribute('entrypoint', ''),
version: $version,
path: $path,
method: $method,
headers: $headers,
runtimeEntrypoint: $command,
cpus: $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT,
memory: $spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT,
logging: $function->getAttribute('logging', true),
);
} catch (ExecutorTimeout $th) {
throw new AppwriteException(AppwriteException::FUNCTION_ASYNCHRONOUS_TIMEOUT, previous: $th);
}
$status = $executionResponse['statusCode'] >= 500 ? 'failed' : 'completed';
src/Executor/Exception.php
+7
View File
@@ -0,0 +1,7 @@
<?php
namespace Executor;
class Exception extends \Exception
{
}
src/Executor/Exception/Timeout.php
+23
View File
@@ -0,0 +1,23 @@
<?php
namespace Executor\Exception;
use Executor\Exception;
use Throwable;
class Timeout extends Exception
{
public function __construct(
string $message,
private readonly int $timeoutSeconds,
int $code = 0,
?Throwable $previous = null,
) {
parent::__construct($message, $code, $previous);
}
public function getTimeoutSeconds(): int
{
return $this->timeoutSeconds;
}
}
src/Executor/Executor.php
+9 -9
View File
@@ -2,9 +2,9 @@
namespace Executor;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Utopia\Fetch\BodyMultipart;
use Exception;
use Executor\Exception as ExecutorException;
use Executor\Exception\Timeout as ExecutorTimeout;
use Utopia\System\System;
class Executor
@@ -104,7 +104,7 @@ class Executor
$status = $response['headers']['status-code'];
if ($status >= 400) {
$message = \is_string($response['body']) ? $response['body'] : $response['body']['message'];
throw new \Exception($message, $status);
throw new ExecutorException($message, $status);
}
return $response['body'];
@@ -163,7 +163,7 @@ class Executor
}
if ($status >= 400) {
throw new \Exception($message, $status);
throw new ExecutorException($message, $status);
}
return $response['body'];
@@ -247,7 +247,7 @@ class Executor
$status = $response['headers']['status-code'];
if ($status >= 400) {
$message = \is_string($response['body']) ? $response['body'] : $response['body']['message'];
throw new \Exception($message, $status);
throw new ExecutorException($message, $status);
}
$headers = $response['body']['headers'] ?? [];
@@ -281,7 +281,7 @@ class Executor
$status = $response['headers']['status-code'];
if ($status >= 400) {
$message = \is_string($response['body']) ? $response['body'] : $response['body']['message'];
throw new \Exception($message, $status);
throw new ExecutorException($message, $status);
}
return $response['body'];
@@ -401,7 +401,7 @@ class Executor
$json = json_decode($responseBody, true);
if ($json === null) {
throw new Exception('Failed to parse response: ' . $responseBody);
throw new ExecutorException('Failed to parse response: ' . $responseBody);
}
$responseBody = $json;
@@ -412,9 +412,9 @@ class Executor
if ($curlError) {
if ($curlError == CURLE_OPERATION_TIMEDOUT) {
throw new AppwriteException(AppwriteException::FUNCTION_SYNCHRONOUS_TIMEOUT);
throw new ExecutorTimeout('Executor request timed out', $timeout);
}
throw new Exception($curlErrorMessage . ' with status code ' . $responseStatus, $responseStatus);
throw new ExecutorException($curlErrorMessage . ' with status code ' . $responseStatus, $responseStatus);
}
$responseHeaders['status-code'] = $responseStatus;