mirror of
https://github.com/appwrite/appwrite.git
synced 2026-05-26 13:51:13 +00:00
Merge pull request #11400 from appwrite/fix-execution-timeout-status
fix: show timed-out executions as failed across API endpoints
This commit is contained in:
@@ -90,6 +90,15 @@ class Delete extends Base
|
||||
}
|
||||
$status = $execution->getAttribute('status');
|
||||
|
||||
// Treat timed-out executions as failed so they can be deleted.
|
||||
if ($status === 'waiting' || $status === 'processing') {
|
||||
$timeout = $function->getAttribute('timeout', 900);
|
||||
$elapsed = \time() - \strtotime($execution->getCreatedAt());
|
||||
if ($elapsed >= $timeout) {
|
||||
$status = 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
if (!in_array($status, ['completed', 'failed', 'scheduled'])) {
|
||||
throw new Exception(Exception::EXECUTION_IN_PROGRESS);
|
||||
}
|
||||
|
||||
@@ -82,6 +82,16 @@ class Get extends Base
|
||||
throw new Exception(Exception::EXECUTION_NOT_FOUND);
|
||||
}
|
||||
|
||||
// Override status in response if the execution is stuck in waiting/processing beyond the function timeout.
|
||||
$status = $execution->getAttribute('status', '');
|
||||
if ($status === 'waiting' || $status === 'processing') {
|
||||
$timeout = $function->getAttribute('timeout', 900);
|
||||
$elapsed = \time() - \strtotime($execution->getCreatedAt());
|
||||
if ($elapsed >= $timeout) {
|
||||
$execution->setAttribute('status', 'failed');
|
||||
}
|
||||
}
|
||||
|
||||
$response->dynamic($execution, Response::MODEL_EXECUTION);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ use Appwrite\Utopia\Database\Documents\User;
|
||||
use Appwrite\Utopia\Database\Validator\Queries\Executions;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Exception\Order as OrderException;
|
||||
use Utopia\Database\Exception\Query as QueryException;
|
||||
@@ -110,6 +111,35 @@ class XList extends Base
|
||||
$cursor->setValue($cursorDocument);
|
||||
}
|
||||
|
||||
// Calculate the cutoff datetime before which a waiting/processing execution is considered timed out.
|
||||
$timeout = $function->getAttribute('timeout', 900);
|
||||
$thresholdDate = new \DateTime("-{$timeout} seconds");
|
||||
$threshold = DateTime::format($thresholdDate);
|
||||
|
||||
// Capture what statuses the caller explicitly requested, before we mutate the query.
|
||||
$requestedStatuses = [];
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'status') {
|
||||
$requestedStatuses = [...$requestedStatuses, ...$query->getValues()];
|
||||
}
|
||||
}
|
||||
|
||||
// If the caller is filtering by 'failed', expand the DB query to also return
|
||||
// waiting/processing executions created before the timeout threshold, so timed-out
|
||||
// executions that were never marked failed in the DB are included in the results.
|
||||
foreach ($queries as $index => $query) {
|
||||
if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'status' && \in_array('failed', $query->getValues())) {
|
||||
$queries[$index] = Query::or([
|
||||
$query,
|
||||
Query::and([
|
||||
Query::equal('status', ['waiting', 'processing']),
|
||||
Query::createdBefore($threshold),
|
||||
]),
|
||||
]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
$filterQueries = Query::groupByType($queries)['filters'];
|
||||
|
||||
try {
|
||||
@@ -119,6 +149,20 @@ class XList extends Base
|
||||
throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null.");
|
||||
}
|
||||
|
||||
// Override status in response for timed-out executions, but only when the caller
|
||||
// did not explicitly request a non-failed status (e.g. waiting/processing).
|
||||
if (empty(\array_diff($requestedStatuses, ['failed']))) {
|
||||
foreach ($results as $execution) {
|
||||
$status = $execution->getAttribute('status', '');
|
||||
if ($status === 'waiting' || $status === 'processing') {
|
||||
$elapsed = \time() - \strtotime($execution->getCreatedAt());
|
||||
if ($elapsed >= $timeout) {
|
||||
$execution->setAttribute('status', 'failed');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$response->dynamic(new Document([
|
||||
'executions' => $results,
|
||||
'total' => $total,
|
||||
|
||||
@@ -71,6 +71,16 @@ class Get extends Base
|
||||
throw new Exception(Exception::LOG_NOT_FOUND);
|
||||
}
|
||||
|
||||
// Override status in response if the log is stuck in waiting/processing beyond the site timeout.
|
||||
$status = $log->getAttribute('status', '');
|
||||
if ($status === 'waiting' || $status === 'processing') {
|
||||
$timeout = $site->getAttribute('timeout', 30);
|
||||
$elapsed = \time() - \strtotime($log->getCreatedAt());
|
||||
if ($elapsed >= $timeout) {
|
||||
$log->setAttribute('status', 'failed');
|
||||
}
|
||||
}
|
||||
|
||||
$response->dynamic($log, Response::MODEL_EXECUTION); //TODO: Change to model log, but model log already exists - decide what to do
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ use Appwrite\Utopia\Database\Validator\Queries\Executions;
|
||||
use Appwrite\Utopia\Database\Validator\Queries\Logs;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Exception\Order as OrderException;
|
||||
use Utopia\Database\Exception\Query as QueryException;
|
||||
@@ -99,6 +100,35 @@ class XList extends Base
|
||||
$cursor->setValue($cursorDocument);
|
||||
}
|
||||
|
||||
// Calculate the cutoff datetime before which a waiting/processing log is considered timed out.
|
||||
$timeout = $site->getAttribute('timeout', 30);
|
||||
$thresholdDate = new \DateTime("-{$timeout} seconds");
|
||||
$threshold = DateTime::format($thresholdDate);
|
||||
|
||||
// Capture what statuses the caller explicitly requested, before we mutate the query.
|
||||
$requestedStatuses = [];
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'status') {
|
||||
$requestedStatuses = [...$requestedStatuses, ...$query->getValues()];
|
||||
}
|
||||
}
|
||||
|
||||
// If the caller is filtering by 'failed', expand the DB query to also return
|
||||
// waiting/processing logs created before the timeout threshold, so timed-out
|
||||
// logs that were never marked failed in the DB are included in the results.
|
||||
foreach ($queries as $index => $query) {
|
||||
if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'status' && \in_array('failed', $query->getValues())) {
|
||||
$queries[$index] = Query::or([
|
||||
$query,
|
||||
Query::and([
|
||||
Query::equal('status', ['waiting', 'processing']),
|
||||
Query::createdBefore($threshold),
|
||||
]),
|
||||
]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
$filterQueries = Query::groupByType($queries)['filters'];
|
||||
|
||||
try {
|
||||
@@ -108,6 +138,20 @@ class XList extends Base
|
||||
throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null.");
|
||||
}
|
||||
|
||||
// Override status in response for timed-out logs, but only when the caller
|
||||
// did not explicitly request a non-failed status (e.g. waiting/processing).
|
||||
if (empty(\array_diff($requestedStatuses, ['failed']))) {
|
||||
foreach ($results as $log) {
|
||||
$status = $log->getAttribute('status', '');
|
||||
if ($status === 'waiting' || $status === 'processing') {
|
||||
$elapsed = \time() - \strtotime($log->getCreatedAt());
|
||||
if ($elapsed >= $timeout) {
|
||||
$log->setAttribute('status', 'failed');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$response->dynamic(new Document([
|
||||
'executions' => $results,
|
||||
'total' => $total,
|
||||
|
||||
Reference in New Issue
Block a user