diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Delete.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Delete.php index d77a76fe14..4a04afa119 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Delete.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Delete.php @@ -90,6 +90,15 @@ class Delete extends Base } $status = $execution->getAttribute('status'); + // Treat timed-out executions as failed so they can be deleted. + if ($status === 'waiting' || $status === 'processing') { + $timeout = $function->getAttribute('timeout', 900); + $elapsed = \time() - \strtotime($execution->getCreatedAt()); + if ($elapsed >= $timeout) { + $status = 'failed'; + } + } + if (!in_array($status, ['completed', 'failed', 'scheduled'])) { throw new Exception(Exception::EXECUTION_IN_PROGRESS); } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Get.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Get.php index 1fa56ef6f7..70912cf58c 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Get.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Get.php @@ -82,6 +82,16 @@ class Get extends Base throw new Exception(Exception::EXECUTION_NOT_FOUND); } + // Override status in response if the execution is stuck in waiting/processing beyond the function timeout. + $status = $execution->getAttribute('status', ''); + if ($status === 'waiting' || $status === 'processing') { + $timeout = $function->getAttribute('timeout', 900); + $elapsed = \time() - \strtotime($execution->getCreatedAt()); + if ($elapsed >= $timeout) { + $execution->setAttribute('status', 'failed'); + } + } + $response->dynamic($execution, Response::MODEL_EXECUTION); } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/XList.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/XList.php index 404b70b00f..dcc3f6ee9c 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/XList.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/XList.php @@ -11,6 +11,7 @@ use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Database\Validator\Queries\Executions; use Appwrite\Utopia\Response; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Order as OrderException; use Utopia\Database\Exception\Query as QueryException; @@ -110,6 +111,35 @@ class XList extends Base $cursor->setValue($cursorDocument); } + // Calculate the cutoff datetime before which a waiting/processing execution is considered timed out. + $timeout = $function->getAttribute('timeout', 900); + $thresholdDate = new \DateTime("-{$timeout} seconds"); + $threshold = DateTime::format($thresholdDate); + + // Capture what statuses the caller explicitly requested, before we mutate the query. + $requestedStatuses = []; + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'status') { + $requestedStatuses = [...$requestedStatuses, ...$query->getValues()]; + } + } + + // If the caller is filtering by 'failed', expand the DB query to also return + // waiting/processing executions created before the timeout threshold, so timed-out + // executions that were never marked failed in the DB are included in the results. + foreach ($queries as $index => $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'status' && \in_array('failed', $query->getValues())) { + $queries[$index] = Query::or([ + $query, + Query::and([ + Query::equal('status', ['waiting', 'processing']), + Query::createdBefore($threshold), + ]), + ]); + break; + } + } + $filterQueries = Query::groupByType($queries)['filters']; try { @@ -119,6 +149,20 @@ class XList extends Base throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); } + // Override status in response for timed-out executions, but only when the caller + // did not explicitly request a non-failed status (e.g. waiting/processing). + if (empty(\array_diff($requestedStatuses, ['failed']))) { + foreach ($results as $execution) { + $status = $execution->getAttribute('status', ''); + if ($status === 'waiting' || $status === 'processing') { + $elapsed = \time() - \strtotime($execution->getCreatedAt()); + if ($elapsed >= $timeout) { + $execution->setAttribute('status', 'failed'); + } + } + } + } + $response->dynamic(new Document([ 'executions' => $results, 'total' => $total, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Logs/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Logs/Get.php index b10bc6babd..c482add2da 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Logs/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Logs/Get.php @@ -71,6 +71,16 @@ class Get extends Base throw new Exception(Exception::LOG_NOT_FOUND); } + // Override status in response if the log is stuck in waiting/processing beyond the site timeout. + $status = $log->getAttribute('status', ''); + if ($status === 'waiting' || $status === 'processing') { + $timeout = $site->getAttribute('timeout', 30); + $elapsed = \time() - \strtotime($log->getCreatedAt()); + if ($elapsed >= $timeout) { + $log->setAttribute('status', 'failed'); + } + } + $response->dynamic($log, Response::MODEL_EXECUTION); //TODO: Change to model log, but model log already exists - decide what to do } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Logs/XList.php b/src/Appwrite/Platform/Modules/Sites/Http/Logs/XList.php index 8b5e38f3db..89fb3ee2e6 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Logs/XList.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Logs/XList.php @@ -11,6 +11,7 @@ use Appwrite\Utopia\Database\Validator\Queries\Executions; use Appwrite\Utopia\Database\Validator\Queries\Logs; use Appwrite\Utopia\Response; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Order as OrderException; use Utopia\Database\Exception\Query as QueryException; @@ -99,6 +100,35 @@ class XList extends Base $cursor->setValue($cursorDocument); } + // Calculate the cutoff datetime before which a waiting/processing log is considered timed out. + $timeout = $site->getAttribute('timeout', 30); + $thresholdDate = new \DateTime("-{$timeout} seconds"); + $threshold = DateTime::format($thresholdDate); + + // Capture what statuses the caller explicitly requested, before we mutate the query. + $requestedStatuses = []; + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'status') { + $requestedStatuses = [...$requestedStatuses, ...$query->getValues()]; + } + } + + // If the caller is filtering by 'failed', expand the DB query to also return + // waiting/processing logs created before the timeout threshold, so timed-out + // logs that were never marked failed in the DB are included in the results. + foreach ($queries as $index => $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'status' && \in_array('failed', $query->getValues())) { + $queries[$index] = Query::or([ + $query, + Query::and([ + Query::equal('status', ['waiting', 'processing']), + Query::createdBefore($threshold), + ]), + ]); + break; + } + } + $filterQueries = Query::groupByType($queries)['filters']; try { @@ -108,6 +138,20 @@ class XList extends Base throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); } + // Override status in response for timed-out logs, but only when the caller + // did not explicitly request a non-failed status (e.g. waiting/processing). + if (empty(\array_diff($requestedStatuses, ['failed']))) { + foreach ($results as $log) { + $status = $log->getAttribute('status', ''); + if ($status === 'waiting' || $status === 'processing') { + $elapsed = \time() - \strtotime($log->getCreatedAt()); + if ($elapsed >= $timeout) { + $log->setAttribute('status', 'failed'); + } + } + } + } + $response->dynamic(new Document([ 'executions' => $results, 'total' => $total,