From 88918906014d7b8119ec2e563a53a1763a5bc116 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 26 Feb 2026 09:22:33 +0530 Subject: [PATCH] fix: show timed-out executions as failed across API endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Executions that time out can remain stuck in waiting or processing status in the database. This mirrors the frontend workaround from console#2788 across the relevant API endpoints for both functions and sites. Changes: - GET execution/log: override status to failed in response if elapsed time since creation exceeds the resource timeout - LIST executions/logs: same in-response override; when caller filters by failed, expands DB query with OR to also fetch waiting/processing entries created before the timeout threshold so they appear in results; skips in-response override when caller explicitly requests a non-failed status to avoid contradicting the filter - DELETE execution: allows deletion of timed-out executions that are still stored as waiting/processing by treating them as failed for the status guard All changes are in-memory only — the database records are not modified. Includes a note to remove once a proper DB-level fix is applied. --- .../Functions/Http/Executions/Delete.php | 9 ++++ .../Modules/Functions/Http/Executions/Get.php | 10 +++++ .../Functions/Http/Executions/XList.php | 44 +++++++++++++++++++ .../Platform/Modules/Sites/Http/Logs/Get.php | 10 +++++ .../Modules/Sites/Http/Logs/XList.php | 44 +++++++++++++++++++ 5 files changed, 117 insertions(+) diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Delete.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Delete.php index d77a76fe14..4a04afa119 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Delete.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Delete.php @@ -90,6 +90,15 @@ class Delete extends Base } $status = $execution->getAttribute('status'); + // Treat timed-out executions as failed so they can be deleted. + if ($status === 'waiting' || $status === 'processing') { + $timeout = $function->getAttribute('timeout', 900); + $elapsed = \time() - \strtotime($execution->getCreatedAt()); + if ($elapsed >= $timeout) { + $status = 'failed'; + } + } + if (!in_array($status, ['completed', 'failed', 'scheduled'])) { throw new Exception(Exception::EXECUTION_IN_PROGRESS); } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Get.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Get.php index 1fa56ef6f7..70912cf58c 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Get.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Get.php @@ -82,6 +82,16 @@ class Get extends Base throw new Exception(Exception::EXECUTION_NOT_FOUND); } + // Override status in response if the execution is stuck in waiting/processing beyond the function timeout. + $status = $execution->getAttribute('status', ''); + if ($status === 'waiting' || $status === 'processing') { + $timeout = $function->getAttribute('timeout', 900); + $elapsed = \time() - \strtotime($execution->getCreatedAt()); + if ($elapsed >= $timeout) { + $execution->setAttribute('status', 'failed'); + } + } + $response->dynamic($execution, Response::MODEL_EXECUTION); } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/XList.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/XList.php index 404b70b00f..dcc3f6ee9c 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/XList.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/XList.php @@ -11,6 +11,7 @@ use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Database\Validator\Queries\Executions; use Appwrite\Utopia\Response; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Order as OrderException; use Utopia\Database\Exception\Query as QueryException; @@ -110,6 +111,35 @@ class XList extends Base $cursor->setValue($cursorDocument); } + // Calculate the cutoff datetime before which a waiting/processing execution is considered timed out. + $timeout = $function->getAttribute('timeout', 900); + $thresholdDate = new \DateTime("-{$timeout} seconds"); + $threshold = DateTime::format($thresholdDate); + + // Capture what statuses the caller explicitly requested, before we mutate the query. + $requestedStatuses = []; + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'status') { + $requestedStatuses = [...$requestedStatuses, ...$query->getValues()]; + } + } + + // If the caller is filtering by 'failed', expand the DB query to also return + // waiting/processing executions created before the timeout threshold, so timed-out + // executions that were never marked failed in the DB are included in the results. + foreach ($queries as $index => $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'status' && \in_array('failed', $query->getValues())) { + $queries[$index] = Query::or([ + $query, + Query::and([ + Query::equal('status', ['waiting', 'processing']), + Query::createdBefore($threshold), + ]), + ]); + break; + } + } + $filterQueries = Query::groupByType($queries)['filters']; try { @@ -119,6 +149,20 @@ class XList extends Base throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); } + // Override status in response for timed-out executions, but only when the caller + // did not explicitly request a non-failed status (e.g. waiting/processing). + if (empty(\array_diff($requestedStatuses, ['failed']))) { + foreach ($results as $execution) { + $status = $execution->getAttribute('status', ''); + if ($status === 'waiting' || $status === 'processing') { + $elapsed = \time() - \strtotime($execution->getCreatedAt()); + if ($elapsed >= $timeout) { + $execution->setAttribute('status', 'failed'); + } + } + } + } + $response->dynamic(new Document([ 'executions' => $results, 'total' => $total, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Logs/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Logs/Get.php index b10bc6babd..c482add2da 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Logs/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Logs/Get.php @@ -71,6 +71,16 @@ class Get extends Base throw new Exception(Exception::LOG_NOT_FOUND); } + // Override status in response if the log is stuck in waiting/processing beyond the site timeout. + $status = $log->getAttribute('status', ''); + if ($status === 'waiting' || $status === 'processing') { + $timeout = $site->getAttribute('timeout', 30); + $elapsed = \time() - \strtotime($log->getCreatedAt()); + if ($elapsed >= $timeout) { + $log->setAttribute('status', 'failed'); + } + } + $response->dynamic($log, Response::MODEL_EXECUTION); //TODO: Change to model log, but model log already exists - decide what to do } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Logs/XList.php b/src/Appwrite/Platform/Modules/Sites/Http/Logs/XList.php index 8b5e38f3db..89fb3ee2e6 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Logs/XList.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Logs/XList.php @@ -11,6 +11,7 @@ use Appwrite\Utopia\Database\Validator\Queries\Executions; use Appwrite\Utopia\Database\Validator\Queries\Logs; use Appwrite\Utopia\Response; use Utopia\Database\Database; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Order as OrderException; use Utopia\Database\Exception\Query as QueryException; @@ -99,6 +100,35 @@ class XList extends Base $cursor->setValue($cursorDocument); } + // Calculate the cutoff datetime before which a waiting/processing log is considered timed out. + $timeout = $site->getAttribute('timeout', 30); + $thresholdDate = new \DateTime("-{$timeout} seconds"); + $threshold = DateTime::format($thresholdDate); + + // Capture what statuses the caller explicitly requested, before we mutate the query. + $requestedStatuses = []; + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'status') { + $requestedStatuses = [...$requestedStatuses, ...$query->getValues()]; + } + } + + // If the caller is filtering by 'failed', expand the DB query to also return + // waiting/processing logs created before the timeout threshold, so timed-out + // logs that were never marked failed in the DB are included in the results. + foreach ($queries as $index => $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'status' && \in_array('failed', $query->getValues())) { + $queries[$index] = Query::or([ + $query, + Query::and([ + Query::equal('status', ['waiting', 'processing']), + Query::createdBefore($threshold), + ]), + ]); + break; + } + } + $filterQueries = Query::groupByType($queries)['filters']; try { @@ -108,6 +138,20 @@ class XList extends Base throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); } + // Override status in response for timed-out logs, but only when the caller + // did not explicitly request a non-failed status (e.g. waiting/processing). + if (empty(\array_diff($requestedStatuses, ['failed']))) { + foreach ($results as $log) { + $status = $log->getAttribute('status', ''); + if ($status === 'waiting' || $status === 'processing') { + $elapsed = \time() - \strtotime($log->getCreatedAt()); + if ($elapsed >= $timeout) { + $log->setAttribute('status', 'failed'); + } + } + } + } + $response->dynamic(new Document([ 'executions' => $results, 'total' => $total,