Merge pull request #12433 from appwrite/revert-execution-log-removal

Revert execution log removal
This commit is contained in:
Chirag Aggarwal authored and GitHub committed 2026-05-30 19:34:07 +05:30
commit 382f08f056
19 files changed
+241 -64

No files matched your search

+1
View File
@@ -89,6 +89,7 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-certificates && \
chmod +x /usr/local/bin/worker-databases && \
chmod +x /usr/local/bin/worker-deletes && \
chmod +x /usr/local/bin/worker-executions && \
chmod +x /usr/local/bin/worker-functions && \
chmod +x /usr/local/bin/worker-mails && \
chmod +x /usr/local/bin/worker-messaging && \
+3 -4
View File
@@ -714,12 +714,11 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S
spec: $spec,
resource: $resource->getArrayCopy(),
));
$execution->setAttribute('logs', '');
$execution->setAttribute('errors', '');
$execution = $authorization->skip(fn () => $dbForProject->createDocument('executions', $execution));
}
$execution->setAttribute('logs', '');
$execution->setAttribute('errors', '');
$headers = [];
foreach (($executionResponse['headers'] ?? []) as $key => $value) {
$headers[] = ['name' => $key, 'value' => \is_array($value) ? \implode(', ', $value) : $value];
+6
View File
@@ -7,6 +7,7 @@ use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Event\Publisher\Certificate as CertificatePublisher;
use Appwrite\Event\Publisher\Database as DatabasePublisher;
use Appwrite\Event\Publisher\Delete as DeletePublisher;
use Appwrite\Event\Publisher\Execution as ExecutionPublisher;
use Appwrite\Event\Publisher\Func as FunctionPublisher;
use Appwrite\Event\Publisher\Mail as MailPublisher;
use Appwrite\Event\Publisher\Messaging as MessagingPublisher;
@@ -104,6 +105,11 @@ $container->set('publisherForUsage', fn (Publisher $publisher) => new UsagePubli
new Queue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME))
), ['publisher']);
$container->set('publisherForExecutions', fn (Publisher $publisher) => new ExecutionPublisher(
$publisher,
new Queue(System::getEnv('_APP_EXECUTIONS_QUEUE_NAME', Event::EXECUTIONS_QUEUE_NAME))
), ['publisher']);
$container->set('publisherForFunctions', fn (Publisher $publisher) => new FunctionPublisher(
$publisher,
new Queue(System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME), 'utopia-queue', Event::FUNCTIONS_QUEUE_TTL)
+2
View File
@@ -1,9 +1,11 @@
<?php
use Appwrite\Bus\Listeners\Log;
use Appwrite\Bus\Listeners\Mails;
use Appwrite\Bus\Listeners\Usage;
return [
new Log(),
new Mails(),
new Usage(),
];
+29
View File
@@ -524,6 +524,35 @@ $hostPath = rtrim($this->getParam('hostPath', ''), '/');
- _APP_DB_PASS
- _APP_LOGGING_CONFIG
appwrite-worker-executions:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-executions
<<: *x-logging
container_name: appwrite-worker-executions
restart: unless-stopped
networks:
- appwrite
depends_on:
redis:
condition: service_healthy
<?= $dbService ?>:
condition: service_healthy
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_ENV
- _APP_DB_ADAPTER
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_LOGGING_CONFIG
appwrite-worker-functions:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-functions
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
exec php /usr/src/code/app/worker.php executions "$@"
+31
View File
@@ -690,6 +690,37 @@ services:
- _APP_LOGGING_CONFIG
- _APP_DATABASE_SHARED_TABLES
appwrite-worker-executions:
entrypoint: worker-executions
<<: [*x-logging, *x-build]
container_name: appwrite-worker-executions
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- ${_APP_DB_HOST:-mongodb}
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_ADAPTER
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_LOGGING_CONFIG
- _APP_DATABASE_SHARED_TABLES
appwrite-worker-functions:
entrypoint: worker-functions
<<: [*x-logging, *x-build]
@@ -1,18 +0,0 @@
<?php
namespace Appwrite\Bus\Events;
use Utopia\Bus\Event;
class ExecutionLogged implements Event
{
/**
* @param array<string, mixed> $execution
* @param array<string, mixed> $project
*/
public function __construct(
public readonly array $execution,
public readonly array $project,
) {
}
}
+50
View File
@@ -0,0 +1,50 @@
<?php
namespace Appwrite\Bus\Listeners;
use Appwrite\Bus\Events\ExecutionCompleted;
use Appwrite\Event\Message\Execution as ExecutionMessage;
use Appwrite\Event\Publisher\Execution as ExecutionPublisher;
use Utopia\Bus\Listener;
use Utopia\Database\Document;
use Utopia\Span\Span;
class Log extends Listener
{
public static function getName(): string
{
return 'log';
}
public static function getEvents(): array
{
return [ExecutionCompleted::class];
}
public function __construct()
{
$this
->desc('Persists execution logs to database via queue')
->inject('publisherForExecutions')
->callback($this->handle(...));
}
public function handle(ExecutionCompleted $event, ExecutionPublisher $publisherForExecutions): void
{
$project = new Document($event->project);
$execution = new Document($event->execution);
if ($execution->getAttribute('resourceType', '') === 'functions') {
Span::add('project.id', $project->getId());
Span::add('function.id', $execution->getAttribute('resourceId', ''));
Span::add('execution.id', $execution->getId());
Span::add('deployment.id', $execution->getAttribute('deploymentId', ''));
Span::add('execution.status', $execution->getAttribute('status', ''));
}
$publisherForExecutions->enqueue(new ExecutionMessage(
project: $project,
execution: $execution,
));
}
}
@@ -3,7 +3,6 @@
namespace Appwrite\Platform\Modules\Functions\Http\Executions;
use Ahc\Jwt\JWT;
use Appwrite\Bus\Events\ExecutionLogged;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Delete as DeleteMessage;
use Appwrite\Event\Message\Func as FunctionMessage;
@@ -25,7 +24,6 @@ use Executor\Executor;
use MaxMind\Db\Reader;
use Utopia\Auth\Proofs\Token;
use Utopia\Auth\Store;
use Utopia\Bus\Bus;
use Utopia\Config\Config;
use Utopia\Console;
use Utopia\Database\Database;
@@ -109,7 +107,6 @@ class Create extends Base
->inject('authorization')
->inject('publisherForDeletes')
->inject('executionsRetentionCount')
->inject('bus')
->callback($this->action(...));
}
@@ -138,7 +135,6 @@ class Create extends Base
Authorization $authorization,
DeletePublisher $publisherForDeletes,
int $executionsRetentionCount,
Bus $bus,
) {
$async = \strval($async) === 'true' || \strval($async) === '1';
@@ -341,11 +337,6 @@ class Create extends Base
->setAttribute('scheduledAt', $scheduledAt);
$execution = $authorization->skip(fn () => $dbForProject->createDocument('executions', $execution));
$bus->dispatch(new ExecutionLogged(
execution: $execution->getArrayCopy(),
project: $project->getArrayCopy(),
));
}
if ($executionsRetentionCount > 0 && ENABLE_EXECUTIONS_LIMIT_ON_ROUTE) {
@@ -516,11 +507,6 @@ class Create extends Base
;
$execution = $authorization->skip(fn () => $dbForProject->createDocument('executions', $execution));
$bus->dispatch(new ExecutionLogged(
execution: $execution->getArrayCopy(),
project: $project->getArrayCopy(),
));
}
$executionResponse['headers']['x-appwrite-execution-id'] = $execution->getId();
@@ -5,6 +5,7 @@ namespace Appwrite\Platform\Services;
use Appwrite\Platform\Workers\Audits;
use Appwrite\Platform\Workers\Certificates;
use Appwrite\Platform\Workers\Deletes;
use Appwrite\Platform\Workers\Executions;
use Appwrite\Platform\Workers\Functions;
use Appwrite\Platform\Workers\Mails;
use Appwrite\Platform\Workers\Messaging;
@@ -23,6 +24,7 @@ class Workers extends Service
->addAction(Audits::getName(), new Audits())
->addAction(Certificates::getName(), new Certificates())
->addAction(Deletes::getName(), new Deletes())
->addAction(Executions::getName(), new Executions())
->addAction(Functions::getName(), new Functions())
->addAction(Mails::getName(), new Mails())
->addAction(Messaging::getName(), new Messaging())
@@ -0,0 +1,51 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Event\Message\Execution;
use Exception;
use Utopia\Database\Database;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\Span\Span;
class Executions extends Action
{
public static function getName(): string
{
return 'executions';
}
/**
* @throws Exception
*/
public function __construct()
{
$this
->desc('Executions worker')
->groups(['executions'])
->inject('message')
->inject('dbForProject')
->callback($this->action(...));
}
public function action(
Message $message,
Database $dbForProject,
): void {
$executionMessage = Execution::fromArray($message->getPayload());
$execution = $executionMessage->execution;
if ($execution->isEmpty()) {
throw new Exception('Missing execution');
}
Span::add('project.id', $executionMessage->project->getId());
Span::add('function.id', $execution->getAttribute('resourceId', ''));
Span::add('execution.id', $execution->getId());
Span::add('deployment.id', $execution->getAttribute('deploymentId', ''));
Span::add('resource.type', $execution->getAttribute('resourceType', ''));
$dbForProject->upsertDocument('executions', $execution);
}
}
+5 -21
View File
@@ -263,7 +263,6 @@ class Functions extends Action
/**
* @param string $message
* @param Database $dbForProject
* @param Document $function
* @param string $trigger
* @param string $path
@@ -275,7 +274,6 @@ class Functions extends Action
*/
private function fail(
string $message,
Database $dbForProject,
Document $project,
Bus $bus,
Document $function,
@@ -315,24 +313,19 @@ class Functions extends Action
'requestPath' => $path,
'requestMethod' => $method,
'requestHeaders' => $headersFiltered,
'errors' => '',
'errors' => $message,
'logs' => '',
'duration' => 0.0,
]);
$executionForEvent = (new Document($execution->getArrayCopy()))
->setAttribute('errors', $message);
Span::add('function.id', $function->getId());
Span::add('execution.id', $execution->getId());
Span::add('deployment.id', $execution->getAttribute('deploymentId', ''));
Span::add('execution.trigger', $trigger);
Span::add('execution.status', $execution->getAttribute('status', ''));
$dbForProject->upsertDocument('executions', $execution);
$bus->dispatch(new ExecutionCompleted(
execution: $executionForEvent->getArrayCopy(),
execution: $execution->getArrayCopy(),
project: $project->getArrayCopy(),
));
}
@@ -397,19 +390,19 @@ class Functions extends Action
if ($deployment->getAttribute('resourceId') !== $functionId) {
$errorMessage = 'The execution could not be completed because a corresponding deployment was not found. A function deployment needs to be created before it can be executed. Please create a deployment for your function and try again.';
$this->fail($errorMessage, $dbForProject, $project, $bus, $function, $trigger, $path, $method, $user, $jwt, $event);
$this->fail($errorMessage, $project, $bus, $function, $trigger, $path, $method, $user, $jwt, $event);
return;
}
if ($deployment->isEmpty()) {
$errorMessage = 'The execution could not be completed because a corresponding deployment was not found. A function deployment needs to be created before it can be executed. Please create a deployment for your function and try again.';
$this->fail($errorMessage, $dbForProject, $project, $bus, $function, $trigger, $path, $method, $user, $jwt, $event);
$this->fail($errorMessage, $project, $bus, $function, $trigger, $path, $method, $user, $jwt, $event);
return;
}
if ($deployment->getAttribute('status') !== 'ready') {
$errorMessage = 'The execution could not be completed because the build is not ready. Please wait for the build to complete and try again.';
$this->fail($errorMessage, $dbForProject, $project, $bus, $function, $trigger, $path, $method, $user, $jwt, $event);
$this->fail($errorMessage, $project, $bus, $function, $trigger, $path, $method, $user, $jwt, $event);
return;
}
@@ -478,8 +471,6 @@ class Functions extends Action
'duration' => 0.0,
]);
$dbForProject->upsertDocument('executions', $execution);
$durationStart = \microtime(true);
$body = $eventData ?? '';
@@ -635,13 +626,6 @@ class Functions extends Action
/** Persist final execution status and record usage */
Span::add('execution.status', $execution->getAttribute('status', ''));
$dbForProject->updateDocument('executions', $execution->getId(), new Document([
'status' => $execution->getAttribute('status'),
'responseStatusCode' => $execution->getAttribute('responseStatusCode'),
'responseHeaders' => $execution->getAttribute('responseHeaders'),
'duration' => $execution->getAttribute('duration'),
]));
$bus->dispatch(new ExecutionCompleted(
execution: $execution->getArrayCopy(),
project: $project->getArrayCopy(),
@@ -590,7 +590,9 @@ class FunctionsCustomClientTest extends Scope
'data' => ['name' => 'Test Document'],
]);
$this->assertEquals(201, $document['headers']['status-code']);
$this->assertEventually(function () use ($functionId) {
$documentId = $document['body']['$id'];
$this->assertEventually(function () use ($functionId, $documentId) {
$executions = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/executions', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
@@ -603,6 +605,7 @@ class FunctionsCustomClientTest extends Scope
$lastExecution = $executions['body']['executions'][0];
$this->assertEquals('completed', $lastExecution['status']);
$this->assertEquals(204, $lastExecution['responseStatusCode']);
$this->assertStringContainsString($documentId, $lastExecution['logs']);
}, 20000, 500);
$this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId, [
@@ -745,13 +745,15 @@ class FunctionsCustomServerTest extends Scope
$executionId = $execution['body']['$id'] ?? '';
$this->assertEventually(function () use ($functionId, $executionId) {
$this->assertEventually(function () use ($functionId, $executionId, $totalUsers) {
$execution = $this->getExecution($functionId, $executionId);
$this->assertEquals(200, $execution['headers']['status-code']);
$this->assertEquals(200, $execution['body']['responseStatusCode']);
$this->assertEquals('completed', $execution['body']['status']);
$this->assertEmpty($execution['body']['responseBody']);
$this->assertEmpty($execution['body']['errors']);
$this->assertStringContainsString("Total users: " . $totalUsers, $execution['body']['logs']);
}, 10000, 500);
$deployment = $this->getDeployment($functionId, $deployment['body']['$id']);
@@ -2121,6 +2123,8 @@ class FunctionsCustomServerTest extends Scope
$this->assertGreaterThan(2, $execution['body']['duration']);
$this->assertLessThan(20, $execution['body']['duration']);
$this->assertEquals('', $execution['body']['responseBody']);
$this->assertEquals('', $execution['body']['logs']);
$this->assertStringContainsString('timed out', $execution['body']['errors']);
}, 10000, 500);
$this->cleanupFunction($functionId);
@@ -2343,7 +2347,7 @@ class FunctionsCustomServerTest extends Scope
$userId = $user['body']['$id'] ?? '';
$this->assertEventually(function () use ($functionId) {
$this->assertEventually(function () use ($functionId, $userId) {
$executions = $this->listExecutions($functionId);
$this->assertEquals(200, $executions['headers']['status-code']);
@@ -2353,6 +2357,8 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals('completed', $lastExecution['status']);
$this->assertEquals(204, $lastExecution['responseStatusCode']);
$this->assertStringContainsString($userId, $lastExecution['logs']);
$this->assertStringContainsString('Event User', $lastExecution['logs']);
$this->assertNotEmpty($lastExecution['$id']);
$headers = array_column($lastExecution['requestHeaders'] ?? [], 'value', 'name');
$this->assertEmpty($headers['x-appwrite-client-ip'] ?? '');
@@ -2420,6 +2426,8 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals('completed', $execution['body']['status']);
$this->assertEquals(200, $execution['body']['responseStatusCode']);
$this->assertGreaterThan(0, $execution['body']['duration']);
$this->assertNotEmpty($execution['body']['logs']);
$this->assertStringContainsString("total", $execution['body']['logs']);
}, 10000, 500);
$this->cleanupFunction($functionId);
@@ -58,6 +58,8 @@ class FunctionsScheduleTest extends Scope
$this->assertEquals('completed', $asyncExecution['status']);
$this->assertEquals(200, $asyncExecution['responseStatusCode']);
$this->assertEquals('', $asyncExecution['responseBody']);
$this->assertNotEmpty($asyncExecution['logs']);
$this->assertNotEmpty($asyncExecution['errors']);
$this->assertGreaterThan(0, $asyncExecution['duration']);
$this->assertNotEmpty($asyncExecution['$id']);
$headers = array_column($asyncExecution['requestHeaders'] ?? [], 'value', 'name');
@@ -130,6 +132,12 @@ class FunctionsScheduleTest extends Scope
$this->assertEquals('completed', $execution['body']['status']);
$this->assertEquals('/custom-path', $execution['body']['requestPath']);
$this->assertEquals('PATCH', $execution['body']['requestMethod']);
$this->assertStringContainsString('body-is-custom-body', $execution['body']['logs']);
$this->assertStringContainsString('custom-header-is-custom-value', $execution['body']['logs']);
$this->assertStringContainsString('method-is-patch', $execution['body']['logs']);
$this->assertStringContainsString('path-is-/custom-path', $execution['body']['logs']);
$this->assertStringContainsString('user-is-' . $this->getUser()['$id'], $execution['body']['logs']);
$this->assertStringContainsString('jwt-is-valid', $execution['body']['logs']);
$this->assertGreaterThan(0, $execution['body']['duration']);
}, 120000, 500);
@@ -2770,8 +2770,9 @@ class SitesCustomServerTest extends Scope
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertStringContainsString("Inline logs printed.", $response['body']);
// Poll for execution entry to be written (async)
// Poll for execution logs to be written (async)
// Filter by requestPath to avoid picking up screenshot worker executions
// Wait for both the execution entry AND its logs field to be populated
$logs = null;
$timeout = 120;
$start = \time();
@@ -2781,16 +2782,21 @@ class SitesCustomServerTest extends Scope
Query::equal('requestPath', ['/logs-inline'])->toString(),
Query::limit(1)->toString(),
]);
if (!empty($logs['body']['executions'])) {
if (!empty($logs['body']['executions']) && !empty($logs['body']['executions'][0]['logs'])) {
break;
}
\usleep(500000);
}
$this->assertNotEmpty($logs['body']['executions'], 'Execution logs were not available within timeout');
$this->assertNotNull($logs['body']['executions'][0]['logs'], 'Execution logs content was not populated within timeout');
$this->assertEquals(200, $logs['headers']['status-code']);
$this->assertStringContainsString($deploymentId, $logs['body']['executions'][0]['deploymentId']);
$this->assertStringContainsString("GET", $logs['body']['executions'][0]['requestMethod']);
$this->assertStringContainsString("/logs-inline", $logs['body']['executions'][0]['requestPath']);
$this->assertStringContainsString("Log1", $logs['body']['executions'][0]['logs']);
$this->assertStringContainsString("Log2", $logs['body']['executions'][0]['logs']);
$this->assertStringContainsString("Error1", $logs['body']['executions'][0]['errors']);
$this->assertStringContainsString("Error2", $logs['body']['executions'][0]['errors']);
$log1Id = $logs['body']['executions'][0]['$id'];
$this->assertNotEmpty($log1Id);
@@ -2824,16 +2830,21 @@ class SitesCustomServerTest extends Scope
Query::equal('requestPath', ['/logs-action'])->toString(),
Query::limit(1)->toString(),
]);
if (!empty($logs['body']['executions'])) {
if (!empty($logs['body']['executions']) && !empty($logs['body']['executions'][0]['logs'])) {
break;
}
\usleep(500000);
}
$this->assertNotEmpty($logs['body']['executions'], 'Action execution logs were not available within timeout');
$this->assertNotNull($logs['body']['executions'][0]['logs'], 'Action execution logs content was not populated within timeout');
$this->assertEquals(200, $logs['headers']['status-code']);
$this->assertStringContainsString($deploymentId, $logs['body']['executions'][0]['deploymentId']);
$this->assertStringContainsString("GET", $logs['body']['executions'][0]['requestMethod']);
$this->assertStringContainsString("/logs-action", $logs['body']['executions'][0]['requestPath']);
$this->assertStringContainsString("Log1", $logs['body']['executions'][0]['logs']);
$this->assertStringContainsString("Log2", $logs['body']['executions'][0]['logs']);
$this->assertStringContainsString("Error1", $logs['body']['executions'][0]['errors']);
$this->assertStringContainsString("Error2", $logs['body']['executions'][0]['errors']);
$log2Id = $logs['body']['executions'][0]['$id'];
$this->assertNotEmpty($log2Id);
+21
View File
@@ -216,6 +216,27 @@ services:
- _APP_DB_USER
- _APP_DB_PASS
appwrite-worker-executions:
entrypoint: worker-executions
container_name: appwrite-worker-executions
build:
context: .
restart: unless-stopped
networks:
- appwrite
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
appwrite-worker-functions:
entrypoint: worker-functions
container_name: appwrite-worker-functions
+1 -1
View File
@@ -23,7 +23,7 @@ class ComposeTest extends TestCase
public function testServices(): void
{
$this->assertCount(15, $this->object->getServices());
$this->assertCount(16, $this->object->getServices());
$this->assertEquals('appwrite', $this->object->getService('appwrite')->getContainerName());
$this->assertEquals('', $this->object->getService('appwrite')->getImageVersion());
$this->assertEquals('3.6', $this->object->getService('traefik')->getImageVersion());