diff --git a/app/init/models.php b/app/init/models.php index b935136a63..826408c075 100644 --- a/app/init/models.php +++ b/app/init/models.php @@ -73,6 +73,7 @@ use Appwrite\Utopia\Response\Model\Framework; use Appwrite\Utopia\Response\Model\FrameworkAdapter; use Appwrite\Utopia\Response\Model\Func; use Appwrite\Utopia\Response\Model\Headers; +use Appwrite\Utopia\Response\Model\HttpLog; use Appwrite\Utopia\Response\Model\HealthAntivirus; use Appwrite\Utopia\Response\Model\HealthCertificate; use Appwrite\Utopia\Response\Model\HealthQueue; @@ -183,6 +184,7 @@ Response::setModel(new BaseList('Frameworks List', Response::MODEL_FRAMEWORK_LIS Response::setModel(new BaseList('Runtimes List', Response::MODEL_RUNTIME_LIST, 'runtimes', Response::MODEL_RUNTIME)); Response::setModel(new BaseList('Deployments List', Response::MODEL_DEPLOYMENT_LIST, 'deployments', Response::MODEL_DEPLOYMENT)); Response::setModel(new BaseList('Executions List', Response::MODEL_EXECUTION_LIST, 'executions', Response::MODEL_EXECUTION)); +Response::setModel(new BaseList('HTTP Logs List', Response::MODEL_HTTP_LOG_LIST, 'httpLogs', Response::MODEL_HTTP_LOG)); Response::setModel(new BaseList('Projects List', Response::MODEL_PROJECT_LIST, 'projects', Response::MODEL_PROJECT, true, false)); Response::setModel(new BaseList('Webhooks List', Response::MODEL_WEBHOOK_LIST, 'webhooks', Response::MODEL_WEBHOOK, true, false)); Response::setModel(new BaseList('API Keys List', Response::MODEL_KEY_LIST, 'keys', Response::MODEL_KEY, true, true)); @@ -303,6 +305,7 @@ Response::setModel(new Framework()); Response::setModel(new FrameworkAdapter()); Response::setModel(new Deployment()); Response::setModel(new Execution()); +Response::setModel(new HttpLog()); Response::setModel(new Project()); Response::setModel(new Webhook()); Response::setModel(new Key()); diff --git a/app/init/resources.php b/app/init/resources.php index a031292ca7..05beccfbde 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -974,6 +974,10 @@ Http::setResource('redis', function () { return $redis; }); +Http::setResource('logs', function (\Redis $redis) { + return new \Appwrite\Logs\Redis($redis); +}, ['redis']); + Http::setResource('timelimit', function (\Redis $redis) { return function (string $key, int $limit, int $time) use ($redis) { return new TimeLimitRedis($key, $limit, $time, $redis); diff --git a/app/worker.php b/app/worker.php index 49635b0381..0e5c4e3206 100644 --- a/app/worker.php +++ b/app/worker.php @@ -266,6 +266,10 @@ Server::setResource('redis', function () { return $redis; }); +Server::setResource('logs', function (\Redis $redis) { + return new \Appwrite\Logs\Redis($redis); +}, ['redis']); + Server::setResource('timelimit', function (\Redis $redis) { return function (string $key, int $limit, int $time) use ($redis) { return new TimeLimitRedis($key, $limit, $time, $redis); diff --git a/phpunit.xml b/phpunit.xml index a8578995c1..349dbf84c3 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -34,6 +34,7 @@ ./tests/e2e/Services/Tokens ./tests/e2e/Services/Webhooks ./tests/e2e/Services/Messaging + ./tests/e2e/Services/Logs ./tests/e2e/Services/Migrations ./tests/e2e/Services/Functions/FunctionsBase.php ./tests/e2e/Services/Functions/FunctionsCustomServerTest.php diff --git a/src/Appwrite/Logs.php b/src/Appwrite/Logs.php new file mode 100644 index 0000000000..8e8b14c2f2 --- /dev/null +++ b/src/Appwrite/Logs.php @@ -0,0 +1,30 @@ + $this->resource->value, + 'resourceId' => $this->resourceId, + 'timestamp' => $this->timestamp, + 'durationSeconds' => $this->durationSeconds, + 'requestMethod' => $this->requestMethod->value, + 'requestScheme' => $this->requestScheme, + 'requestHost' => $this->requestHost, + 'requestPath' => $this->requestPath, + 'requestQuery' => $this->requestQuery, + 'requestSizeBytes' => $this->requestSizeBytes, + 'responseStatusCode' => $this->responseStatusCode, + 'responseSizeBytes' => $this->responseSizeBytes, + ]; + } + + public static function fromArray(array $data): self + { + return new self( + resource: Resource::from($data['resource']), + resourceId: $data['resourceId'], + timestamp: (float) $data['timestamp'], + durationSeconds: (float) $data['durationSeconds'], + requestMethod: Method::tryFrom($data['requestMethod']) ?? Method::Other, + requestScheme: $data['requestScheme'] ?? '', + requestHost: $data['requestHost'] ?? '', + requestPath: $data['requestPath'] ?? '', + requestQuery: $data['requestQuery'] ?? '', + requestSizeBytes: (int) ($data['requestSizeBytes'] ?? 0), + responseStatusCode: (int) ($data['responseStatusCode'] ?? 0), + responseSizeBytes: (int) ($data['responseSizeBytes'] ?? 0), + ); + } +} diff --git a/src/Appwrite/Logs/Method.php b/src/Appwrite/Logs/Method.php new file mode 100644 index 0000000000..18454ec3d6 --- /dev/null +++ b/src/Appwrite/Logs/Method.php @@ -0,0 +1,17 @@ +resource, $log->resourceId); + + $this->redis->hMSet($key, [ + 'resource' => $log->resource->value, + 'resourceId' => $log->resourceId, + 'timestamp' => $log->timestamp, + 'durationSeconds' => $log->durationSeconds, + 'requestMethod' => $log->requestMethod->value, + 'requestScheme' => $log->requestScheme, + 'requestHost' => $log->requestHost, + 'requestPath' => $log->requestPath, + 'requestQuery' => $log->requestQuery, + 'requestSizeBytes' => $log->requestSizeBytes, + 'responseStatusCode' => $log->responseStatusCode, + 'responseSizeBytes' => $log->responseSizeBytes, + ]); + + $this->redis->expire($key, self::TTL_SECONDS); + + $this->redis->zAdd($indexKey, $log->timestamp, $id); + $this->redis->expire($indexKey, self::TTL_SECONDS); + } + + public function get(string $id): ?Log + { + $key = self::PREFIX_LOG . $id; + + $data = $this->redis->hGetAll($key); + + if (empty($data)) { + return null; + } + + return self::toLog($data); + } + + /** + * @return array + */ + public function list( + Resource $resource, + string $resourceId, + int $limit = 100, + int $offset = 0, + ): array { + $indexKey = self::indexKey($resource, $resourceId); + + $ids = $this->redis->zRevRange($indexKey, $offset, $offset + $limit - 1); + + if (empty($ids)) { + return []; + } + + $results = []; + + foreach ($ids as $id) { + $data = $this->redis->hGetAll(self::PREFIX_LOG . $id); + + if (!empty($data)) { + $results[$id] = self::toLog($data); + } + } + + return $results; + } + + public function count( + Resource $resource, + string $resourceId, + ): int { + $indexKey = self::indexKey($resource, $resourceId); + + return (int) $this->redis->zCard($indexKey); + } + + public function delete(string $id): void + { + $key = self::PREFIX_LOG . $id; + + $data = $this->redis->hGetAll($key); + + if (!empty($data)) { + $indexKey = self::indexKey( + Resource::from($data['resource']), + $data['resourceId'], + ); + + $this->redis->zRem($indexKey, $id); + } + + $this->redis->del($key); + } + + private static function indexKey(Resource $resource, string $resourceId): string + { + return self::PREFIX_INDEX . $resource->value . ':' . $resourceId; + } + + private static function toLog(array $data): Log + { + return new Log( + resource: Resource::from($data['resource']), + resourceId: $data['resourceId'], + timestamp: (float) $data['timestamp'], + durationSeconds: (float) $data['durationSeconds'], + requestMethod: Method::from($data['requestMethod']), + requestScheme: $data['requestScheme'], + requestHost: $data['requestHost'], + requestPath: $data['requestPath'], + requestQuery: $data['requestQuery'], + requestSizeBytes: (int) $data['requestSizeBytes'], + responseStatusCode: (int) $data['responseStatusCode'], + responseSizeBytes: (int) $data['responseSizeBytes'], + ); + } +} diff --git a/src/Appwrite/Logs/Resource.php b/src/Appwrite/Logs/Resource.php new file mode 100644 index 0000000000..7451e10dc7 --- /dev/null +++ b/src/Appwrite/Logs/Resource.php @@ -0,0 +1,9 @@ +addModule(new Tokens\Module()); $this->addModule(new Storage\Module()); $this->addModule(new VCS\Module()); + $this->addModule(new Logs\Module()); } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Delete.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Delete.php index c7a9a6d330..33cda0fd0d 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Delete.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Delete.php @@ -4,6 +4,8 @@ namespace Appwrite\Platform\Modules\Functions\Http\Executions; use Appwrite\Event\Event; use Appwrite\Extend\Exception; +use Appwrite\Logs; +use Appwrite\Logs\Resource; use Appwrite\Platform\Modules\Compute\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -17,6 +19,7 @@ use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; +use Utopia\System\System; class Delete extends Base { @@ -62,6 +65,7 @@ class Delete extends Base ->inject('dbForPlatform') ->inject('queueForEvents') ->inject('authorization') + ->inject('logs') ->callback($this->action(...)); } @@ -72,7 +76,8 @@ class Delete extends Base Database $dbForProject, Database $dbForPlatform, Event $queueForEvents, - Authorization $authorization + Authorization $authorization, + Logs $logs, ) { $function = $dbForProject->getDocument('functions', $functionId); @@ -80,6 +85,34 @@ class Delete extends Base throw new Exception(Exception::FUNCTION_NOT_FOUND); } + if (System::getEnv('FEATURE_LOGS', 'enabled') === 'enabled') { + $log = $logs->get($executionId); + + if ($log === null) { + throw new Exception(Exception::EXECUTION_NOT_FOUND); + } + + if ($log->resource !== Resource::Deployment) { + throw new Exception(Exception::EXECUTION_NOT_FOUND); + } + + $deployment = $authorization->skip(fn () => $dbForProject->getDocument('deployments', $log->resourceId)); + + if ($deployment->isEmpty() || $deployment->getAttribute('resourceId') !== $functionId) { + throw new Exception(Exception::EXECUTION_NOT_FOUND); + } + + $logs->delete($executionId); + + $queueForEvents + ->setParam('functionId', $function->getId()) + ->setParam('executionId', $executionId); + + $response->noContent(); + + return; + } + $execution = $dbForProject->getDocument('executions', $executionId); if ($execution->isEmpty()) { throw new Exception(Exception::EXECUTION_NOT_FOUND); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Get.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Get.php index c5eebe139e..40471a26f0 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Get.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Get.php @@ -3,6 +3,8 @@ namespace Appwrite\Platform\Modules\Functions\Http\Executions; use Appwrite\Extend\Exception; +use Appwrite\Logs; +use Appwrite\Logs\Resource; use Appwrite\Platform\Modules\Compute\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; @@ -10,10 +12,12 @@ use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response; use Utopia\Database\Database; +use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; +use Utopia\System\System; class Get extends Base { @@ -53,6 +57,7 @@ class Get extends Base ->inject('response') ->inject('dbForProject') ->inject('authorization') + ->inject('logs') ->callback($this->action(...)); } @@ -61,7 +66,8 @@ class Get extends Base string $executionId, Response $response, Database $dbForProject, - Authorization $authorization + Authorization $authorization, + Logs $logs, ) { $function = $authorization->skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -72,6 +78,45 @@ class Get extends Base throw new Exception(Exception::FUNCTION_NOT_FOUND); } + if (System::getEnv('FEATURE_LOGS', 'enabled') === 'enabled') { + $log = $logs->get($executionId); + + if ($log === null) { + throw new Exception(Exception::EXECUTION_NOT_FOUND); + } + + if ($log->resource !== Resource::Deployment) { + throw new Exception(Exception::EXECUTION_NOT_FOUND); + } + + $deployment = $authorization->skip(fn () => $dbForProject->getDocument('deployments', $log->resourceId)); + + if ($deployment->isEmpty() || $deployment->getAttribute('resourceId') !== $functionId) { + throw new Exception(Exception::EXECUTION_NOT_FOUND); + } + + $response->dynamic(new Document([ + '$id' => $executionId, + '$createdAt' => \date('Y-m-d\TH:i:s.vP', (int) $log->timestamp), + '$permissions' => [], + 'functionId' => $functionId, + 'deploymentId' => $log->resourceId, + 'trigger' => 'http', + 'status' => $log->responseStatusCode >= 500 ? 'failed' : 'completed', + 'requestMethod' => $log->requestMethod->value, + 'requestPath' => $log->requestPath, + 'requestHeaders' => [], + 'responseStatusCode' => $log->responseStatusCode, + 'responseBody' => '', + 'responseHeaders' => [], + 'logs' => '', + 'errors' => '', + 'duration' => $log->durationSeconds, + ]), Response::MODEL_EXECUTION); + + return; + } + $execution = $dbForProject->getDocument('executions', $executionId); if ($execution->getAttribute('resourceType') !== 'functions' || $execution->getAttribute('resourceInternalId') !== $function->getSequence()) { diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/XList.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/XList.php index f82207eaee..3699cabfbb 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/XList.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/XList.php @@ -3,6 +3,8 @@ namespace Appwrite\Platform\Modules\Functions\Http\Executions; use Appwrite\Extend\Exception; +use Appwrite\Logs; +use Appwrite\Logs\Resource; use Appwrite\Platform\Modules\Compute\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; @@ -20,6 +22,7 @@ use Utopia\Database\Validator\Query\Cursor; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; +use Utopia\System\System; use Utopia\Validator\Boolean; class XList extends Base @@ -61,6 +64,7 @@ class XList extends Base ->inject('response') ->inject('dbForProject') ->inject('authorization') + ->inject('logs') ->callback($this->action(...)); } @@ -70,7 +74,8 @@ class XList extends Base bool $includeTotal, Response $response, Database $dbForProject, - Authorization $authorization + Authorization $authorization, + Logs $logs, ) { $function = $authorization->skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -81,6 +86,69 @@ class XList extends Base throw new Exception(Exception::FUNCTION_NOT_FOUND); } + if (System::getEnv('FEATURE_LOGS', 'enabled') === 'enabled') { + $deploymentId = $function->getAttribute('deploymentId', ''); + + if (empty($deploymentId)) { + $response->dynamic(new Document([ + 'executions' => [], + 'total' => 0, + ]), Response::MODEL_EXECUTION_LIST); + + return; + } + + try { + $parsed = Query::parseQueries($queries); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + $limit = Query::getByType($parsed, [Query::TYPE_LIMIT])[0]?->getValue() ?? 25; + $offset = Query::getByType($parsed, [Query::TYPE_OFFSET])[0]?->getValue() ?? 0; + + $results = $logs->list( + resource: Resource::Deployment, + resourceId: $deploymentId, + limit: $limit, + offset: $offset, + ); + + $total = $includeTotal ? $logs->count( + resource: Resource::Deployment, + resourceId: $deploymentId, + ) : 0; + + $executions = []; + foreach ($results as $id => $log) { + $executions[] = new Document([ + '$id' => $id, + '$createdAt' => \date('Y-m-d\TH:i:s.vP', (int) $log->timestamp), + '$permissions' => [], + 'functionId' => $functionId, + 'deploymentId' => $log->resourceId, + 'trigger' => 'http', + 'status' => $log->responseStatusCode >= 500 ? 'failed' : 'completed', + 'requestMethod' => $log->requestMethod->value, + 'requestPath' => $log->requestPath, + 'requestHeaders' => [], + 'responseStatusCode' => $log->responseStatusCode, + 'responseBody' => '', + 'responseHeaders' => [], + 'logs' => '', + 'errors' => '', + 'duration' => $log->durationSeconds, + ]); + } + + $response->dynamic(new Document([ + 'executions' => $executions, + 'total' => $total, + ]), Response::MODEL_EXECUTION_LIST); + + return; + } + try { $queries = Query::parseQueries($queries); } catch (QueryException $e) { diff --git a/src/Appwrite/Platform/Modules/Logs/Http/Delete.php b/src/Appwrite/Platform/Modules/Logs/Http/Delete.php new file mode 100644 index 0000000000..87be847a8e --- /dev/null +++ b/src/Appwrite/Platform/Modules/Logs/Http/Delete.php @@ -0,0 +1,91 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/logs/:logId') + ->desc('Delete HTTP log') + ->groups(['api', 'logs']) + ->label('scope', 'log.write') + ->label('sdk', new Method( + namespace: 'logs', + group: 'logs', + name: 'delete', + description: <<param('logId', '', new UID(), 'Log ID.') + ->inject('response') + ->inject('project') + ->inject('dbForProject') + ->inject('logs') + ->callback($this->action(...)); + } + + public function action( + string $logId, + Response $response, + Document $project, + Database $dbForProject, + Logs $logs, + ) { + $log = $logs->get($logId); + + if ($log === null) { + throw new Exception(Exception::LOG_NOT_FOUND); + } + + switch ($log->resource) { + case Resource::Project: + if ($log->resourceId !== $project->getId()) { + throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN); + } + break; + case Resource::Deployment: + $deployment = $dbForProject->getDocument('deployments', $log->resourceId); + if ($deployment->isEmpty()) { + throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN); + } + break; + } + + $logs->delete($logId); + + $response->noContent(); + } +} diff --git a/src/Appwrite/Platform/Modules/Logs/Http/Get.php b/src/Appwrite/Platform/Modules/Logs/Http/Get.php new file mode 100644 index 0000000000..d9b3b81c21 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Logs/Http/Get.php @@ -0,0 +1,100 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/logs/:logId') + ->desc('Get HTTP log') + ->groups(['api', 'logs']) + ->label('scope', 'log.read') + ->label('sdk', new Method( + namespace: 'logs', + group: 'logs', + name: 'get', + description: <<param('logId', '', new UID(), 'Log ID.') + ->inject('response') + ->inject('project') + ->inject('dbForProject') + ->inject('logs') + ->callback($this->action(...)); + } + + public function action( + string $logId, + Response $response, + Document $project, + Database $dbForProject, + Logs $logs, + ) { + $log = $logs->get($logId); + + if ($log === null) { + throw new Exception(Exception::LOG_NOT_FOUND); + } + + switch ($log->resource) { + case Resource::Project: + if ($log->resourceId !== $project->getId()) { + throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN); + } + break; + case Resource::Deployment: + $deployment = $dbForProject->getDocument('deployments', $log->resourceId); + if ($deployment->isEmpty()) { + throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN); + } + break; + } + + $response->dynamic(new Document([ + '$id' => $logId, + 'resource' => $log->resource->value, + 'resourceId' => $log->resourceId, + 'durationSeconds' => $log->durationSeconds, + 'requestMethod' => $log->requestMethod->value, + 'requestScheme' => $log->requestScheme, + 'requestHost' => $log->requestHost, + 'requestPath' => $log->requestPath, + 'requestQuery' => $log->requestQuery, + 'requestSizeBytes' => $log->requestSizeBytes, + 'responseStatusCode' => $log->responseStatusCode, + 'responseSizeBytes' => $log->responseSizeBytes, + ]), Response::MODEL_HTTP_LOG); + } +} diff --git a/src/Appwrite/Platform/Modules/Logs/Http/XList.php b/src/Appwrite/Platform/Modules/Logs/Http/XList.php new file mode 100644 index 0000000000..4bcce62278 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Logs/Http/XList.php @@ -0,0 +1,126 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/logs') + ->desc('List HTTP logs') + ->groups(['api', 'logs']) + ->label('scope', 'log.read') + ->label('sdk', new Method( + namespace: 'logs', + group: 'logs', + name: 'list', + description: <<param('resource', '', new WhiteList([Resource::Project->value, Resource::Deployment->value]), 'Resource type. Possible values: `project`, `deployment`.') + ->param('resourceId', '', new \Utopia\Validator\Text(512), 'Resource ID.') + ->param('limit', 100, new Integer(), 'Maximum number of logs to return. Maximum value is 100.', true) + ->param('offset', 0, new Integer(), 'Offset value. The default value is 0.', true) + ->inject('response') + ->inject('project') + ->inject('dbForProject') + ->inject('logs') + ->callback($this->action(...)); + } + + public function action( + string $resource, + string $resourceId, + int $limit, + int $offset, + Response $response, + Document $project, + Database $dbForProject, + Logs $logs, + ) { + $resourceEnum = Resource::from($resource); + + switch ($resourceEnum) { + case Resource::Project: + if ($resourceId !== $project->getId()) { + throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN); + } + break; + case Resource::Deployment: + $deployment = $dbForProject->getDocument('deployments', $resourceId); + if ($deployment->isEmpty()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + break; + } + + $limit = \min($limit, 100); + $offset = \max($offset, 0); + + $results = $logs->list( + resource: $resourceEnum, + resourceId: $resourceId, + limit: $limit, + offset: $offset, + ); + + $total = $logs->count( + resource: $resourceEnum, + resourceId: $resourceId, + ); + + $documents = []; + foreach ($results as $id => $log) { + $documents[] = new Document([ + '$id' => $id, + 'resource' => $log->resource->value, + 'resourceId' => $log->resourceId, + 'durationSeconds' => $log->durationSeconds, + 'requestMethod' => $log->requestMethod->value, + 'requestScheme' => $log->requestScheme, + 'requestHost' => $log->requestHost, + 'requestPath' => $log->requestPath, + 'requestQuery' => $log->requestQuery, + 'requestSizeBytes' => $log->requestSizeBytes, + 'responseStatusCode' => $log->responseStatusCode, + 'responseSizeBytes' => $log->responseSizeBytes, + ]); + } + + $response->dynamic(new Document([ + 'httpLogs' => $documents, + 'total' => $total, + ]), Response::MODEL_HTTP_LOG_LIST); + } +} diff --git a/src/Appwrite/Platform/Modules/Logs/Module.php b/src/Appwrite/Platform/Modules/Logs/Module.php new file mode 100644 index 0000000000..90523be9d5 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Logs/Module.php @@ -0,0 +1,14 @@ +addService('http', new Http()); + } +} diff --git a/src/Appwrite/Platform/Modules/Logs/Services/Http.php b/src/Appwrite/Platform/Modules/Logs/Services/Http.php new file mode 100644 index 0000000000..568b6db782 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Logs/Services/Http.php @@ -0,0 +1,20 @@ +type = Service::TYPE_HTTP; + + $this->addAction(XList::getName(), new XList()); + $this->addAction(Get::getName(), new Get()); + $this->addAction(Delete::getName(), new Delete()); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Logs/Delete.php b/src/Appwrite/Platform/Modules/Sites/Http/Logs/Delete.php index bcb20cc789..75b1ce8f82 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Logs/Delete.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Logs/Delete.php @@ -4,15 +4,19 @@ namespace Appwrite\Platform\Modules\Sites\Http\Logs; use Appwrite\Event\Event; use Appwrite\Extend\Exception; +use Appwrite\Logs; +use Appwrite\Logs\Resource; use Appwrite\Platform\Modules\Compute\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; use Utopia\Database\Database; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; +use Utopia\System\System; class Delete extends Base { @@ -55,17 +59,54 @@ class Delete extends Base ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') + ->inject('authorization') + ->inject('logs') ->callback($this->action(...)); } - public function action(string $siteId, string $logId, Response $response, Database $dbForProject, Event $queueForEvents) - { + public function action( + string $siteId, + string $logId, + Response $response, + Database $dbForProject, + Event $queueForEvents, + Authorization $authorization, + Logs $logs, + ) { $site = $dbForProject->getDocument('sites', $siteId); if ($site->isEmpty()) { throw new Exception(Exception::SITE_NOT_FOUND); } + if (System::getEnv('FEATURE_LOGS', 'enabled') === 'enabled') { + $log = $logs->get($logId); + + if ($log === null) { + throw new Exception(Exception::LOG_NOT_FOUND); + } + + if ($log->resource !== Resource::Deployment) { + throw new Exception(Exception::LOG_NOT_FOUND); + } + + $deployment = $authorization->skip(fn () => $dbForProject->getDocument('deployments', $log->resourceId)); + + if ($deployment->isEmpty() || $deployment->getAttribute('resourceId') !== $siteId) { + throw new Exception(Exception::LOG_NOT_FOUND); + } + + $logs->delete($logId); + + $queueForEvents + ->setParam('siteId', $site->getId()) + ->setParam('logId', $logId); + + $response->noContent(); + + return; + } + $log = $dbForProject->getDocument('executions', $logId); if ($log->isEmpty()) { throw new Exception(Exception::LOG_NOT_FOUND); @@ -82,7 +123,7 @@ class Delete extends Base $queueForEvents ->setParam('siteId', $site->getId()) ->setParam('logId', $log->getId()) - ->setPayload($response->output($log, Response::MODEL_EXECUTION)); // TODO: Update model + ->setPayload($response->output($log, Response::MODEL_EXECUTION)); $response->noContent(); } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Logs/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Logs/Get.php index 0769cd60cf..55bf06c855 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Logs/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Logs/Get.php @@ -3,15 +3,20 @@ namespace Appwrite\Platform\Modules\Sites\Http\Logs; use Appwrite\Extend\Exception; +use Appwrite\Logs; +use Appwrite\Logs\Resource; use Appwrite\Platform\Modules\Compute\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; +use Utopia\System\System; class Get extends Base { @@ -50,17 +55,64 @@ class Get extends Base ->param('logId', '', new UID(), 'Log ID.') ->inject('response') ->inject('dbForProject') + ->inject('authorization') + ->inject('logs') ->callback($this->action(...)); } - public function action(string $siteId, string $logId, Response $response, Database $dbForProject) - { + public function action( + string $siteId, + string $logId, + Response $response, + Database $dbForProject, + Authorization $authorization, + Logs $logs, + ) { $site = $dbForProject->getDocument('sites', $siteId); if ($site->isEmpty() || !$site->getAttribute('enabled')) { throw new Exception(Exception::SITE_NOT_FOUND); } + if (System::getEnv('FEATURE_LOGS', 'enabled') === 'enabled') { + $log = $logs->get($logId); + + if ($log === null) { + throw new Exception(Exception::LOG_NOT_FOUND); + } + + if ($log->resource !== Resource::Deployment) { + throw new Exception(Exception::LOG_NOT_FOUND); + } + + $deployment = $authorization->skip(fn () => $dbForProject->getDocument('deployments', $log->resourceId)); + + if ($deployment->isEmpty() || $deployment->getAttribute('resourceId') !== $siteId) { + throw new Exception(Exception::LOG_NOT_FOUND); + } + + $response->dynamic(new Document([ + '$id' => $logId, + '$createdAt' => \date('Y-m-d\TH:i:s.vP', (int) $log->timestamp), + '$permissions' => [], + 'resourceId' => $siteId, + 'deploymentId' => $log->resourceId, + 'trigger' => 'http', + 'status' => $log->responseStatusCode >= 500 ? 'failed' : 'completed', + 'requestMethod' => $log->requestMethod->value, + 'requestPath' => $log->requestPath, + 'requestHeaders' => [], + 'responseStatusCode' => $log->responseStatusCode, + 'responseBody' => '', + 'responseHeaders' => [], + 'logs' => '', + 'errors' => '', + 'duration' => $log->durationSeconds, + ]), Response::MODEL_EXECUTION); + + return; + } + $log = $dbForProject->getDocument('executions', $logId); if ($log->getAttribute('resourceType') !== 'sites' && $log->getAttribute('resourceInternalId') !== $site->getSequence()) { @@ -71,6 +123,6 @@ class Get extends Base throw new Exception(Exception::LOG_NOT_FOUND); } - $response->dynamic($log, Response::MODEL_EXECUTION); //TODO: Change to model log, but model log already exists - decide what to do + $response->dynamic($log, Response::MODEL_EXECUTION); } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Logs/XList.php b/src/Appwrite/Platform/Modules/Sites/Http/Logs/XList.php index 38c6d4b29a..691cc07463 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Logs/XList.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Logs/XList.php @@ -3,12 +3,13 @@ namespace Appwrite\Platform\Modules\Sites\Http\Logs; use Appwrite\Extend\Exception; +use Appwrite\Logs; +use Appwrite\Logs\Resource; use Appwrite\Platform\Modules\Compute\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Database\Validator\Queries\Executions; -use Appwrite\Utopia\Database\Validator\Queries\Logs; use Appwrite\Utopia\Response; use Utopia\Database\Database; use Utopia\Database\Document; @@ -19,6 +20,7 @@ use Utopia\Database\Validator\Query\Cursor; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; +use Utopia\System\System; use Utopia\Validator\Boolean; class XList extends Base @@ -55,21 +57,91 @@ class XList extends Base ] )) ->param('siteId', '', new UID(), 'Site ID.') - ->param('queries', [], new Logs(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Executions::ALLOWED_ATTRIBUTES), true) + ->param('queries', [], new \Appwrite\Utopia\Database\Validator\Queries\Logs(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Executions::ALLOWED_ATTRIBUTES), true) ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) ->inject('response') ->inject('dbForProject') + ->inject('logs') ->callback($this->action(...)); } - public function action(string $siteId, array $queries, bool $includeTotal, Response $response, Database $dbForProject) - { + public function action( + string $siteId, + array $queries, + bool $includeTotal, + Response $response, + Database $dbForProject, + Logs $logs, + ) { $site = $dbForProject->getDocument('sites', $siteId); if ($site->isEmpty() || !$site->getAttribute('enabled')) { throw new Exception(Exception::SITE_NOT_FOUND); } + if (System::getEnv('FEATURE_LOGS', 'enabled') === 'enabled') { + $deploymentId = $site->getAttribute('deploymentId', ''); + + if (empty($deploymentId)) { + $response->dynamic(new Document([ + 'executions' => [], + 'total' => 0, + ]), Response::MODEL_EXECUTION_LIST); + + return; + } + + try { + $parsed = Query::parseQueries($queries); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + $limit = Query::getByType($parsed, [Query::TYPE_LIMIT])[0]?->getValue() ?? 25; + $offset = Query::getByType($parsed, [Query::TYPE_OFFSET])[0]?->getValue() ?? 0; + + $results = $logs->list( + resource: Resource::Deployment, + resourceId: $deploymentId, + limit: $limit, + offset: $offset, + ); + + $total = $includeTotal ? $logs->count( + resource: Resource::Deployment, + resourceId: $deploymentId, + ) : 0; + + $executions = []; + foreach ($results as $id => $log) { + $executions[] = new Document([ + '$id' => $id, + '$createdAt' => \date('Y-m-d\TH:i:s.vP', (int) $log->timestamp), + '$permissions' => [], + 'resourceId' => $siteId, + 'deploymentId' => $log->resourceId, + 'trigger' => 'http', + 'status' => $log->responseStatusCode >= 500 ? 'failed' : 'completed', + 'requestMethod' => $log->requestMethod->value, + 'requestPath' => $log->requestPath, + 'requestHeaders' => [], + 'responseStatusCode' => $log->responseStatusCode, + 'responseBody' => '', + 'responseHeaders' => [], + 'logs' => '', + 'errors' => '', + 'duration' => $log->durationSeconds, + ]); + } + + $response->dynamic(new Document([ + 'executions' => $executions, + 'total' => $total, + ]), Response::MODEL_EXECUTION_LIST); + + return; + } + try { $queries = Query::parseQueries($queries); } catch (QueryException $e) { diff --git a/src/Appwrite/Platform/Workers/Executions.php b/src/Appwrite/Platform/Workers/Executions.php index 300a84162c..c948c4bbe1 100644 --- a/src/Appwrite/Platform/Workers/Executions.php +++ b/src/Appwrite/Platform/Workers/Executions.php @@ -2,6 +2,10 @@ namespace Appwrite\Platform\Workers; +use Appwrite\Logs; +use Appwrite\Logs\Log; +use Appwrite\Logs\Resource; +use Appwrite\Logs\Method; use Exception; use Utopia\Database\Database; use Utopia\Database\Document; @@ -26,12 +30,14 @@ class Executions extends Action ->groups(['executions']) ->inject('message') ->inject('dbForProject') + ->inject('logs') ->callback($this->action(...)); } public function action( Message $message, Database $dbForProject, + Logs $logs ): void { $payload = $message->getPayload() ?? []; @@ -45,7 +51,29 @@ class Executions extends Action throw new Exception('Missing execution'); } - if (System::getEnv('_APP_REGION') !== 'nyc') { // TODO: Remove region check + if (System::getEnv('FEATURE_LOGS', 'enabled') === 'enabled') { + $logs->append(new Log( + // Meta + resource: Resource::Deployment, + resourceId: $execution->getAttribute('deploymentId'), + timestamp: microtime(true), + durationSeconds: $execution->getAttribute('duration'), + + // Request + requestMethod: Method::tryFrom($execution->getAttribute('requestMethod')), + requestScheme: '', + requestHost: '', + requestPath: $execution->getAttribute('requestPath'), + requestQuery: '', + requestSizeBytes: 0, + + // Response + responseStatusCode: $execution->getAttribute('responseStatusCode'), + responseSizeBytes: 0, + )); + + + } else { $dbForProject->upsertDocument('executions', $execution); } } diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 9892fd5f78..adca723a2c 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -210,6 +210,8 @@ class Response extends SwooleResponse public const MODEL_DEPLOYMENT_LIST = 'deploymentList'; public const MODEL_EXECUTION = 'execution'; public const MODEL_EXECUTION_LIST = 'executionList'; + public const MODEL_HTTP_LOG = 'httpLog'; + public const MODEL_HTTP_LOG_LIST = 'httpLogList'; public const MODEL_FUNC_PERMISSIONS = 'funcPermissions'; public const MODEL_HEADERS = 'headers'; public const MODEL_SPECIFICATION = 'specification'; diff --git a/src/Appwrite/Utopia/Response/Model/HttpLog.php b/src/Appwrite/Utopia/Response/Model/HttpLog.php new file mode 100644 index 0000000000..4972631b07 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/HttpLog.php @@ -0,0 +1,96 @@ +addRule('$id', [ + 'type' => self::TYPE_STRING, + 'description' => 'Log ID.', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) + ->addRule('resource', [ + 'type' => self::TYPE_STRING, + 'description' => 'Resource type. Possible values: `project`, `deployment`.', + 'default' => '', + 'example' => 'deployment', + ]) + ->addRule('resourceId', [ + 'type' => self::TYPE_STRING, + 'description' => 'Resource ID.', + 'default' => '', + 'example' => '5e5ea6g16897e', + ]) + ->addRule('durationSeconds', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Request duration in seconds.', + 'default' => 0, + 'example' => 1, + ]) + ->addRule('requestMethod', [ + 'type' => self::TYPE_STRING, + 'description' => 'HTTP request method.', + 'default' => '', + 'example' => 'GET', + ]) + ->addRule('requestScheme', [ + 'type' => self::TYPE_STRING, + 'description' => 'HTTP request scheme.', + 'default' => '', + 'example' => 'https', + ]) + ->addRule('requestHost', [ + 'type' => self::TYPE_STRING, + 'description' => 'HTTP request host.', + 'default' => '', + 'example' => 'example.com', + ]) + ->addRule('requestPath', [ + 'type' => self::TYPE_STRING, + 'description' => 'HTTP request path.', + 'default' => '', + 'example' => '/articles', + ]) + ->addRule('requestQuery', [ + 'type' => self::TYPE_STRING, + 'description' => 'HTTP request query string.', + 'default' => '', + 'example' => 'id=5', + ]) + ->addRule('requestSizeBytes', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'HTTP request size in bytes.', + 'default' => 0, + 'example' => 256, + ]) + ->addRule('responseStatusCode', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'HTTP response status code.', + 'default' => 0, + 'example' => 200, + ]) + ->addRule('responseSizeBytes', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'HTTP response size in bytes.', + 'default' => 0, + 'example' => 1024, + ]); + } + + public function getName(): string + { + return 'HttpLog'; + } + + public function getType(): string + { + return Response::MODEL_HTTP_LOG; + } +} diff --git a/tests/e2e/Services/Logs/LogsConsoleClientTest.php b/tests/e2e/Services/Logs/LogsConsoleClientTest.php new file mode 100644 index 0000000000..85b4748ff5 --- /dev/null +++ b/tests/e2e/Services/Logs/LogsConsoleClientTest.php @@ -0,0 +1,154 @@ +getProject()['$id']; + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()), [ + 'resource' => 'project', + 'resourceId' => $projectId, + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertIsArray($response['body']['httpLogs']); + $this->assertIsInt($response['body']['total']); + + /** + * Test for FAILURE - wrong project ID + */ + $response = $this->client->call(Client::METHOD_GET, '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()), [ + 'resource' => 'project', + 'resourceId' => 'invalid-project-id', + ]); + + $this->assertEquals(403, $response['headers']['status-code']); + + /** + * Test for FAILURE - missing resource + */ + $response = $this->client->call(Client::METHOD_GET, '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()), [ + 'resourceId' => $projectId, + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + /** + * Test for FAILURE - missing resourceId + */ + $response = $this->client->call(Client::METHOD_GET, '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()), [ + 'resource' => 'project', + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + /** + * Test for FAILURE - invalid resource type + */ + $response = $this->client->call(Client::METHOD_GET, '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()), [ + 'resource' => 'invalid', + 'resourceId' => $projectId, + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + } + + public function testListLogsByDeployment(): void + { + $projectId = $this->getProject()['$id']; + + /** + * Test for FAILURE - deployment not found + */ + $response = $this->client->call(Client::METHOD_GET, '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()), [ + 'resource' => 'deployment', + 'resourceId' => 'nonexistent-deployment', + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + } + + public function testListLogsPagination(): void + { + $projectId = $this->getProject()['$id']; + + /** + * Test for SUCCESS - custom limit and offset + */ + $response = $this->client->call(Client::METHOD_GET, '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()), [ + 'resource' => 'project', + 'resourceId' => $projectId, + 'limit' => 10, + 'offset' => 0, + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertIsArray($response['body']['httpLogs']); + $this->assertLessThanOrEqual(10, count($response['body']['httpLogs'])); + } + + public function testGetLog(): void + { + $projectId = $this->getProject()['$id']; + + /** + * Test for FAILURE - log not found + */ + $response = $this->client->call(Client::METHOD_GET, '/logs/nonexistent-log-id', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders())); + + $this->assertEquals(404, $response['headers']['status-code']); + } + + public function testDeleteLog(): void + { + $projectId = $this->getProject()['$id']; + + /** + * Test for FAILURE - log not found + */ + $response = $this->client->call(Client::METHOD_DELETE, '/logs/nonexistent-log-id', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders())); + + $this->assertEquals(404, $response['headers']['status-code']); + } +}