From 9faf165613b4418b606a6bfa389a1fbe7f4dd7d2 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 7 Sep 2026 19:01:50 +0000 Subject: [PATCH] feat: add S3-compatible storage service --- app/init/resources/request.php | 20 +- src/Appwrite/Platform/Appwrite.php | 2 + .../Platform/Modules/S3/Auth/SignatureV4.php | 337 ++++++ .../Platform/Modules/S3/Http/Init.php | 109 ++ .../Platform/Modules/S3/Http/S3/Base.php | 1015 +++++++++++++++++ .../Platform/Modules/S3/Http/S3/Create.php | 225 ++++ .../Platform/Modules/S3/Http/S3/Delete.php | 104 ++ .../Platform/Modules/S3/Http/S3/Get.php | 311 +++++ .../Modules/S3/Http/Shutdown/Events.php | 103 ++ .../Modules/S3/Http/Shutdown/Usage.php | 77 ++ src/Appwrite/Platform/Modules/S3/Module.php | 14 + .../Modules/S3/Requests/AwsChunked.php | 128 +++ .../Platform/Modules/S3/Responses/S3Xml.php | 360 ++++++ .../Platform/Modules/S3/Services/Http.php | 26 + .../Modules/S3/Auth/SignatureV4Test.php | 324 ++++++ .../Modules/S3/Requests/AwsChunkedTest.php | 105 ++ .../Modules/S3/Responses/S3XmlTest.php | 166 +++ 17 files changed, 3425 insertions(+), 1 deletion(-) create mode 100644 src/Appwrite/Platform/Modules/S3/Auth/SignatureV4.php create mode 100644 src/Appwrite/Platform/Modules/S3/Http/Init.php create mode 100644 src/Appwrite/Platform/Modules/S3/Http/S3/Base.php create mode 100644 src/Appwrite/Platform/Modules/S3/Http/S3/Create.php create mode 100644 src/Appwrite/Platform/Modules/S3/Http/S3/Delete.php create mode 100644 src/Appwrite/Platform/Modules/S3/Http/S3/Get.php create mode 100644 src/Appwrite/Platform/Modules/S3/Http/Shutdown/Events.php create mode 100644 src/Appwrite/Platform/Modules/S3/Http/Shutdown/Usage.php create mode 100644 src/Appwrite/Platform/Modules/S3/Module.php create mode 100644 src/Appwrite/Platform/Modules/S3/Requests/AwsChunked.php create mode 100644 src/Appwrite/Platform/Modules/S3/Responses/S3Xml.php create mode 100644 src/Appwrite/Platform/Modules/S3/Services/Http.php create mode 100644 tests/unit/Platform/Modules/S3/Auth/SignatureV4Test.php create mode 100644 tests/unit/Platform/Modules/S3/Requests/AwsChunkedTest.php create mode 100644 tests/unit/Platform/Modules/S3/Responses/S3XmlTest.php diff --git a/app/init/resources/request.php b/app/init/resources/request.php index d161832a57..fc9337cf5b 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -621,11 +621,29 @@ return function (Container $context): void { $projectId = (string) $request->getQuery('project', ''); } + $route = $utopia->match($request)?->route; + + // S3 uses the Appwrite project ID as its SigV4 access key. Header-signed + // requests carry the credential scope in Authorization; presigned URLs + // carry it in the X-Amz-Credential query parameter. + if ($projectId === '' && \in_array('s3', $route?->getGroups() ?? [], true)) { + $credential = ''; + $authorizationHeader = $request->getHeaderLine('authorization', ''); + if (\preg_match('/Credential=([^\s,]+)/', $authorizationHeader, $matches) === 1) { + $credential = $matches[1]; + } else { + $credential = $request->getQuery('X-Amz-Credential', ''); + } + + if (\is_string($credential)) { + $projectId = \explode('/', $credential, 2)[0]; + } + } + // Backwards compatibility for new services, originally project resources // These endpoints moved from /v1/projects/:projectId/ to /v1/ // When accessed via the old alias path, extract projectId from the URI $deprecatedProjectPathPrefix = '/v1/projects/'; - $route = $utopia->match($request)?->route; if (!empty($route)) { $isDeprecatedAlias = $projectIdFromPath !== '' && !\str_starts_with($route->getPath(), $deprecatedProjectPathPrefix); diff --git a/src/Appwrite/Platform/Appwrite.php b/src/Appwrite/Platform/Appwrite.php index b2037b6ebc..ad5a9a9a56 100644 --- a/src/Appwrite/Platform/Appwrite.php +++ b/src/Appwrite/Platform/Appwrite.php @@ -17,6 +17,7 @@ use Appwrite\Platform\Modules\Presences; use Appwrite\Platform\Modules\Project; use Appwrite\Platform\Modules\Projects; use Appwrite\Platform\Modules\Proxy; +use Appwrite\Platform\Modules\S3; use Appwrite\Platform\Modules\Sites; use Appwrite\Platform\Modules\Storage; use Appwrite\Platform\Modules\Teams; @@ -43,6 +44,7 @@ class Appwrite extends Platform $this->addModule(new Sites\Module()); $this->addModule(new Console\Module()); $this->addModule(new Proxy\Module()); + $this->addModule(new S3\Module()); $this->addModule(new Teams\Module()); $this->addModule(new Tokens\Module()); $this->addModule(new Users\Module()); diff --git a/src/Appwrite/Platform/Modules/S3/Auth/SignatureV4.php b/src/Appwrite/Platform/Modules/S3/Auth/SignatureV4.php new file mode 100644 index 0000000000..4dafa42e61 --- /dev/null +++ b/src/Appwrite/Platform/Modules/S3/Auth/SignatureV4.php @@ -0,0 +1,337 @@ +getHeaderLine('authorization'); + $queryAuthentication = $this->queryAuthentication($request); + + if ($authorization !== '' && $queryAuthentication !== null) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Multiple AWS Signature V4 authentication methods were provided.'); + } + + if ($queryAuthentication !== null) { + $algorithm = $queryAuthentication['X-Amz-Algorithm'] ?? ''; + if ($algorithm !== 'AWS4-HMAC-SHA256') { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Unsupported AWS Signature V4 algorithm.'); + } + + $credential = \explode('/', $queryAuthentication['X-Amz-Credential'] ?? ''); + $signedHeaders = \explode(';', $queryAuthentication['X-Amz-SignedHeaders'] ?? ''); + $signature = $queryAuthentication['X-Amz-Signature'] ?? ''; + $date = $queryAuthentication['X-Amz-Date'] ?? ''; + $expires = $queryAuthentication['X-Amz-Expires'] ?? ''; + $presigned = true; + } else { + if (!\str_starts_with($authorization, 'AWS4-HMAC-SHA256 ')) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Missing AWS Signature V4 authentication.'); + } + + $parts = $this->parseAuthorization($authorization); + $credential = \explode('/', $parts['Credential'] ?? ''); + $signedHeaders = \explode(';', $parts['SignedHeaders'] ?? ''); + $signature = $parts['Signature'] ?? ''; + $date = $request->getHeaderLine('x-amz-date') ?: $request->getHeaderLine('date'); + $expires = null; + $presigned = false; + } + + if ( + \count($credential) !== 5 + || $credential[1] === '' + || $credential[2] === '' + || $credential[3] !== 's3' + || $credential[4] !== 'aws4_request' + ) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid AWS Signature V4 credential scope.'); + } + + $projectId = $credential[0]; + + if ($projectId === '') { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Missing AWS credential project ID.'); + } + + if ($projectId !== $project->getId()) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS credential project ID does not belong to the selected project.'); + } + + $normalizedSignedHeaders = \array_values(\array_filter(\array_map( + fn (string $header): string => \strtolower(\trim($header)), + $signedHeaders + ))); + $sortedSignedHeaders = $normalizedSignedHeaders; + \sort($sortedSignedHeaders, SORT_STRING); + if ( + $signature === '' + || $date === '' + || ($presigned && \preg_match('/^[a-f0-9]{64}$/D', $signature) !== 1) + || ($presigned && !\in_array('host', $normalizedSignedHeaders, true)) + || ($presigned && $normalizedSignedHeaders !== $sortedSignedHeaders) + ) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Incomplete AWS Signature V4 authentication.'); + } + + if ($presigned) { + $this->validatePresignedDate($date, $credential[1], (string) $expires); + } else { + $this->validateDate($date, $credential[1]); + } + + $canonicalRequest = $this->canonicalRequest($request, $normalizedSignedHeaders, $presigned); + $scope = \implode('/', \array_slice($credential, 1, 4)); + $stringToSign = "AWS4-HMAC-SHA256\n{$date}\n{$scope}\n" . \hash('sha256', $canonicalRequest); + + foreach ($this->keySecrets($project) as $secret) { + $signingKey = $this->signingKey($secret, $credential[1], $credential[2], $credential[3]); + $expected = \hash_hmac('sha256', $stringToSign, $signingKey); + + if (!\hash_equals($expected, $signature)) { + continue; + } + + $apiKey = Key::decode($project, $team, $user, $secret); + if ($apiKey->getRole() !== User::ROLE_KEYS || $apiKey->isExpired()) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid or expired API key.'); + } + + if (empty(\array_intersect($requiredScopes, $apiKey->getScopes()))) { + throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, 'API key is missing required S3 storage scopes.'); + } + + return $apiKey; + } + + throw new Exception(Exception::USER_UNAUTHORIZED, 'Signature does not match.'); + } + + private function validatePresignedDate(string $date, string $credentialDate, string $expires): void + { + if ($expires === '' || !\ctype_digit($expires)) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid AWS Signature V4 presigned URL expiry.'); + } + + $expires = (int) $expires; + if ($expires < 1 || $expires > self::MAX_PRESIGNED_EXPIRY) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 presigned URL expiry is outside the allowed range.'); + } + + $timestamp = $this->timestamp($date); + if ($timestamp->format('Ymd') !== $credentialDate) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 credential date does not match request timestamp.'); + } + + $now = \time(); + if ($timestamp->getTimestamp() - $now > self::MAX_CLOCK_SKEW) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 timestamp is outside the allowed time window.'); + } + if ($now > $timestamp->getTimestamp() + $expires) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 presigned URL has expired.'); + } + } + + private function validateDate(string $date, string $credentialDate): void + { + $timestamp = \DateTimeImmutable::createFromFormat('!Ymd\THis\Z', $date, new \DateTimeZone('UTC')); + if ($timestamp === false) { + $parsed = \strtotime($date); + if ($parsed === false) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid AWS Signature V4 timestamp.'); + } + + $timestamp = (new \DateTimeImmutable('@' . $parsed))->setTimezone(new \DateTimeZone('UTC')); + } + + if ($timestamp->format('Ymd') !== $credentialDate) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 credential date does not match request timestamp.'); + } + + if (\abs(\time() - $timestamp->getTimestamp()) > self::MAX_CLOCK_SKEW) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 timestamp is outside the allowed time window.'); + } + } + + private function timestamp(string $date): \DateTimeImmutable + { + $timestamp = \DateTimeImmutable::createFromFormat('!Ymd\THis\Z', $date, new \DateTimeZone('UTC')); + $errors = \DateTimeImmutable::getLastErrors(); + if ( + $timestamp === false + || (\is_array($errors) && ($errors['warning_count'] > 0 || $errors['error_count'] > 0)) + || $timestamp->format('Ymd\THis\Z') !== $date + ) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid AWS Signature V4 timestamp.'); + } + + return $timestamp; + } + + /** + * @return array + */ + private function keySecrets(Document $project): array + { + $secrets = []; + foreach ($project->getAttribute('keys', []) as $key) { + $secret = $key instanceof Document + ? $key->getAttribute('secret', '') + : (string) ($key['secret'] ?? ''); + + if ($secret !== '') { + $secrets[] = $secret; + } + } + + return $secrets; + } + + private function parseAuthorization(string $authorization): array + { + $authorization = \substr($authorization, \strlen('AWS4-HMAC-SHA256 ')); + $result = []; + foreach (\explode(',', $authorization) as $part) { + [$key, $value] = \array_pad(\explode('=', \trim($part), 2), 2, ''); + $result[$key] = $value; + } + return $result; + } + + /** + * @return array|null + */ + private function queryAuthentication(Request $request): ?array + { + $authenticationParameters = [ + 'X-Amz-Algorithm', + 'X-Amz-Credential', + 'X-Amz-Date', + 'X-Amz-Expires', + 'X-Amz-Signature', + 'X-Amz-SignedHeaders', + ]; + $parameters = []; + $hasAuthenticationParameter = false; + + foreach ($this->queryPairs($request) as [$name, $value]) { + if (!\in_array($name, $authenticationParameters, true)) { + continue; + } + + $hasAuthenticationParameter = true; + if (\array_key_exists($name, $parameters)) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Duplicate AWS Signature V4 authentication parameter.'); + } + $parameters[$name] = $value; + } + + if (!$hasAuthenticationParameter) { + return null; + } + + foreach ($authenticationParameters as $parameter) { + if (!\array_key_exists($parameter, $parameters)) { + throw new Exception(Exception::USER_UNAUTHORIZED, 'Incomplete AWS Signature V4 query authentication.'); + } + } + + return $parameters; + } + + private function canonicalRequest(Request $request, array $signedHeaders, bool $presigned): string + { + $headers = []; + foreach ($signedHeaders as $header) { + $header = \strtolower(\trim($header)); + if ($header === '') { + continue; + } + $headers[$header] = \preg_replace('/\s+/', ' ', \trim($request->getHeaderLine($header))); + } + \ksort($headers); + + $canonicalHeaders = ''; + foreach ($headers as $key => $value) { + $canonicalHeaders .= $key . ':' . $value . "\n"; + } + + $payloadHash = $presigned ? 'UNSIGNED-PAYLOAD' : $request->getHeaderLine('x-amz-content-sha256'); + $payloadHash = $payloadHash === '' ? \hash('sha256', $request->getRawPayload()) : $payloadHash; + + return \implode("\n", [ + $request->getMethod(), + $this->canonicalUri($request), + $this->canonicalQuery($request, $presigned), + $canonicalHeaders, + \implode(';', \array_keys($headers)), + $payloadHash, + ]); + } + + private function canonicalUri(Request $request): string + { + $path = \parse_url($request->getURI(), PHP_URL_PATH); + return $path === null || $path === false || $path === '' ? '/' : $path; + } + + private function canonicalQuery(Request $request, bool $presigned): string + { + $pairs = []; + foreach ($this->queryPairs($request) as [$key, $value]) { + if ($presigned && $key === 'X-Amz-Signature') { + continue; + } + + $pairs[] = [\rawurlencode($key), \rawurlencode($value)]; + } + + \usort($pairs, function (array $left, array $right): int { + $keyOrder = \strcmp($left[0], $right[0]); + return $keyOrder === 0 ? \strcmp($left[1], $right[1]) : $keyOrder; + }); + + return \implode('&', \array_map(fn (array $pair): string => $pair[0] . '=' . $pair[1], $pairs)); + } + + /** + * @return array + */ + private function queryPairs(Request $request): array + { + $query = $request->getServer('query_string', '') ?: (\parse_url($request->getURI(), PHP_URL_QUERY) ?: ''); + if ($query === '') { + return []; + } + + $pairs = []; + foreach (\explode('&', $query) as $part) { + [$name, $value] = \array_pad(\explode('=', $part, 2), 2, ''); + $pairs[] = [\rawurldecode($name), \rawurldecode($value)]; + } + + return $pairs; + } + + private function signingKey(string $secret, string $date, string $region, string $service): string + { + $kDate = \hash_hmac('sha256', $date, 'AWS4' . $secret, true); + $kRegion = \hash_hmac('sha256', $region, $kDate, true); + $kService = \hash_hmac('sha256', $service, $kRegion, true); + return \hash_hmac('sha256', 'aws4_request', $kService, true); + } +} diff --git a/src/Appwrite/Platform/Modules/S3/Http/Init.php b/src/Appwrite/Platform/Modules/S3/Http/Init.php new file mode 100644 index 0000000000..4290c0f89c --- /dev/null +++ b/src/Appwrite/Platform/Modules/S3/Http/Init.php @@ -0,0 +1,109 @@ +setType(Action::TYPE_INIT) + ->groups(['s3']) + ->desc('S3 gateway request setup') + ->inject('request') + ->inject('response') + ->inject('cors') + ->inject('project') + ->inject('user') + ->inject('queueForEvents') + ->inject('usage') + ->inject('geo') + ->callback($this->action(...)); + } + + public function action(Request $request, Response $response, Cors $cors, Document $project, User $user, Event $queueForEvents, UsageContext $usage, Geo $geo): void + { + $corsHeaders = $cors->headers($request->getOrigin()); + $exposedHeaders = \array_filter(\array_map('trim', \explode(',', (string) ($corsHeaders[Cors::HEADER_EXPOSE_HEADERS] ?? '')))); + $corsHeaders[Cors::HEADER_EXPOSE_HEADERS] = \implode(', ', \array_unique(\array_merge($exposedHeaders, [ + 'Accept-Ranges', + 'Content-Length', + 'Content-Range', + 'ETag', + 'Last-Modified', + 'x-amz-server-side-encryption', + ]))); + foreach ($corsHeaders as $name => $value) { + $response->addHeader($name, $value); + } + + // The S3 actions set the event name and params themselves; project and + // user identify the tenant for event fan-out and for the cache-delete + // messages the module builds from this queue. + $queueForEvents + ->setProject($project) + ->setUser($user); + + // Mirrors the `api` group's "Enrich usage context with request + // metadata" hook in app/controllers/general.php — the gateway meters + // network usage through the same context. + $country = ''; + $geoRecord = $geo->get($request->getIP()); + if (!$geoRecord->isEmpty()) { + $country = \strtolower($geoRecord->getCountryCode()); + } + + $uri = $request->getURI(); + $parts = explode('/', trim($uri, '/')); + $service = count($parts) >= 2 ? $parts[1] : $parts[0]; + + $usage + ->setPath($uri) + ->setMethod($request->getMethod()) + ->setUserAgent($request->getUserAgent('')) + ->setHostname($request->getOrigin('') ?: $request->getHostname()) + ->setCountry($country) + ->setIp($request->getIP()) + ->setSdk(\strtolower($request->getHeaderLine('x-sdk-name', ''))) + ->setSdkVersion($request->getHeaderLine('x-sdk-version', '')) + ->setRegion(System::getEnv('_APP_REGION', 'default')) + ->setService($service); + + if (!$project->isEmpty()) { + $usage + ->setTeamId((string) $project->getAttribute('teamId', '')) + ->setTeamInternalId((string) $project->getAttribute('teamInternalId', '')); + } + + // Always reset — clears values carried over from a previous request + // when this Context instance is reused across requests in the same worker. + $usage->setResourceType(''); + $usage->setResourceId(''); + $usage->setResourceInternalId(''); + $usage->setResourcePath(''); + } +} diff --git a/src/Appwrite/Platform/Modules/S3/Http/S3/Base.php b/src/Appwrite/Platform/Modules/S3/Http/S3/Base.php new file mode 100644 index 0000000000..60f1c3f0f7 --- /dev/null +++ b/src/Appwrite/Platform/Modules/S3/Http/S3/Base.php @@ -0,0 +1,1015 @@ +setHttpMethod($method) + ->setHttpPath(self::PREFIX) + ->httpAlias(self::PREFIX . '/*') + ->desc('S3 compatibility API') + ->groups(['s3']) + ->label('scope', 'public') + ->label('origin', '*') + ->label('event', 'buckets.[bucketId].files.[fileId]') + ->label('audits.resource', 'file/{response.$id}') + ->label('usage.resource', 'file/{response.$id}') + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('team') + ->inject('user') + ->inject('deviceForFiles') + ->inject('locks') + ->inject('cache') + ->inject('queueForEvents') + ->inject('publisherForDeletes') + ->inject('publisherForAudits') + ->inject('queueForRealtime') + ->inject('publisherForFunctions') + ->inject('queueForWebhooks') + ->inject('eventProcessor') + ->callback($this->action(...)); + } + + public function action( + Request $request, + Response $response, + Database $dbForProject, + Document $project, + Document $team, + User $user, + Device $deviceForFiles, + callable $locks, + Cache $cache, + Event $queueForEvents, + DeletePublisher $publisherForDeletes, + AuditPublisher $publisherForAudits, + Event $queueForRealtime, + FunctionPublisher $publisherForFunctions, + Event $queueForWebhooks, + EventProcessor $eventProcessor, + ): void { + try { + $this->route($request, $response, $dbForProject, $project, $team, $user, $deviceForFiles, $locks, $cache, $queueForEvents, $publisherForDeletes, $publisherForAudits, $queueForRealtime, $publisherForFunctions, $queueForWebhooks, $eventProcessor); + } catch (AppwriteException $error) { + $this->sendError($response, $this->mapError($error), $error->getMessage(), $request->getURI(), $this->mapStatus($error)); + } catch (\Throwable $error) { + $this->sendError($response, 'InternalError', $error->getMessage(), $request->getURI(), Response::STATUS_CODE_INTERNAL_SERVER_ERROR); + } + } + + abstract protected function route(Request $request, Response $response, Database $dbForProject, Document $project, Document $team, User $user, Device $deviceForFiles, callable $locks, Cache $cache, Event $queueForEvents, DeletePublisher $publisherForDeletes, AuditPublisher $publisherForAudits, Event $queueForRealtime, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, EventProcessor $eventProcessor): void; + + protected function authorize(Request $request, Document $project, Document $team, User $user, array $scopes): void + { + (new SignatureV4())->verify($request, $project, $team, $user, $scopes); + } + + protected function parts(Request $request): array + { + $path = \parse_url($request->getURI(), PHP_URL_PATH) ?: ''; + $path = \rawurldecode($path); + if ($path === self::PREFIX) { + return ['', '']; + } + + $tail = \ltrim(\substr($path, \strlen(self::PREFIX)), '/'); + [$bucketId, $key] = \array_pad(\explode('/', $tail, 2), 2, ''); + return [$bucketId, $key]; + } + + protected function query(Request $request, string $key, string $default = ''): string + { + $query = $request->getServer('query_string', '') ?: (\parse_url($request->getURI(), PHP_URL_QUERY) ?: ''); + \parse_str($query, $params); + return (string) ($params[$key] ?? $default); + } + + protected function hasQuery(Request $request, string $key): bool + { + $query = $request->getServer('query_string', '') ?: (\parse_url($request->getURI(), PHP_URL_QUERY) ?: ''); + \parse_str($query, $params); + return \array_key_exists($key, $params); + } + + protected function bucket(Database $dbForProject, string $bucketId): Document + { + $bucket = $dbForProject->getAuthorization()->skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + if ($bucket->isEmpty() || !$bucket->getAttribute('enabled', true)) { + throw new AppwriteException(AppwriteException::STORAGE_BUCKET_NOT_FOUND); + } + return $bucket; + } + + protected function objectKey(Document $file): string + { + return $file->getAttribute('folder', '') . $file->getAttribute('name', $file->getId()); + } + + /** + * @return array{folder: string, name: string} + */ + protected function objectPath(string $key): array + { + $slash = \strrpos($key, '/'); + $folder = $slash === false ? '' : \substr($key, 0, $slash + 1); + $name = $slash === false ? $key : \substr($key, $slash + 1); + $validator = new Folder(); + + if ($name === '' || !$validator->isValid($folder)) { + throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, $validator->getDescription()); + } + + return [ + 'folder' => Folder::normalize($folder), + 'name' => $name, + ]; + } + + protected function isFolderMarker(string $key): bool + { + return $key !== '' && \str_ends_with($key, '/'); + } + + protected function multipartStateKey(Document $project, string $uploadId): string + { + return 's3.multipart.state.' . $project->getId() . '.' . $uploadId; + } + + protected function multipartIndexKey(Document $project, Document $bucket): string + { + return 's3.multipart.index.' . $project->getId() . '.' . $bucket->getId(); + } + + protected function createMultipartUpload(Database $dbForProject, Cache $cache, Document $project, Device $deviceForFiles, Document $bucket, string $key, string $uploadId, string $contentType, array $metadata): array + { + $existing = $this->findObject($dbForProject, $bucket, $key); + $object = $existing === null ? $this->objectPath($key) : [ + 'folder' => $existing->getAttribute('folder', ''), + 'name' => $existing->getAttribute('name', ''), + ]; + // Stage each upload independently; destination identity is chosen only at locked completion. + $fileId = ID::unique(); + $name = $object['name']; + $this->validateFileConstraints($bucket, $name, 0); + $path = $this->path($deviceForFiles, $bucket, $fileId, $name); + $contentType = $this->contentType($name, $contentType); + $deviceMetadata = []; + $deviceForFiles->prepare($path, $contentType, 2, $deviceMetadata); + + $upload = [ + 'uploadId' => $uploadId, + 'bucketId' => $bucket->getId(), + 'key' => $key, + 'requestedKey' => $key, + 'folder' => $object['folder'], + 'name' => $name, + 'fileId' => $fileId, + 'path' => $path, + 'contentType' => $contentType, + 'metadata' => ['object' => $metadata, 'device' => $deviceMetadata], + 'parts' => [], + 'initiated' => \gmdate('c'), + ]; + + $cache->save($this->multipartStateKey($project, $uploadId), \json_encode($upload, JSON_THROW_ON_ERROR)); + $cache->save($this->multipartIndexKey($project, $bucket), $uploadId, $uploadId); + + return $upload; + } + + protected function multipartUpload(Cache $cache, Document $project, string $uploadId): array + { + $upload = $cache->load($this->multipartStateKey($project, $uploadId), self::MULTIPART_TTL); + if (!\is_string($upload)) { + throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Multipart upload not found.'); + } + + return \json_decode($upload, true, flags: JSON_THROW_ON_ERROR); + } + + protected function multipartUploadForBucket(Cache $cache, Document $project, string $uploadId, string $bucketId, string $key = ''): array + { + $upload = $this->multipartUpload($cache, $project, $uploadId); + if (($upload['bucketId'] ?? '') !== $bucketId || ($key !== '' && isset($upload['requestedKey']) && $upload['requestedKey'] !== $key)) { + throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Multipart upload not found.'); + } + + return $upload; + } + + protected function saveMultipartUpload(Cache $cache, Document $project, array $upload): void + { + $cache->save($this->multipartStateKey($project, (string) $upload['uploadId']), \json_encode($upload, JSON_THROW_ON_ERROR)); + } + + protected function uploadMultipartPart(Cache $cache, Document $project, Device $deviceForFiles, array $upload, int $partNumber, string $body, string $etag): array + { + $metadata = $upload['metadata'] ?? []; + $deviceMetadata = $metadata['device'] ?? []; + $chunks = \max(2, (int) ($deviceMetadata['chunks'] ?? 0) + 2, $partNumber + 1); + $deviceForFiles->upload( + new Stream($body), + (string) $upload['path'], + (string) ($upload['contentType'] ?? 'application/octet-stream'), + $partNumber, + $chunks, + $deviceMetadata + ); + + $metadata['device'] = $deviceMetadata; + $upload['metadata'] = $metadata; + $upload['parts'][(string) $partNumber] = ['etag' => $etag, 'size' => \strlen($body)]; + $this->saveMultipartUpload($cache, $project, $upload); + + return $upload; + } + + protected function unsupportedSubresource(Request $request): bool + { + // Requests are SigV4-signed, so every query key is deliberate SDK output. + // Any key outside this allow-list is an S3 feature Appwrite Storage does + // not implement (?tagging, ?legal-hold, ?restore, ?publicAccessBlock, …); + // refusing loudly beats falling through to an object write or a bucket + // delete the client never asked for. + $allowed = [ + 'X-Amz-Algorithm', 'X-Amz-Content-Sha256', 'X-Amz-Credential', + 'X-Amz-Date', 'X-Amz-Expires', 'X-Amz-Signature', + 'X-Amz-SignedHeaders', + 'acl', 'continuation-token', 'delete', 'delimiter', 'encoding-type', + 'fetch-owner', 'list-type', 'location', 'marker', 'max-keys', + 'partNumber', 'prefix', 'response-cache-control', + 'response-content-disposition', 'response-content-encoding', + 'response-content-language', 'response-content-type', + 'response-expires', 'start-after', 'uploadId', 'uploads', + 'versionId', 'x-id', + ]; + + $query = $request->getServer('query_string', '') ?: (\parse_url($request->getURI(), PHP_URL_QUERY) ?: ''); + \parse_str($query, $params); + foreach (\array_keys($params) as $key) { + if (!\in_array((string) $key, $allowed, true)) { + return true; + } + } + + return false; + } + + protected function requestBody(Request $request): string + { + $payload = $request->getRawPayload(); + if (!AwsChunked::applies($request->getHeaderLine('x-amz-content-sha256'), $request->getHeaderLine('content-encoding'))) { + return $payload; + } + + $declaredLength = $request->getHeaderLine('x-amz-decoded-content-length'); + return AwsChunked::decode($payload, $declaredLength === '' ? null : (int) $declaredLength); + } + + protected function path(Device $deviceForFiles, Document $bucket, string $fileId, string $key): string + { + $extension = \pathinfo($key, PATHINFO_EXTENSION) ?: 'bin'; + $path = $deviceForFiles->getPath($fileId . '.' . $extension); + return \str_ireplace($deviceForFiles->getRoot(), $deviceForFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); + } + + protected function contentType(string $name, string $contentType, string $body = ''): string + { + $contentType = \strtolower(\trim(\explode(';', $contentType)[0])); + if ($contentType !== '' && $contentType !== 'application/octet-stream' && $contentType !== 'binary/octet-stream') { + return $contentType; + } + + $extension = \strtolower(\pathinfo($name, PATHINFO_EXTENSION)); + $byExtension = [ + 'avif' => 'image/avif', + 'bmp' => 'image/bmp', + 'css' => 'text/css', + 'csv' => 'text/csv', + 'gif' => 'image/gif', + 'htm' => 'text/html', + 'html' => 'text/html', + 'ico' => 'image/x-icon', + 'jpeg' => 'image/jpeg', + 'jpg' => 'image/jpeg', + 'js' => 'text/javascript', + 'json' => 'application/json', + 'm4v' => 'video/x-m4v', + 'mov' => 'video/quicktime', + 'mp3' => 'audio/mpeg', + 'mp4' => 'video/mp4', + 'pdf' => 'application/pdf', + 'png' => 'image/png', + 'svg' => 'image/svg+xml', + 'txt' => 'text/plain', + 'wasm' => 'application/wasm', + 'webm' => 'video/webm', + 'webp' => 'image/webp', + 'xml' => 'application/xml', + 'zip' => 'application/zip', + ]; + if (isset($byExtension[$extension])) { + return $byExtension[$extension]; + } + + if ($body !== '') { + $detected = (new \finfo(FILEINFO_MIME_TYPE))->buffer($body); + if (\is_string($detected) && $detected !== '') { + return $detected; + } + } + + return 'application/octet-stream'; + } + + protected function validateFileConstraints(Document $bucket, string $name, int $size): void + { + $maximumFileSize = (int) $bucket->getAttribute('maximumFileSize', 0); + if ($maximumFileSize > 0 && $size > $maximumFileSize) { + throw new AppwriteException(AppwriteException::STORAGE_INVALID_FILE_SIZE, 'File size not allowed'); + } + + $allowed = $bucket->getAttribute('allowedFileExtensions', []); + if (!empty($allowed)) { + $extension = \strtolower(\pathinfo($name, PATHINFO_EXTENSION)); + $allowed = \array_map(fn (string $value): string => \strtolower(\ltrim($value, '.')), $allowed); + if ($extension === '' || !\in_array($extension, $allowed, true)) { + throw new AppwriteException(AppwriteException::STORAGE_FILE_TYPE_UNSUPPORTED, 'File extension not allowed'); + } + } + } + + protected function validateCopySize(Document $bucket, Document $sourceFile): void + { + $maximumFileSize = (int) $bucket->getAttribute('maximumFileSize', 0); + $sourceSize = (int) $sourceFile->getAttribute('sizeOriginal', 0); + + if ($maximumFileSize > 0 && $sourceSize > $maximumFileSize) { + throw new AppwriteException(AppwriteException::STORAGE_INVALID_FILE_SIZE, 'Copied file size not allowed'); + } + } + + protected function queueFileEvent(Event $queueForEvents, Response $response, Document $bucket, Document $file, string $action): void + { + $queueForEvents + ->setEvent('buckets.[bucketId].files.[fileId].' . $action) + ->setParam('bucketId', $bucket->getId()) + ->setParam('fileId', $file->getId()) + ->setContext('bucket', $bucket) + ->setPayload($response->output($file, Response::MODEL_FILE)); + } + + /** + * Batch operations emit one event per file, but the events queue holds a + * single event, so the shutdown flush would only carry the last file — and + * only when the whole batch completes. Dispatch realtime, functions and + * webhooks inline as each file is deleted — the platform's bulk pattern + * (Documents\Action::triggerBulk) — so a later batch failure cannot lose + * events for files already gone, then reset the queues so the shutdown + * flush does not re-fire this file. + */ + protected function triggerFileEvent(Event $queueForEvents, Event $queueForRealtime, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, EventProcessor $eventProcessor, Database $dbForProject, Response $response, Document $bucket, Document $file, string $action): void + { + $queueForEvents + ->setEvent('buckets.[bucketId].files.[fileId].' . $action) + ->setParam('bucketId', $bucket->getId()) + ->setParam('fileId', $file->getId()) + ->setContext('bucket', $bucket) + ->setPayload($response->output($file, Response::MODEL_FILE)); + + // Get project and function/webhook events (cached per project) + $project = $queueForEvents->getProject(); + $functionsEvents = $eventProcessor->getFunctionsEvents($project, $dbForProject); + $webhooksEvents = $eventProcessor->getWebhooksEvents($project); + + $queueForRealtime + ->from($queueForEvents) + ->trigger(); + + $generatedEvents = Event::generateEvents( + $queueForEvents->getEvent(), + $queueForEvents->getParams() + ); + + if (!empty($functionsEvents)) { + foreach ($generatedEvents as $event) { + if (isset($functionsEvents[$event])) { + $publisherForFunctions->enqueue(FunctionMessage::fromEvent( + event: $queueForEvents->getEvent(), + params: $queueForEvents->getParams(), + project: $queueForEvents->getProject(), + user: $queueForEvents->getUser(), + userId: $queueForEvents->getUserId(), + payload: $queueForEvents->getPayload(), + platform: $queueForEvents->getPlatform(), + )); + break; + } + } + } + + if (!empty($webhooksEvents)) { + foreach ($generatedEvents as $event) { + if (isset($webhooksEvents[$event])) { + $queueForWebhooks + ->from($queueForEvents) + ->trigger(); + break; + } + } + } + + $queueForEvents->reset(); + $queueForRealtime->reset(); + $queueForWebhooks->reset(); + } + + protected function queueBucketEvent(Event $queueForEvents, Response $response, Document $bucket, string $action): void + { + $queueForEvents + ->setEvent('buckets.[bucketId].' . $action) + ->setParam('bucketId', $bucket->getId()) + ->setContext('bucket', $bucket) + ->setPayload($response->output($bucket, Response::MODEL_BUCKET)); + } + + protected function enqueueFileCacheDelete(DeletePublisher $publisherForDeletes, Event $queueForEvents, Document $bucket, Document $file): void + { + $publisherForDeletes->enqueue(new DeleteMessage( + project: $queueForEvents->getProject(), + type: DELETE_TYPE_CACHE_BY_RESOURCE, + resource: 'file/' . $file->getId(), + resourceType: 'bucket/' . $bucket->getId(), + )); + } + + protected function enqueueAudit(AuditPublisher $publisherForAudits, Request $request, Document $project, User $user, string $event, Document $resource, string $resourceType): void + { + $publisherForAudits->enqueue(new AuditMessage( + event: $event, + payload: $resource->getArrayCopy(), + project: $project, + user: $user, + resource: $resourceType . '/' . $resource->getId(), + mode: 'api', + ip: $request->getIP(), + userAgent: $request->getUserAgent(''), + hostname: $request->getHostname(), + )); + } + + protected function createBucket(Database $dbForProject, string $bucketId): void + { + $files = (Config::getParam('collections', [])['buckets'] ?? [])['files'] ?? []; + if (empty($files)) { + throw new AppwriteException(AppwriteException::GENERAL_SERVER_ERROR, 'Files collection is not configured.'); + } + + $attributes = []; + foreach ($files['attributes'] as $attribute) { + $attributes[] = new Document([ + '$id' => $attribute['$id'], + 'type' => $attribute['type'], + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'array' => $attribute['array'], + 'filters' => $attribute['filters'], + 'default' => $attribute['default'] ?? null, + 'format' => $attribute['format'] ?? '', + ]); + } + + $indexes = []; + foreach ($files['indexes'] as $index) { + $indexes[] = new Document([ + '$id' => $index['$id'], + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'] ?? [], + 'orders' => $index['orders'] ?? [], + ]); + } + + $permissions = Permission::aggregate(null) ?? []; + + try { + $dbForProject->getAuthorization()->skip(fn () => $dbForProject->createDocument('buckets', new Document([ + '$id' => $bucketId, + '$collection' => 'buckets', + '$permissions' => $permissions, + 'name' => $bucketId, + 'maximumFileSize' => (int) \Utopia\System\System::getEnv('_APP_STORAGE_LIMIT', 0), + 'allowedFileExtensions' => [], + 'fileSecurity' => false, + 'enabled' => true, + 'compression' => 'none', + 'encryption' => false, + 'antivirus' => false, + 'transformations' => true, + 'search' => $bucketId, + ]))); + } catch (DuplicateException) { + throw new AppwriteException(AppwriteException::STORAGE_BUCKET_ALREADY_EXISTS); + } + + try { + $bucket = $dbForProject->getAuthorization()->skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + $dbForProject->getAuthorization()->skip(fn () => $dbForProject->createCollection('bucket_' . $bucket->getSequence(), $attributes, $indexes, permissions: $permissions, documentSecurity: false)); + } catch (\Throwable $error) { + // Roll back the bucket document so a failed collection creation does + // not leave an unusable bucket with no backing collection. + $dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteDocument('buckets', $bucketId)); + throw $error; + } + } + + protected function putObject(Database $dbForProject, Device $deviceForFiles, Document $bucket, string $key, string $body, string $contentType, array $metadata = []): Document + { + $existing = $this->findObject($dbForProject, $bucket, $key); + $object = $existing === null ? $this->objectPath($key) : [ + 'folder' => $existing->getAttribute('folder', ''), + 'name' => $existing->getAttribute('name', ''), + ]; + $fileId = $existing?->getId() ?: ID::unique(); + $folder = $object['folder']; + $name = $object['name']; + $this->validateFileConstraints($bucket, $name, \strlen($body)); + $path = $existing?->getAttribute('path', '') ?: $this->path($deviceForFiles, $bucket, $fileId, $name); + $contentType = $this->contentType($name, $contentType, $body); + $etag = \md5($body); + + if (!$deviceForFiles->write($path, new Stream($body), $contentType)) { + throw new AppwriteException(AppwriteException::GENERAL_SERVER_ERROR, 'Failed to save object.'); + } + + $document = new Document([ + '$id' => ID::custom($fileId), + '$permissions' => [], + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $name, + 'folder' => $folder, + 'path' => $path, + 'signature' => $etag, + 'mimeType' => $contentType, + 'sizeOriginal' => \strlen($body), + 'sizeActual' => $deviceForFiles->getFileSize($path), + 'algorithm' => 'none', + 'comment' => '', + 'chunksTotal' => 1, + 'chunksUploaded' => 1, + 'search' => \implode(' ', [$fileId, $folder, $name]), + 'metadata' => $metadata, + ]); + + try { + return $dbForProject->getAuthorization()->skip(fn () => $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $document)); + } catch (DuplicateException) { + return $dbForProject->getAuthorization()->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([ + 'name' => $name, + 'folder' => $folder, + 'path' => $path, + 'signature' => $etag, + 'mimeType' => $contentType, + 'sizeOriginal' => \strlen($body), + 'sizeActual' => $deviceForFiles->getFileSize($path), + 'search' => \implode(' ', [$fileId, $folder, $name]), + 'metadata' => $metadata, + ]))); + } catch (\Throwable $error) { + // The bytes were just written for a brand-new object; drop them so a + // failed DB write does not leave an unreachable file in storage. + if ($existing === null) { + $deviceForFiles->delete($path); + } + throw $error; + } + } + + /** + * @return array{0: Document, 1: string} + */ + protected function putObjectLocked(callable $locks, Database $dbForProject, Document $project, Device $deviceForFiles, Document $bucket, string $key, string $body, string $contentType, array $metadata = []): array + { + // Storage permits duplicate names, so lookup and write must share one key-scoped critical section. + return $locks($this->objectLockKey($project, $bucket, $key), 600, function () use ($dbForProject, $deviceForFiles, $bucket, $key, $body, $contentType, $metadata): array { + $existing = $this->findObject($dbForProject, $bucket, $key); + $file = $this->putObject($dbForProject, $deviceForFiles, $bucket, $key, $body, $contentType, $metadata); + + return [$file, $existing === null ? 'create' : 'update']; + }, timeout: 120.0); + } + + protected function objectLockKey(Document $project, Document $bucket, string $key): string + { + return 's3:object:' . $project->getId() . ':' . $bucket->getId() . ':' . \hash('sha256', $key); + } + + /** + * Serializes part uploads and abort of one multipart upload: the cached + * upload state and the device chunk bookkeeping are read-modify-write, so + * concurrent parts would drop each other's entries. Never held together + * with the object lock — the lock pool fails fast instead of queueing. + */ + protected function multipartLockKey(Document $project, string $uploadId): string + { + return 's3:multipart:' . $project->getId() . ':' . $uploadId; + } + + protected function createObjectDocument(Database $dbForProject, Device $deviceForFiles, Document $bucket, string $folder, string $name, string $fileId, string $path, string $etag, string $contentType, int $size, int $chunksTotal, array $metadata = []): Document + { + $contentType = $this->contentType($name, $contentType); + $document = new Document([ + '$id' => ID::custom($fileId), + '$permissions' => [], + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $name, + 'folder' => $folder, + 'path' => $path, + 'signature' => $etag, + 'mimeType' => $contentType, + 'sizeOriginal' => $size, + 'sizeActual' => $deviceForFiles->getFileSize($path), + 'algorithm' => 'none', + 'comment' => '', + 'chunksTotal' => $chunksTotal, + 'chunksUploaded' => $chunksTotal, + 'search' => \implode(' ', [$fileId, $folder, $name]), + 'metadata' => $metadata, + ]); + + try { + return $dbForProject->getAuthorization()->skip(fn () => $dbForProject->createDocument('bucket_' . $bucket->getSequence(), $document)); + } catch (DuplicateException) { + return $dbForProject->getAuthorization()->skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getSequence(), $fileId, new Document([ + 'name' => $name, + 'folder' => $folder, + 'path' => $path, + 'signature' => $etag, + 'mimeType' => $contentType, + 'sizeOriginal' => $size, + 'sizeActual' => $deviceForFiles->getFileSize($path), + 'chunksTotal' => $chunksTotal, + 'chunksUploaded' => $chunksTotal, + 'search' => \implode(' ', [$fileId, $folder, $name]), + 'metadata' => $metadata, + ]))); + } + } + + protected function requestObjectMetadata(Request $request): array + { + $metadata = ['userMetadata' => []]; + foreach ($request->getHeaders() as $name => $values) { + if (\str_starts_with($name, 'x-amz-meta-')) { + $metadata['userMetadata'][\substr($name, \strlen('x-amz-meta-'))] = \implode(',', $values); + } + } + + $sse = $request->getHeaderLine('x-amz-server-side-encryption'); + if ($sse !== '') { + $metadata['serverSideEncryption'] = $sse; + } + + foreach ([ + 'cache-control' => 'cacheControl', + 'content-disposition' => 'contentDisposition', + 'content-language' => 'contentLanguage', + 'expires' => 'expires', + ] as $header => $attribute) { + $value = $request->getHeaderLine($header); + if ($value !== '') { + $metadata[$attribute] = $value; + } + } + + $contentEncodings = \array_filter( + \array_map('trim', \explode(',', $request->getHeaderLine('content-encoding'))), + fn (string $encoding): bool => $encoding !== '' && \strcasecmp($encoding, 'aws-chunked') !== 0 + ); + if ($contentEncodings !== []) { + $metadata['contentEncoding'] = \implode(', ', $contentEncodings); + } + + return $metadata; + } + + /** + * @return array + */ + protected function deleteObjectKeys(string $body): array + { + if (!\preg_match_all('/\s*([^<]+)\s*<\/Key>/', $body, $matches)) { + return []; + } + + return \array_map('html_entity_decode', $matches[1]); + } + + protected function deleteQuiet(string $body): bool + { + return \preg_match('/\s*true\s*<\/Quiet>/i', $body) === 1; + } + + protected function getObject(Database $dbForProject, Document $bucket, string $key): Document + { + $file = $this->findObject($dbForProject, $bucket, $key); + if ($file === null) { + throw new AppwriteException(AppwriteException::STORAGE_FILE_NOT_FOUND); + } + return $file; + } + + protected function findObject(Database $dbForProject, Document $bucket, string $key): ?Document + { + $collection = 'bucket_' . $bucket->getSequence(); + $object = $this->objectPath($key); + $files = $dbForProject->getAuthorization()->skip(fn () => $dbForProject->find($collection, [ + Query::equal('folder', [$object['folder']]), + Query::equal('name', [$object['name']]), + Query::limit(2), + ])); + + if (\count($files) > 1) { + throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, "Multiple files match S3 object key '{$key}'."); + } + + return $files[0] ?? null; + } + + /** + * @param array $files + */ + protected function assertUniqueObjectKeys(array $files): void + { + $keys = []; + foreach ($files as $file) { + $key = $this->objectKey($file); + if (isset($keys[$key])) { + throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, "Multiple files match S3 object key '{$key}'."); + } + $keys[$key] = true; + } + } + + /** + * @return array + */ + protected function getMultipartParts(array $upload): array + { + $parts = []; + foreach (($upload['parts'] ?? []) as $partNumber => $part) { + $partNumber = (int) $partNumber; + $parts[$partNumber] = [ + 'partNumber' => $partNumber, + 'etag' => (string) ($part['etag'] ?? ''), + 'size' => (int) ($part['size'] ?? 0), + ]; + } + + \ksort($parts); + return \array_values($parts); + } + + /** + * @return array + */ + protected function getMultipartUploads(Cache $cache, Document $project, Document $bucket): array + { + $uploads = []; + foreach ($cache->list($this->multipartIndexKey($project, $bucket)) as $uploadId) { + $state = $cache->load($this->multipartStateKey($project, $uploadId), self::MULTIPART_TTL); + if (!\is_string($state)) { + $cache->purge($this->multipartIndexKey($project, $bucket), $uploadId); + continue; + } + $upload = \json_decode($state, true, flags: JSON_THROW_ON_ERROR); + + $uploads[] = [ + 'key' => (string) ($upload['requestedKey'] ?? $upload['key'] ?? ''), + 'uploadId' => $uploadId, + 'initiated' => (string) ($upload['initiated'] ?? ''), + ]; + } + + return $uploads; + } + + /** + * @return array + */ + protected function completedPartNumbers(string $body): array + { + if (!\preg_match_all('/\s*(\d+)\s*<\/PartNumber>/', $body, $matches)) { + return []; + } + + return \array_map('intval', $matches[1]); + } + + /** + * @return array{0: Document, 1: string|null} + */ + protected function completeMultipartUpload(callable $locks, Database $dbForProject, Cache $cache, Document $project, Device $deviceForFiles, Document $bucket, array $upload, array $selected): array + { + $parts = $this->getMultipartParts($upload); + $partsByNumber = []; + foreach ($parts as $part) { + $partsByNumber[$part['partNumber']] = $part; + } + + $selected = $selected ?: \array_keys($partsByNumber); + \sort($selected); + if ($selected === []) { + throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Multipart upload has no parts.'); + } + + $size = 0; + $etagBytes = ''; + $expected = 1; + foreach ($selected as $partNumber) { + if ($partNumber !== $expected || !isset($partsByNumber[$partNumber])) { + throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Multipart parts must be contiguous and start at part 1.'); + } + $etagBytes .= \hex2bin($partsByNumber[$partNumber]['etag']) ?: ''; + $size += $partsByNumber[$partNumber]['size']; + $expected++; + } + + $this->validateFileConstraints($bucket, (string) ($upload['name'] ?? $upload['key']), $size); + + $etag = \md5($etagBytes) . '-' . \count($selected); + $key = (string) ($upload['requestedKey'] ?? $upload['key'] ?? ''); + // Finalization and document replacement are serialized with ordinary PUTs for this key. + $result = $locks($this->objectLockKey($project, $bucket, $key), 600, function () use ($dbForProject, $deviceForFiles, $bucket, &$upload, $selected, $size, $etag, $key): array { + $path = (string) $upload['path']; + $committed = $this->findObject($dbForProject, $bucket, $key); + if ($committed !== null && $committed->getAttribute('path', '') === $path) { + // A retry retained this upload's state after its unique staging path became live. + return [$committed, null]; + } + $cleanup = true; + + try { + $upload['metadata']['device'] = $upload['metadata']['device'] ?? []; + $chunks = \count($selected); + if ($chunks === 1) { + $this->uploadEmptyMultipartFinalChunk($deviceForFiles, $upload); + $chunks = 2; + } + if (!$deviceForFiles->finalize($path, $chunks, $upload['metadata']['device'])) { + throw new AppwriteException(AppwriteException::GENERAL_SERVER_ERROR, 'Failed to finalize multipart upload.'); + } + + $existing = $this->findObject($dbForProject, $bucket, $key); + $previousPath = $existing?->getAttribute('path', '') ?? ''; + $cleanup = $previousPath !== $path; + $file = $this->createObjectDocument( + $dbForProject, + $deviceForFiles, + $bucket, + (string) ($upload['folder'] ?? ''), + (string) ($upload['name'] ?? $upload['key']), + $existing?->getId() ?? (string) $upload['fileId'], + $path, + $etag, + (string) $upload['contentType'], + $size, + \count($selected), + $upload['metadata']['object'] ?? [] + ); + // The document owns this path now; later cleanup failures must never delete it. + $cleanup = false; + + if ($previousPath !== '' && $previousPath !== $path) { + try { + $deviceForFiles->delete($previousPath); + } catch (\Throwable) { + // The replacement is already committed; old-path cleanup is best effort. + } + } + + return [$file, $existing === null ? 'create' : 'update']; + } catch (\Throwable $error) { + if ($cleanup) { + $deviceForFiles->delete($path); + } + throw $error; + } + }, timeout: 120.0); + $this->purgeMultipartUpload($cache, $project, $bucket, $upload); + + return $result; + } + + protected function deleteMultipartUpload(Cache $cache, Document $project, Device $deviceForFiles, Document $bucket, array $upload): void + { + $path = (string) ($upload['path'] ?? ''); + if ($path !== '') { + $deviceForFiles->abort($path, (string) ($upload['metadata']['device']['uploadId'] ?? '')); + } + $this->purgeMultipartUpload($cache, $project, $bucket, $upload); + } + + protected function uploadEmptyMultipartFinalChunk(Device $deviceForFiles, array &$upload): void + { + $deviceForFiles->upload( + new Stream(''), + (string) $upload['path'], + (string) ($upload['contentType'] ?? 'application/octet-stream'), + 2, + 2, + $upload['metadata']['device'] + ); + } + + protected function purgeMultipartUpload(Cache $cache, Document $project, Document $bucket, array $upload): void + { + $uploadId = (string) ($upload['uploadId'] ?? ''); + if ($uploadId !== '') { + $cache->purge($this->multipartStateKey($project, $uploadId)); + $cache->purge($this->multipartIndexKey($project, $bucket), $uploadId); + } + } + + protected function sendXml(Response $response, string $xml, int $status = Response::STATUS_CODE_OK): void + { + $response + ->setStatusCode($status) + ->setContentType('application/xml') + ->send($xml); + } + + protected function sendError(Response $response, string $code, string $message, string $resource, int $status = Response::STATUS_CODE_BAD_REQUEST): void + { + $response + ->setStatusCode($status) + ->setContentType('application/xml') + ->send(S3Xml::error($code, $message, $resource)); + } + + private function mapError(AppwriteException $error): string + { + return match ($error->getType()) { + AppwriteException::STORAGE_BUCKET_NOT_FOUND => 'NoSuchBucket', + AppwriteException::STORAGE_FILE_NOT_FOUND => 'NoSuchKey', + AppwriteException::STORAGE_BUCKET_ALREADY_EXISTS => 'BucketAlreadyOwnedByYou', + AppwriteException::GENERAL_UNAUTHORIZED_SCOPE, AppwriteException::USER_UNAUTHORIZED => 'AccessDenied', + default => 'InvalidRequest', + }; + } + + private function mapStatus(AppwriteException $error): int + { + return match ($error->getType()) { + AppwriteException::STORAGE_BUCKET_NOT_FOUND, + AppwriteException::STORAGE_FILE_NOT_FOUND => Response::STATUS_CODE_NOT_FOUND, + AppwriteException::STORAGE_BUCKET_ALREADY_EXISTS => Response::STATUS_CODE_CONFLICT, + AppwriteException::GENERAL_UNAUTHORIZED_SCOPE, + AppwriteException::USER_UNAUTHORIZED => Response::STATUS_CODE_FORBIDDEN, + default => Response::STATUS_CODE_BAD_REQUEST, + }; + } +} diff --git a/src/Appwrite/Platform/Modules/S3/Http/S3/Create.php b/src/Appwrite/Platform/Modules/S3/Http/S3/Create.php new file mode 100644 index 0000000000..81cbdf7a6f --- /dev/null +++ b/src/Appwrite/Platform/Modules/S3/Http/S3/Create.php @@ -0,0 +1,225 @@ +setHttpMethods([Action::HTTP_REQUEST_METHOD_PUT, Action::HTTP_REQUEST_METHOD_POST]); + } + + protected function route(Request $request, Response $response, Database $dbForProject, Document $project, Document $team, User $user, Device $deviceForFiles, callable $locks, Cache $cache, Event $queueForEvents, DeletePublisher $publisherForDeletes, AuditPublisher $publisherForAudits, Event $queueForRealtime, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, EventProcessor $eventProcessor): void + { + [$bucketId, $key] = $this->parts($request); + if ($bucketId === '') { + throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Bucket is required.'); + } + + if ($this->unsupportedSubresource($request)) { + $this->authorize($request, $project, $team, $user, $key === '' ? ['buckets.write'] : ['files.write']); + $this->sendXml($response, S3Xml::error('NotImplemented', 'This S3 feature is not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED); + return; + } + + if ($this->hasQuery($request, 'acl')) { + $this->authorize($request, $project, $team, $user, $key === '' ? ['buckets.write'] : ['files.write']); + $bucket = $this->bucket($dbForProject, $bucketId); + if ($key !== '') { + $this->getObject($dbForProject, $bucket, $key); + } + $response->setStatusCode(Response::STATUS_CODE_OK)->send(''); + return; + } + + if ($request->getMethod() === Action::HTTP_REQUEST_METHOD_POST && $key === '' && $this->hasQuery($request, 'delete')) { + $this->authorize($request, $project, $team, $user, ['files.write']); + $bucket = $this->bucket($dbForProject, $bucketId); + $payload = $this->requestBody($request); + $deleted = []; + foreach ($this->deleteObjectKeys($payload) as $objectKey) { + [$file, $removed] = $locks($this->objectLockKey($project, $bucket, $objectKey), 600, function () use ($dbForProject, $deviceForFiles, $bucket, $objectKey): array { + $file = $this->findObject($dbForProject, $bucket, $objectKey); + if ($file === null) { + return [null, true]; + } + + $path = $file->getAttribute('path', ''); + if ($path !== '' && !$deviceForFiles->delete($path)) { + // Storage delete failed; keep the DB document so the + // object remains reachable and report no deletion. + return [$file, false]; + } + $dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $file->getId())); + + return [$file, true]; + }, timeout: 120.0); + + if (!$removed) { + continue; + } + if ($file !== null) { + $this->enqueueFileCacheDelete($publisherForDeletes, $queueForEvents, $bucket, $file); + $this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.delete', $file, 'file'); + $this->triggerFileEvent($queueForEvents, $queueForRealtime, $publisherForFunctions, $queueForWebhooks, $eventProcessor, $dbForProject, $response, $bucket, $file, 'delete'); + } + $deleted[] = $objectKey; + } + + $this->sendXml($response, S3Xml::deleteObjects($deleted, $this->deleteQuiet($payload))); + return; + } + + if ($request->getMethod() === Action::HTTP_REQUEST_METHOD_POST && $key !== '' && $this->hasQuery($request, 'uploads')) { + $this->authorize($request, $project, $team, $user, ['files.write']); + if ($this->isFolderMarker($key)) { + $this->sendXml($response, S3Xml::error('NotImplemented', 'S3 folders are not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED); + return; + } + $bucket = $this->bucket($dbForProject, $bucketId); + $uploadId = \bin2hex(\random_bytes(16)); + $this->createMultipartUpload($dbForProject, $cache, $project, $deviceForFiles, $bucket, $key, $uploadId, $request->getHeaderLine('content-type', 'application/octet-stream'), $this->requestObjectMetadata($request)); + $this->sendXml($response, S3Xml::initiateMultipart($bucketId, $key, $uploadId)); + return; + } + + if ($request->getMethod() === Action::HTTP_REQUEST_METHOD_PUT && $key !== '' && $this->query($request, 'uploadId') !== '' && $this->query($request, 'partNumber') !== '') { + $this->authorize($request, $project, $team, $user, ['files.write']); + $bucket = $this->bucket($dbForProject, $bucketId); + $partNumber = (int) $this->query($request, 'partNumber'); + $uploadId = $this->query($request, 'uploadId'); + + $body = $this->requestBody($request); + $copySource = $request->getHeaderLine('x-amz-copy-source'); + if ($copySource !== '') { + // Copy reads the source bytes, so the key must also hold files.read. + $this->authorize($request, $project, $team, $user, ['files.read']); + $copySource = \rawurldecode(\ltrim($copySource, '/')); + [$sourceBucketId, $sourceKey] = \array_pad(\explode('/', $copySource, 2), 2, ''); + $sourceBucket = $this->bucket($dbForProject, $sourceBucketId); + $sourceFile = $this->getObject($dbForProject, $sourceBucket, $sourceKey); + $this->validateCopySize($bucket, $sourceFile); + $body = (string) $deviceForFiles->read($sourceFile->getAttribute('path', '')); + if (\preg_match('/^bytes=(\d+)-(\d+)$/', $request->getHeaderLine('x-amz-copy-source-range'), $matches) === 1) { + $start = (int) $matches[1]; + $end = (int) $matches[2]; + $body = \substr($body, $start, $end - $start + 1); + } + } + + $etag = \md5($body); + // SDKs upload parts of one upload concurrently; the cached parts map + // and device chunk bookkeeping are read-modify-write, so load and + // save must share the upload's critical section. + $locks($this->multipartLockKey($project, $uploadId), 600, function () use ($cache, $project, $deviceForFiles, $uploadId, $bucketId, $key, $partNumber, $body, $etag): void { + $upload = $this->multipartUploadForBucket($cache, $project, $uploadId, $bucketId, $key); + $this->uploadMultipartPart($cache, $project, $deviceForFiles, $upload, $partNumber, $body, $etag); + }, timeout: 120.0); + if ($copySource !== '') { + $this->sendXml($response, S3Xml::copyPart($etag)); + return; + } + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->addHeader('ETag', '"' . $etag . '"') + ->send(''); + return; + } + + if ($request->getMethod() === Action::HTTP_REQUEST_METHOD_POST && $key !== '' && $this->query($request, 'uploadId') !== '') { + $this->authorize($request, $project, $team, $user, ['files.write']); + $bucket = $this->bucket($dbForProject, $bucketId); + // No multipart lock here: clients complete only after every part + // returned 200, and each part's state save commits inside its lock + // before that response, so the snapshot below always holds all + // acknowledged parts. Nesting the multipart lock around the object + // lock inside completeMultipartUpload would also need two + // connections from the fail-fast lock pool at once. + $selected = $this->completedPartNumbers($this->requestBody($request)); + $upload = $this->multipartUploadForBucket($cache, $project, $this->query($request, 'uploadId'), $bucketId, $key); + [$file, $event] = $this->completeMultipartUpload($locks, $dbForProject, $cache, $project, $deviceForFiles, $bucket, $upload, $selected); + if ($event !== null) { + $this->queueFileEvent($queueForEvents, $response, $bucket, $file, $event); + $this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.' . $event, $file, 'file'); + } + $this->sendXml($response, S3Xml::completeMultipart($bucketId, $key, $file->getAttribute('signature', ''))); + return; + } + + if ($key === '') { + $this->authorize($request, $project, $team, $user, ['buckets.write']); + $this->createBucket($dbForProject, $bucketId); + $bucket = $this->bucket($dbForProject, $bucketId); + $this->queueBucketEvent($queueForEvents, $response, $bucket, 'create'); + $this->enqueueAudit($publisherForAudits, $request, $project, $user, 'bucket.create', $bucket, 'bucket'); + $response->setStatusCode(Response::STATUS_CODE_OK)->send(''); + return; + } + + $this->authorize($request, $project, $team, $user, ['files.write']); + $bucket = $this->bucket($dbForProject, $bucketId); + if ($this->isFolderMarker($key)) { + if ($request->getHeaderLine('x-amz-copy-source') !== '') { + $this->sendXml($response, S3Xml::error('NotImplemented', 'Copying to an S3 folder marker is not supported.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED); + return; + } + $this->objectPath($key . '.folder'); + $response->setStatusCode(Response::STATUS_CODE_OK)->send(''); + return; + } + + $copySource = $request->getHeaderLine('x-amz-copy-source'); + if ($copySource !== '') { + // Copy reads the source bytes, so the key must also hold files.read. + $this->authorize($request, $project, $team, $user, ['files.read']); + $copySource = \rawurldecode(\ltrim($copySource, '/')); + [$sourceBucketId, $sourceKey] = \array_pad(\explode('/', $copySource, 2), 2, ''); + $sourceBucket = $this->bucket($dbForProject, $sourceBucketId); + $sourceFile = $this->getObject($dbForProject, $sourceBucket, $sourceKey); + $this->validateCopySize($bucket, $sourceFile); + $metadata = $sourceFile->getAttribute('metadata', []); + if (\strtoupper($request->getHeaderLine('x-amz-metadata-directive')) === 'REPLACE') { + $metadata = $this->requestObjectMetadata($request); + } + [$file, $event] = $this->putObjectLocked($locks, $dbForProject, $project, $deviceForFiles, $bucket, $key, (string) $deviceForFiles->read($sourceFile->getAttribute('path', '')), $sourceFile->getAttribute('mimeType', 'application/octet-stream'), $metadata); + $this->queueFileEvent($queueForEvents, $response, $bucket, $file, $event); + $this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.' . $event, $file, 'file'); + $this->sendXml($response, S3Xml::copyObject($file->getAttribute('signature', ''), $file->getUpdatedAt() ?: $file->getCreatedAt())); + return; + } + + [$file, $event] = $this->putObjectLocked($locks, $dbForProject, $project, $deviceForFiles, $bucket, $key, $this->requestBody($request), $request->getHeaderLine('content-type', 'application/octet-stream'), $this->requestObjectMetadata($request)); + $this->queueFileEvent($queueForEvents, $response, $bucket, $file, $event); + $this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.' . $event, $file, 'file'); + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->addHeader('ETag', '"' . $file->getAttribute('signature', '') . '"'); + $sse = (string) ($file->getAttribute('metadata', [])['serverSideEncryption'] ?? ''); + if ($sse !== '') { + $response->addHeader('x-amz-server-side-encryption', $sse); + } + $response->send(''); + } +} diff --git a/src/Appwrite/Platform/Modules/S3/Http/S3/Delete.php b/src/Appwrite/Platform/Modules/S3/Http/S3/Delete.php new file mode 100644 index 0000000000..cde1e7a0dc --- /dev/null +++ b/src/Appwrite/Platform/Modules/S3/Http/S3/Delete.php @@ -0,0 +1,104 @@ +parts($request); + if ($bucketId === '') { + throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Bucket is required.'); + } + + if ($this->unsupportedSubresource($request)) { + $this->authorize($request, $project, $team, $user, $key === '' ? ['buckets.write'] : ['files.write']); + $this->sendXml($response, S3Xml::error('NotImplemented', 'This S3 feature is not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED); + return; + } + + if ($key === '') { + $this->authorize($request, $project, $team, $user, ['buckets.write']); + $bucket = $this->bucket($dbForProject, $bucketId); + + $files = $dbForProject->getAuthorization()->skip(fn () => $dbForProject->find('bucket_' . $bucket->getSequence(), [Query::limit(1)])); + if ($files !== []) { + $this->sendXml($response, S3Xml::error('BucketNotEmpty', 'The bucket you tried to delete is not empty.', $request->getURI()), Response::STATUS_CODE_CONFLICT); + return; + } + + $dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteCollection('bucket_' . $bucket->getSequence())); + $dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteDocument('buckets', $bucketId)); + $this->queueBucketEvent($queueForEvents, $response, $bucket, 'delete'); + $this->enqueueAudit($publisherForAudits, $request, $project, $user, 'bucket.delete', $bucket, 'bucket'); + $response->noContent(); + return; + } + + $this->authorize($request, $project, $team, $user, ['files.write']); + $bucket = $this->bucket($dbForProject, $bucketId); + + if ($this->isFolderMarker($key)) { + $response->noContent(); + return; + } + + if ($this->query($request, 'uploadId') !== '') { + $uploadId = $this->query($request, 'uploadId'); + // Abort races in-flight part uploads for the same upload; take the + // upload's critical section so a late part cannot resurrect state + // that was just purged or upload into an aborted device staging. + $locks($this->multipartLockKey($project, $uploadId), 600, function () use ($cache, $project, $deviceForFiles, $bucket, $uploadId, $bucketId, $key): void { + $this->deleteMultipartUpload($cache, $project, $deviceForFiles, $bucket, $this->multipartUploadForBucket($cache, $project, $uploadId, $bucketId, $key)); + }, timeout: 120.0); + $response->noContent(); + return; + } + + $file = $locks($this->objectLockKey($project, $bucket, $key), 600, function () use ($dbForProject, $deviceForFiles, $bucket, $key): ?Document { + $file = $this->findObject($dbForProject, $bucket, $key); + if ($file === null) { + return null; + } + if (!$deviceForFiles->delete($file->getAttribute('path', ''))) { + throw new AppwriteException(AppwriteException::GENERAL_SERVER_ERROR, 'Failed to delete object.'); + } + $dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $file->getId())); + + return $file; + }, timeout: 120.0); + if ($file !== null) { + $this->enqueueFileCacheDelete($publisherForDeletes, $queueForEvents, $bucket, $file); + $this->queueFileEvent($queueForEvents, $response, $bucket, $file, 'delete'); + $this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.delete', $file, 'file'); + } + $response->noContent(); + } +} diff --git a/src/Appwrite/Platform/Modules/S3/Http/S3/Get.php b/src/Appwrite/Platform/Modules/S3/Http/S3/Get.php new file mode 100644 index 0000000000..170731782b --- /dev/null +++ b/src/Appwrite/Platform/Modules/S3/Http/S3/Get.php @@ -0,0 +1,311 @@ +getMethod() === Action::HTTP_REQUEST_METHOD_HEAD; + [$bucketId, $key] = $this->parts($request); + + if ($bucketId === '') { + $this->authorize($request, $project, $team, $user, ['buckets.read']); + if ($isHead) { + $response->send(''); + return; + } + + $buckets = $dbForProject->getAuthorization()->skip(fn () => $dbForProject->find('buckets', [Query::limit(1000)])); + $this->sendXml($response, S3Xml::listBuckets($buckets)); + return; + } + + if ($key === '') { + $this->authorize($request, $project, $team, $user, ['files.read', 'buckets.read']); + $bucket = $this->bucket($dbForProject, $bucketId); + + if ($isHead) { + $response->send(''); + return; + } + + if ($this->hasQuery($request, 'acl')) { + $this->sendXml($response, S3Xml::accessControlPolicy()); + return; + } + if ($this->hasQuery($request, 'location')) { + $this->sendXml($response, S3Xml::bucketLocation()); + return; + } + if ($this->unsupportedSubresource($request)) { + $this->sendXml($response, S3Xml::error('NotImplemented', 'This S3 feature is not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED); + return; + } + if ($this->hasQuery($request, 'uploads')) { + $this->sendXml($response, S3Xml::listMultipartUploads($bucketId, $this->getMultipartUploads($cache, $project, $bucket))); + return; + } + + // iterate() treats the limit as its page size and follows cursors until every file is loaded. + $files = $dbForProject->getAuthorization()->skip(fn () => \iterator_to_array($dbForProject->iterate('bucket_' . $bucket->getSequence(), [ + Query::orderAsc('$sequence'), + Query::limit(1000), + ]), false)); + $this->assertUniqueObjectKeys($files); + if ($this->query($request, 'list-type') === '2') { + $this->sendXml($response, S3Xml::listObjectsV2( + $bucketId, + $files, + $this->query($request, 'prefix'), + $this->query($request, 'delimiter'), + (int) $this->query($request, 'max-keys', '1000'), + $this->query($request, 'continuation-token'), + $this->query($request, 'start-after') + )); + return; + } + + $this->sendXml($response, S3Xml::listObjects( + $bucketId, + $files, + $this->query($request, 'prefix'), + $this->query($request, 'delimiter'), + (int) $this->query($request, 'max-keys', '1000'), + $this->query($request, 'marker') + )); + return; + } + + $this->authorize($request, $project, $team, $user, ['files.read']); + $bucket = $this->bucket($dbForProject, $bucketId); + + if ($this->query($request, 'uploadId') !== '') { + $parts = $this->getMultipartParts($this->multipartUploadForBucket($cache, $project, $this->query($request, 'uploadId'), $bucketId, $key)); + $this->sendXml($response, S3Xml::listParts($bucketId, $key, $this->query($request, 'uploadId'), $parts)); + return; + } + + $isAcl = $this->hasQuery($request, 'acl'); + $isUnsupported = $this->unsupportedSubresource($request); + $ifNoneMatch = $request->getHeaderLine('if-none-match'); + $ifMatch = $request->getHeaderLine('if-match'); + // Ordinary PUT overwrites its Device path in place, so metadata lookup and + // body materialization must not overlap that key's write critical section. + [$file, $body, $changedRepeatedly] = $locks($this->objectLockKey($project, $bucket, $key), 600, function () use ($dbForProject, $deviceForFiles, $bucket, $key, $isHead, $isAcl, $isUnsupported, $ifNoneMatch, $ifMatch, $request): array { + $file = $this->getObject($dbForProject, $bucket, $key); + $body = null; + $retried = false; + $changedRepeatedly = false; + while (true) { + $path = $file->getAttribute('path', ''); + $size = (int) $file->getAttribute('sizeOriginal', 0); + $range = $this->range($request, $size); + $etag = '"' . $file->getAttribute('signature', '') . '"'; + + if ( + $isHead + || $isAcl + || $isUnsupported + || $ifNoneMatch === $etag + || ($ifMatch !== '' && $ifMatch !== $etag) + || $range === false + ) { + break; + } + + try { + $body = (string) $deviceForFiles->read($path); + break; + } catch (\Throwable $error) { + $latest = $this->getObject($dbForProject, $bucket, $key); + if ($latest->getAttribute('path', '') === $path) { + throw $error; + } + // One moved-path retry closes races with writes outside the S3 gateway. + if ($retried) { + $changedRepeatedly = true; + break; + } + $retried = true; + $file = $latest; + } + } + + return [$file, $body, $changedRepeatedly]; + }, timeout: 120.0); + + if ($changedRepeatedly) { + $this->sendXml($response, S3Xml::error('SlowDown', 'The object changed repeatedly while it was being read. Please retry.', $request->getURI()), Response::STATUS_CODE_SERVICE_UNAVAILABLE); + return; + } + + if ($isAcl) { + $this->sendXml($response, S3Xml::accessControlPolicy()); + return; + } + if ($isUnsupported) { + $this->sendXml($response, S3Xml::error('NotImplemented', 'This S3 feature is not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED); + return; + } + + $size = (int) $file->getAttribute('sizeOriginal', 0); + $range = $this->range($request, $size); + $etag = '"' . $file->getAttribute('signature', '') . '"'; + + if ($ifNoneMatch === $etag) { + $response->setStatusCode(Response::STATUS_CODE_NOT_MODIFIED)->send(''); + return; + } + if ($ifMatch !== '' && $ifMatch !== $etag) { + $this->sendXml($response, S3Xml::error('PreconditionFailed', 'At least one of the pre-conditions you specified did not hold.', $request->getURI()), Response::STATUS_CODE_PRECONDITION_FAILED); + return; + } + + if ($range === false) { + $response->addHeader('Content-Range', 'bytes */' . $size); + $this->sendXml($response, S3Xml::error('InvalidRange', 'The requested range is not satisfiable.', $request->getURI()), Response::STATUS_CODE_REQUESTED_RANGE_NOT_SATISFIABLE); + return; + } + + $response + ->setContentType($file->getAttribute('mimeType', 'application/octet-stream')) + ->addHeader('ETag', $etag) + ->addHeader('Accept-Ranges', 'bytes'); + + $lastModified = \strtotime($file->getUpdatedAt() ?: $file->getCreatedAt()); + if ($lastModified !== false) { + $response->addHeader('Last-Modified', \gmdate('D, d M Y H:i:s', $lastModified) . ' GMT'); + } + + $metadata = $file->getAttribute('metadata', []); + foreach (($metadata['userMetadata'] ?? []) as $name => $value) { + $response->addHeader('x-amz-meta-' . $name, (string) $value); + } + $sse = (string) ($metadata['serverSideEncryption'] ?? ''); + if ($sse !== '') { + $response->addHeader('x-amz-server-side-encryption', $sse); + } + + foreach ([ + 'cacheControl' => 'Cache-Control', + 'contentDisposition' => 'Content-Disposition', + 'contentEncoding' => 'Content-Encoding', + 'contentLanguage' => 'Content-Language', + 'expires' => 'Expires', + ] as $attribute => $header) { + $value = (string) ($metadata[$attribute] ?? ''); + if ($value !== '') { + $response->addHeader($header, $value); + } + } + + foreach ([ + 'response-cache-control' => 'Cache-Control', + 'response-content-disposition' => 'Content-Disposition', + 'response-content-encoding' => 'Content-Encoding', + 'response-content-language' => 'Content-Language', + 'response-expires' => 'Expires', + ] as $parameter => $header) { + $value = $this->query($request, $parameter); + if ($value !== '') { + $response->setHeader($header, $value); + } + } + + $responseContentType = $this->query($request, 'response-content-type'); + if ($responseContentType !== '') { + $response->setContentType($responseContentType); + } + + if ($range !== null) { + [$start, $end] = $range; + $response + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT) + ->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size) + ->addHeader('Content-Length', (string) ($end - $start + 1)); + } else { + $response->addHeader('Content-Length', (string) $size); + } + + if ($isHead) { + $response->send(''); + return; + } + + if ($range !== null) { + [$start, $end] = $range; + $body = \substr((string) $body, $start, $end - $start + 1); + } + + $response->send((string) $body); + } + + /** + * @return array{0: int, 1: int}|false|null + */ + private function range(Request $request, int $size): array|false|null + { + $header = \trim($request->getHeaderLine('range')); + if ($header === '') { + return null; + } + + if (!\preg_match('/^bytes=(\d*)-(\d*)$/', $header, $matches)) { + return false; + } + if ($size <= 0) { + return false; + } + + $start = $matches[1]; + $end = $matches[2]; + if ($start === '' && $end === '') { + return false; + } + + if ($start === '') { + $suffix = (int) $end; + if ($suffix <= 0) { + return false; + } + + return [\max(0, $size - $suffix), \max(0, $size - 1)]; + } + + $start = (int) $start; + $end = $end === '' ? $size - 1 : (int) $end; + + if ($start >= $size || $start > $end) { + return false; + } + + return [$start, \min($end, $size - 1)]; + } +} diff --git a/src/Appwrite/Platform/Modules/S3/Http/Shutdown/Events.php b/src/Appwrite/Platform/Modules/S3/Http/Shutdown/Events.php new file mode 100644 index 0000000000..8a99193a3e --- /dev/null +++ b/src/Appwrite/Platform/Modules/S3/Http/Shutdown/Events.php @@ -0,0 +1,103 @@ +setType(Action::TYPE_SHUTDOWN) + ->groups(['s3']) + ->desc('S3 gateway event fan-out') + ->inject('route') + ->inject('response') + ->inject('project') + ->inject('queueForEvents') + ->inject('publisherForFunctions') + ->inject('queueForWebhooks') + ->inject('queueForRealtime') + ->inject('dbForProject') + ->inject('eventProcessor') + ->callback($this->action(...)); + } + + public function action(Route $route, Response $response, Document $project, Event $queueForEvents, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, EventProcessor $eventProcessor): void + { + if (empty($queueForEvents->getEvent())) { + return; + } + + if (empty($queueForEvents->getPayload())) { + $queueForEvents->setPayload($response->getPayload()); + } + + // Get project and function/webhook events (cached) + $functionsEvents = $eventProcessor->getFunctionsEvents($project, $dbForProject); + $webhooksEvents = $eventProcessor->getWebhooksEvents($project); + + // Generate events for this operation + $generatedEvents = Event::generateEvents( + $queueForEvents->getEvent(), + $queueForEvents->getParams() + ); + + $allowedOnConsole = !empty(\array_intersect($route->getGroups(), Realtime::CONSOLE_ALLOWLIST)); + if ($project->getId() !== 'console' || $allowedOnConsole) { + $queueForRealtime + ->from($queueForEvents) + ->trigger(); + } + + // Only trigger functions if there are matching function events + if (!empty($functionsEvents)) { + foreach ($generatedEvents as $event) { + if (isset($functionsEvents[$event])) { + $publisherForFunctions->enqueue(FunctionMessage::fromEvent( + event: $queueForEvents->getEvent(), + params: $queueForEvents->getParams(), + project: $queueForEvents->getProject(), + user: $queueForEvents->getUser(), + userId: $queueForEvents->getUserId(), + payload: $queueForEvents->getPayload(), + platform: $queueForEvents->getPlatform(), + )); + break; + } + } + } + + // Only trigger webhooks if there are matching webhook events + if (!empty($webhooksEvents)) { + foreach ($generatedEvents as $event) { + if (isset($webhooksEvents[$event])) { + $queueForWebhooks + ->from($queueForEvents) + ->trigger(); + break; + } + } + } + } +} diff --git a/src/Appwrite/Platform/Modules/S3/Http/Shutdown/Usage.php b/src/Appwrite/Platform/Modules/S3/Http/Shutdown/Usage.php new file mode 100644 index 0000000000..ef965e87d0 --- /dev/null +++ b/src/Appwrite/Platform/Modules/S3/Http/Shutdown/Usage.php @@ -0,0 +1,77 @@ +setType(Action::TYPE_SHUTDOWN) + ->groups(['s3']) + ->desc('S3 gateway usage metering') + ->inject('request') + ->inject('response') + ->inject('project') + ->inject('user') + ->inject('usage') + ->inject('publisherForUsage') + ->inject('authorization') + ->inject('bus') + ->callback($this->action(...)); + } + + public function action(Request $request, Response $response, Document $project, User $user, UsageContext $usage, UsagePublisher $publisherForUsage, Authorization $authorization, Bus $bus): void + { + if ($project->isEmpty() || $project->getId() === 'console') { + return; + } + + $usage->setStatus($response->getStatusCode()); + if ($usage->getResourcePath() === '') { + $usage->fillMissingResource('project', $project->getId(), (string) $project->getSequence()); + } + + // The usage bus listener adds METRIC_NETWORK_REQUESTS/INBOUND/OUTBOUND, + // exactly as on the `api` group. The SigV4 key never becomes the + // `apiKey` resource, so per-key disabled-metrics filtering cannot + // apply here. + if (!$user->isPrivileged($authorization->getRoles())) { + $bus->dispatch(new RequestCompleted( + project: $project->getArrayCopy(), + request: $request, + response: $response, + )); + } + + if (!$usage->isEmpty()) { + $publisherForUsage->enqueue(new UsageMessage( + project: $project, + metrics: $usage->getMetrics(), + reduce: $usage->getReduce(), + )); + } + } +} diff --git a/src/Appwrite/Platform/Modules/S3/Module.php b/src/Appwrite/Platform/Modules/S3/Module.php new file mode 100644 index 0000000000..5f4a4de2f2 --- /dev/null +++ b/src/Appwrite/Platform/Modules/S3/Module.php @@ -0,0 +1,14 @@ +addService('http', new Http()); + } +} diff --git a/src/Appwrite/Platform/Modules/S3/Requests/AwsChunked.php b/src/Appwrite/Platform/Modules/S3/Requests/AwsChunked.php new file mode 100644 index 0000000000..63a99ef831 --- /dev/null +++ b/src/Appwrite/Platform/Modules/S3/Requests/AwsChunked.php @@ -0,0 +1,128 @@ +[;chunk-signature=…]\r\n\r\n` chunks, terminated by a + * zero-size chunk and optional trailing headers. Storing the raw payload + * would persist the framing bytes as object content. + * + * Per-chunk signatures are stripped without verification — the request is + * already authenticated by the seed signature over the canonical request, + * which pins the STREAMING-* payload mode. Integrity is checked through the + * declared decoded length and, when present, the CRC32 trailer checksum. + */ +class AwsChunked +{ + public static function applies(string $contentSha256, string $contentEncoding): bool + { + if (\str_starts_with(\strtoupper(\trim($contentSha256)), 'STREAMING-')) { + return true; + } + + foreach (\explode(',', $contentEncoding) as $encoding) { + if (\strtolower(\trim($encoding)) === 'aws-chunked') { + return true; + } + } + + return false; + } + + public static function decode(string $payload, ?int $declaredLength = null): string + { + $decoded = ''; + $offset = 0; + $length = \strlen($payload); + + while (true) { + if ($offset >= $length) { + throw self::malformed('Missing final zero-size chunk.'); + } + + $lineEnd = \strpos($payload, "\r\n", $offset); + if ($lineEnd === false) { + throw self::malformed('Chunk header is not CRLF-terminated.'); + } + + $sizeHex = \explode(';', \substr($payload, $offset, $lineEnd - $offset), 2)[0]; + if ($sizeHex === '' || \strlen($sizeHex) > 8 || \preg_match('/^[0-9a-fA-F]+$/', $sizeHex) !== 1) { + throw self::malformed('Chunk size is not hexadecimal.'); + } + + $size = (int) \hexdec($sizeHex); + $offset = $lineEnd + 2; + + if ($size === 0) { + self::verifyTrailers(self::trailers($payload, $offset), $decoded); + break; + } + + if ($offset + $size > $length) { + throw self::malformed('Chunk data is truncated.'); + } + + $decoded .= \substr($payload, $offset, $size); + $offset += $size; + + if (\substr($payload, $offset, 2) !== "\r\n") { + throw self::malformed('Chunk data is not CRLF-terminated.'); + } + $offset += 2; + } + + if ($declaredLength !== null && \strlen($decoded) !== $declaredLength) { + throw self::malformed('Decoded length does not match x-amz-decoded-content-length.'); + } + + return $decoded; + } + + /** + * @return array + */ + private static function trailers(string $payload, int $offset): array + { + $trailers = []; + $length = \strlen($payload); + + while ($offset < $length) { + $lineEnd = \strpos($payload, "\r\n", $offset); + $line = $lineEnd === false ? \substr($payload, $offset) : \substr($payload, $offset, $lineEnd - $offset); + $offset = $lineEnd === false ? $length : $lineEnd + 2; + + if (\trim($line) === '') { + continue; + } + + [$name, $value] = \array_pad(\explode(':', $line, 2), 2, ''); + $trailers[\strtolower(\trim($name))] = \trim($value); + } + + return $trailers; + } + + /** + * @param array $trailers + */ + private static function verifyTrailers(array $trailers, string $decoded): void + { + // CRC32 is the SDK default checksum algorithm; other algorithms + // (crc32c, sha1, sha256, crc64nvme) are accepted but not verified. + $crc32 = $trailers['x-amz-checksum-crc32'] ?? ''; + if ($crc32 !== '' && \base64_encode(\hash('crc32b', $decoded, true)) !== $crc32) { + throw self::malformed('CRC32 trailer checksum does not match the decoded payload.'); + } + } + + private static function malformed(string $reason): AppwriteException + { + return new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Malformed aws-chunked payload. ' . $reason); + } +} diff --git a/src/Appwrite/Platform/Modules/S3/Responses/S3Xml.php b/src/Appwrite/Platform/Modules/S3/Responses/S3Xml.php new file mode 100644 index 0000000000..9b2b980e4e --- /dev/null +++ b/src/Appwrite/Platform/Modules/S3/Responses/S3Xml.php @@ -0,0 +1,360 @@ + $code, + 'Message' => $message, + 'Resource' => $resource, + 'RequestId' => $requestId ?: \bin2hex(\random_bytes(8)), + ]); + } + + /** + * @param array $buckets + */ + public static function listBuckets(array $buckets): string + { + $items = ''; + foreach ($buckets as $bucket) { + $items .= self::element('Bucket', [ + 'Name' => $bucket->getId(), + 'CreationDate' => self::date($bucket->getCreatedAt()), + ]); + } + + return self::rawDocument('ListAllMyBucketsResult', '' . $items . ''); + } + + public static function accessControlPolicy(): string + { + return self::rawDocument('AccessControlPolicy', self::element('Owner', [ + 'ID' => 'appwrite', + 'DisplayName' => 'appwrite', + ]) . '' . self::element('Grant', [ + 'Grantee' => [ + 'ID' => 'appwrite', + 'DisplayName' => 'appwrite', + ], + 'Permission' => 'FULL_CONTROL', + ]) . ''); + } + + public static function bucketLocation(string $region = 'us-east-1'): string + { + return self::rawDocument('LocationConstraint', $region === 'us-east-1' ? '' : self::escape($region)); + } + + public static function copyObject(string $etag, ?string $lastModified = null): string + { + return self::document('CopyObjectResult', [ + 'LastModified' => self::date($lastModified), + 'ETag' => '"' . $etag . '"', + ]); + } + + public static function copyPart(string $etag, ?string $lastModified = null): string + { + return self::document('CopyPartResult', [ + 'LastModified' => self::date($lastModified), + 'ETag' => '"' . $etag . '"', + ]); + } + + /** + * @param array $keys + */ + public static function deleteObjects(array $keys, bool $quiet = false): string + { + $items = ''; + if (!$quiet) { + foreach ($keys as $key) { + $items .= self::element('Deleted', ['Key' => $key]); + } + } + + return self::rawDocument('DeleteResult', $items); + } + + /** + * @param array $uploads + */ + public static function listMultipartUploads(string $bucketId, array $uploads): string + { + $items = ''; + foreach ($uploads as $upload) { + $items .= self::element('Upload', [ + 'Key' => $upload['key'], + 'UploadId' => $upload['uploadId'], + 'Initiated' => self::date($upload['initiated']), + ]); + } + + return self::rawDocument('ListMultipartUploadsResult', self::element('Bucket', $bucketId) . self::element('IsTruncated', 'false') . $items); + } + + /** + * @param array $files + */ + public static function listObjects(string $bucketId, array $files, string $prefix = '', string $delimiter = '', int $maxKeys = 1000, string $marker = ''): string + { + $maxKeys = \max(0, \min(1000, $maxKeys)); + $entries = []; + $commonPrefixes = []; + foreach ($files as $file) { + $key = self::fileKey($file); + if (($prefix !== '' && !\str_starts_with($key, $prefix)) || ($marker !== '' && \strcmp($key, $marker) <= 0)) { + continue; + } + + if ($delimiter !== '') { + $rest = \substr($key, \strlen($prefix)); + $position = \strpos($rest, $delimiter); + if ($position !== false) { + $commonPrefixes[$prefix . \substr($rest, 0, $position + \strlen($delimiter))] = true; + continue; + } + } + + $entries[$key] = ['type' => 'file', 'file' => $file]; + } + + foreach (\array_keys($commonPrefixes) as $commonPrefix) { + if ($marker !== '' && \strcmp($commonPrefix, $marker) <= 0) { + continue; + } + $entries[$commonPrefix] = ['type' => 'prefix', 'prefix' => $commonPrefix]; + } + + \ksort($entries, SORT_STRING); + $selected = \array_slice($entries, 0, $maxKeys, true); + $isTruncated = \count($entries) > $maxKeys; + $lastKey = $selected === [] ? '' : (string) \array_key_last($selected); + $items = ''; + foreach ($selected as $key => $entry) { + if ($entry['type'] === 'prefix') { + $items .= self::element('CommonPrefixes', ['Prefix' => (string) $entry['prefix']]); + continue; + } + + $file = $entry['file']; + $items .= self::element('Contents', [ + 'Key' => $key, + 'LastModified' => self::date($file->getUpdatedAt() ?: $file->getCreatedAt()), + 'ETag' => '"' . $file->getAttribute('signature', '') . '"', + 'Size' => (string) $file->getAttribute('sizeOriginal', 0), + 'StorageClass' => 'STANDARD', + ]); + } + + $body = self::elements([ + 'Name' => $bucketId, + 'Prefix' => $prefix, + 'Marker' => $marker, + 'MaxKeys' => (string) $maxKeys, + 'IsTruncated' => $isTruncated ? 'true' : 'false', + ]); + if ($delimiter !== '') { + $body .= self::element('Delimiter', $delimiter); + } + if ($isTruncated && $lastKey !== '') { + $body .= self::element('NextMarker', $lastKey); + } + + return self::rawDocument('ListBucketResult', $body . $items); + } + + /** + * @param array $files + */ + public static function listObjectsV2( + string $bucketId, + array $files, + string $prefix = '', + string $delimiter = '', + int $maxKeys = 1000, + string $continuationToken = '', + string $startAfter = '' + ): string { + $maxKeys = \max(0, \min(1000, $maxKeys)); + $after = $continuationToken !== '' ? self::decodeContinuationToken($continuationToken) : $startAfter; + $entries = []; + $commonPrefixes = []; + + foreach ($files as $file) { + $key = self::fileKey($file); + if ($prefix !== '' && !\str_starts_with($key, $prefix)) { + continue; + } + if ($after !== '' && \strcmp($key, $after) <= 0) { + continue; + } + + if ($delimiter !== '') { + $rest = \substr($key, \strlen($prefix)); + $position = \strpos($rest, $delimiter); + if ($position !== false) { + $commonPrefixes[$prefix . \substr($rest, 0, $position + \strlen($delimiter))] = true; + continue; + } + } + + $entries[$key] = ['type' => 'file', 'file' => $file]; + } + + foreach (\array_keys($commonPrefixes) as $commonPrefix) { + if ($after !== '' && \strcmp($commonPrefix, $after) <= 0) { + continue; + } + $entries[$commonPrefix] = ['type' => 'prefix', 'prefix' => $commonPrefix]; + } + + // PHP casts numeric keys to int; without SORT_STRING they sort numerically, + // which breaks the strcmp-based continuation filter and skips objects. + \ksort($entries, SORT_STRING); + + $selected = \array_slice($entries, 0, $maxKeys, true); + $isTruncated = \count($entries) > $maxKeys; + $lastKey = $selected === [] ? '' : (string) \array_key_last($selected); + + $items = ''; + foreach ($selected as $entry) { + if ($entry['type'] === 'prefix') { + $items .= self::element('CommonPrefixes', [ + 'Prefix' => (string) $entry['prefix'], + ]); + continue; + } + + $file = $entry['file']; + $items .= self::element('Contents', [ + 'Key' => self::fileKey($file), + 'LastModified' => self::date($file->getUpdatedAt() ?: $file->getCreatedAt()), + 'ETag' => '"' . $file->getAttribute('signature', '') . '"', + 'Size' => (string) $file->getAttribute('sizeOriginal', 0), + 'StorageClass' => 'STANDARD', + ]); + } + + $body = self::elements([ + 'Name' => $bucketId, + 'Prefix' => $prefix, + 'KeyCount' => (string) \count($selected), + 'MaxKeys' => (string) $maxKeys, + 'IsTruncated' => $isTruncated ? 'true' : 'false', + ]); + + if ($delimiter !== '') { + $body .= self::element('Delimiter', $delimiter); + } + if ($continuationToken !== '') { + $body .= self::element('ContinuationToken', $continuationToken); + } + if ($startAfter !== '') { + $body .= self::element('StartAfter', $startAfter); + } + if ($isTruncated && $lastKey !== '') { + $body .= self::element('NextContinuationToken', self::encodeContinuationToken($lastKey)); + } + + return self::rawDocument('ListBucketResult', $body . $items); + } + + public static function initiateMultipart(string $bucketId, string $key, string $uploadId): string + { + return self::document('InitiateMultipartUploadResult', [ + 'Bucket' => $bucketId, + 'Key' => $key, + 'UploadId' => $uploadId, + ]); + } + + public static function completeMultipart(string $bucketId, string $key, string $etag): string + { + return self::document('CompleteMultipartUploadResult', [ + 'Location' => '/' . $bucketId . '/' . $key, + 'Bucket' => $bucketId, + 'Key' => $key, + 'ETag' => '"' . $etag . '"', + ]); + } + + /** + * @param array $parts + */ + public static function listParts(string $bucketId, string $key, string $uploadId, array $parts): string + { + $items = ''; + foreach ($parts as $part) { + $items .= self::element('Part', [ + 'PartNumber' => (string) $part['partNumber'], + 'ETag' => '"' . $part['etag'] . '"', + 'Size' => (string) $part['size'], + ]); + } + + return self::rawDocument('ListPartsResult', self::elements([ + 'Bucket' => $bucketId, + 'Key' => $key, + 'UploadId' => $uploadId, + ]) . $items); + } + + private static function document(string $root, array $values): string + { + return self::rawDocument($root, self::elements($values)); + } + + private static function rawDocument(string $root, string $body): string + { + return '' . '<' . $root . '>' . $body . ''; + } + + private static function elements(array $values): string + { + $xml = ''; + foreach ($values as $key => $value) { + $xml .= self::element((string) $key, \is_array($value) ? $value : (string) $value); + } + return $xml; + } + + private static function element(string $name, string|array $value): string + { + $content = \is_array($value) ? self::elements($value) : self::escape($value); + return '<' . $name . '>' . $content . ''; + } + + private static function fileKey(Document $file): string + { + return $file->getAttribute('folder', '') . $file->getAttribute('name', $file->getId()); + } + + private static function encodeContinuationToken(string $key): string + { + return \rtrim(\strtr(\base64_encode($key), '+/', '-_'), '='); + } + + private static function decodeContinuationToken(string $token): string + { + $decoded = \base64_decode(\strtr($token, '-_', '+/'), true); + return \is_string($decoded) ? $decoded : ''; + } + + private static function escape(string $value): string + { + return \htmlspecialchars($value, ENT_XML1 | ENT_COMPAT, 'UTF-8'); + } + + private static function date(?string $date): string + { + $time = \strtotime($date ?? ''); + return \gmdate('Y-m-d\TH:i:s.000\Z', $time === false ? \time() : $time); + } +} diff --git a/src/Appwrite/Platform/Modules/S3/Services/Http.php b/src/Appwrite/Platform/Modules/S3/Services/Http.php new file mode 100644 index 0000000000..87bb115b08 --- /dev/null +++ b/src/Appwrite/Platform/Modules/S3/Services/Http.php @@ -0,0 +1,26 @@ +type = Service::TYPE_HTTP; + + $this->addAction(Init::getName(), new Init()); + $this->addAction(Get::getName(), new Get()); + $this->addAction(Create::getName(), new Create()); + $this->addAction(Delete::getName(), new Delete()); + $this->addAction(ShutdownEvents::getName(), new ShutdownEvents()); + $this->addAction(ShutdownUsage::getName(), new ShutdownUsage()); + } +} diff --git a/tests/unit/Platform/Modules/S3/Auth/SignatureV4Test.php b/tests/unit/Platform/Modules/S3/Auth/SignatureV4Test.php new file mode 100644 index 0000000000..a84a483e7b --- /dev/null +++ b/tests/unit/Platform/Modules/S3/Auth/SignatureV4Test.php @@ -0,0 +1,324 @@ +signedRequest('GET', '/v1/s3/bucket/object.txt', '', $secret); + + $key = (new SignatureV4())->verify( + $request, + $this->project($secret, ['files.read']), + new Document(), + new User(), + ['files.read'] + ); + + $this->assertSame('project-test', $key->getProjectId()); + $this->assertContains('files.read', $key->getScopes()); + } + + public function testVerifiesUnsignedPayloadSignature(): void + { + $secret = 'test-api-key-secret'; + $request = $this->signedRequest('GET', '/v1/s3/bucket', '', $secret, 'UNSIGNED-PAYLOAD', 'delimiter=%2F&list-type=2&max-keys=100'); + + $key = (new SignatureV4())->verify( + $request, + $this->project($secret, ['files.read']), + new Document(), + new User(), + ['files.read'] + ); + + $this->assertSame('project-test', $key->getProjectId()); + } + + public function testRejectsExpiredPresignedRequest(): void + { + $secret = 'test-api-key-secret'; + $request = $this->presignedRequestAt( + 'GET', + '/v1/s3/bucket/object.txt', + $secret, + \gmdate('Ymd\THis\Z', \time() - 120), + 60, + ); + + $this->expectException(AppwriteException::class); + $this->expectExceptionMessage('presigned URL has expired'); + + (new SignatureV4())->verify( + $request, + $this->project($secret, ['files.read']), + new Document(), + new User(), + ['files.read'] + ); + } + public function testVerifiesOpenDalRangedGetSignature(): void + { + $secret = 'test-api-key-secret'; + $request = $this->signedRequest( + 'GET', + '/v1/s3/test/ai_history_fundamentals_quiz_quizizz.xlsx', + '', + $secret, + 'UNSIGNED-PAYLOAD', + additionalHeaders: ['range' => 'bytes=0-8218'], + ); + + $key = (new SignatureV4())->verify( + $request, + $this->project($secret, ['files.read']), + new Document(), + new User(), + ['files.read'] + ); + + $this->assertSame('project-test', $key->getProjectId()); + } + + public function testVerifiesOpenDalDeleteSignature(): void + { + $secret = 'test-api-key-secret'; + $request = $this->signedRequest( + 'DELETE', + '/v1/s3/test/ai_history_fundamentals_quiz_quizizz.xlsx', + '', + $secret, + 'UNSIGNED-PAYLOAD', + ); + + $key = (new SignatureV4())->verify( + $request, + $this->project($secret, ['files.write']), + new Document(), + new User(), + ['files.write'] + ); + + $this->assertSame('project-test', $key->getProjectId()); + } + + public function testRejectsValidSignatureWhenApiKeyMissingRequiredScope(): void + { + $secret = 'test-api-key-secret'; + $request = $this->signedRequest('PUT', '/v1/s3/bucket/object.txt', 'body', $secret); + + $this->expectException(AppwriteException::class); + $this->expectExceptionMessage('API key is missing required S3 storage scopes.'); + + (new SignatureV4())->verify( + $request, + $this->project($secret, ['files.read']), + new Document(), + new User(), + ['files.write'] + ); + } + + public function testRejectsInvalidSignature(): void + { + $secret = 'test-api-key-secret'; + $request = $this->signedRequest('GET', '/v1/s3/bucket/object.txt', '', $secret); + $request->headerMap['authorization'] = \preg_replace('/Signature=[0-9a-f]+/', 'Signature=deadbeef', $request->headerMap['authorization']); + + $this->expectException(AppwriteException::class); + $this->expectExceptionMessage('Signature does not match.'); + + (new SignatureV4())->verify( + $request, + $this->project($secret, ['files.read']), + new Document(), + new User(), + ['files.read'] + ); + } + + public function testRejectsExpiredSignatureTimestamp(): void + { + $secret = 'test-api-key-secret'; + $date = \gmdate('Ymd\THis\Z', \time() - 901); + $request = $this->signedRequest('GET', '/v1/s3/bucket/object.txt', '', $secret, date: $date); + + $this->expectException(AppwriteException::class); + $this->expectExceptionMessage('AWS Signature V4 timestamp is outside the allowed time window.'); + + (new SignatureV4())->verify( + $request, + $this->project($secret, ['files.read']), + new Document(), + new User(), + ['files.read'] + ); + } + + public function testRejectsFutureSignatureTimestamp(): void + { + $secret = 'test-api-key-secret'; + $date = \gmdate('Ymd\THis\Z', \time() + 901); + $request = $this->signedRequest('GET', '/v1/s3/bucket/object.txt', '', $secret, date: $date); + + $this->expectException(AppwriteException::class); + $this->expectExceptionMessage('AWS Signature V4 timestamp is outside the allowed time window.'); + + (new SignatureV4())->verify( + $request, + $this->project($secret, ['files.read']), + new Document(), + new User(), + ['files.read'] + ); + } + + private function project(string $secret, array $scopes): Document + { + return new Document([ + '$id' => 'project-test', + 'keys' => [ + [ + '$id' => 'key-test', + 'name' => 'S3 Test Key', + 'secret' => $secret, + 'scopes' => $scopes, + 'expire' => null, + ], + ], + ]); + } + + private function presignedRequestAt(string $method, string $uri, string $secret, string $date, int $expires): TestRequest + { + $shortDate = \substr($date, 0, 8); + $scope = "{$shortDate}/us-east-1/s3/aws4_request"; + $parameters = [ + 'X-Amz-Algorithm' => 'AWS4-HMAC-SHA256', + 'X-Amz-Credential' => 'project-test/' . $scope, + 'X-Amz-Date' => $date, + 'X-Amz-Expires' => (string) $expires, + 'X-Amz-SignedHeaders' => 'host', + ]; + $encoded = []; + foreach ($parameters as $name => $value) { + $encoded[] = \rawurlencode($name) . '=' . \rawurlencode($value); + } + \sort($encoded, SORT_STRING); + $canonicalQuery = \implode('&', $encoded); + $canonicalRequest = \implode("\n", [ + $method, + $uri, + $canonicalQuery, + "host:cloud.appwrite.test\n", + 'host', + 'UNSIGNED-PAYLOAD', + ]); + $stringToSign = "AWS4-HMAC-SHA256\n{$date}\n{$scope}\n" . \hash('sha256', $canonicalRequest); + $signature = \hash_hmac('sha256', $stringToSign, $this->signingKey($secret, $shortDate, 'us-east-1', 's3')); + $query = $canonicalQuery . '&X-Amz-Signature=' . $signature; + + return new TestRequest($method, $uri . '?' . $query, ['host' => 'cloud.appwrite.test'], '', $query); + } + + private function signedRequest(string $method, string $uri, string $body, string $secret, ?string $payloadHash = null, string $query = '', ?string $date = null, array $additionalHeaders = []): TestRequest + { + $date ??= \gmdate('Ymd\THis\Z'); + $shortDate = \substr($date, 0, 8); + $region = 'us-east-1'; + $service = 's3'; + $headers = \array_merge([ + 'host' => 'cloud.appwrite.test', + 'x-amz-content-sha256' => $payloadHash ?? \hash('sha256', $body), + 'x-amz-date' => $date, + ], $additionalHeaders); + \ksort($headers); + + $signedHeaders = \implode(';', \array_keys($headers)); + $canonicalHeaders = ''; + foreach ($headers as $name => $value) { + $canonicalHeaders .= $name . ':' . \preg_replace('/\s+/', ' ', \trim($value)) . "\n"; + } + $canonicalRequest = \implode("\n", [ + $method, + $uri, + $query, + $canonicalHeaders, + $signedHeaders, + $headers['x-amz-content-sha256'], + ]); + + $scope = "{$shortDate}/{$region}/{$service}/aws4_request"; + $stringToSign = "AWS4-HMAC-SHA256\n{$date}\n{$scope}\n" . \hash('sha256', $canonicalRequest); + $signature = \hash_hmac('sha256', $stringToSign, $this->signingKey($secret, $shortDate, $region, $service)); + + $headers['authorization'] = "AWS4-HMAC-SHA256 Credential=project-test/{$scope}, SignedHeaders={$signedHeaders}, Signature={$signature}"; + + return new TestRequest($method, $query === '' ? $uri : $uri . '?' . $query, $headers, $body, $query); + } + + private function signingKey(string $secret, string $date, string $region, string $service): string + { + $kDate = \hash_hmac('sha256', $date, 'AWS4' . $secret, true); + $kRegion = \hash_hmac('sha256', $region, $kDate, true); + $kService = \hash_hmac('sha256', $service, $kRegion, true); + return \hash_hmac('sha256', 'aws4_request', $kService, true); + } +} + +final class TestRequest extends Request +{ + /** + * @param array $headerMap + */ + public function __construct( + private readonly string $method, + public readonly string $uri, + public array $headerMap, + private readonly string $body, + public readonly string $query = '', + ) { + } + + public function getMethod(): string + { + return $this->method; + } + + public function getURI(): string + { + return $this->uri; + } + + public function getHeaderLine(string $key, string $default = ''): string + { + return $this->headerMap[\strtolower($key)] ?? $default; + } + + public function getRawPayload(): string + { + return $this->body; + } + + public function getServer(string $key, ?string $default = null): ?string + { + return $key === 'query_string' ? $this->query : $default; + } + + public function getSwooleRequest(): SwooleRequest + { + throw new \LogicException('Not used by this test.'); + } +} diff --git a/tests/unit/Platform/Modules/S3/Requests/AwsChunkedTest.php b/tests/unit/Platform/Modules/S3/Requests/AwsChunkedTest.php new file mode 100644 index 0000000000..8ed96a4f34 --- /dev/null +++ b/tests/unit/Platform/Modules/S3/Requests/AwsChunkedTest.php @@ -0,0 +1,105 @@ +assertTrue(AwsChunked::applies('STREAMING-UNSIGNED-PAYLOAD-TRAILER', '')); + $this->assertTrue(AwsChunked::applies('streaming-aws4-hmac-sha256-payload', '')); + $this->assertTrue(AwsChunked::applies('', 'aws-chunked')); + $this->assertTrue(AwsChunked::applies('', 'gzip, aws-chunked')); + + $this->assertFalse(AwsChunked::applies(\hash('sha256', 'body'), '')); + $this->assertFalse(AwsChunked::applies('UNSIGNED-PAYLOAD', '')); + $this->assertFalse(AwsChunked::applies('', 'gzip')); + } + + public function testDecodesUnsignedTrailerFraming(): void + { + $payload = "6\r\nhello \r\n" + . "5\r\nworld\r\n" + . "0\r\n" + . 'x-amz-checksum-crc32:' . \base64_encode(\hash('crc32b', 'hello world', true)) . "\r\n" + . "\r\n"; + + $this->assertSame('hello world', AwsChunked::decode($payload, 11)); + } + + public function testDecodesSignedChunkFraming(): void + { + $payload = "6;chunk-signature=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\nhello \r\n" + . "5;chunk-signature=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\r\nworld\r\n" + . "0;chunk-signature=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\r\n\r\n"; + + $this->assertSame('hello world', AwsChunked::decode($payload, 11)); + } + + public function testDecodesEmptyBody(): void + { + $this->assertSame('', AwsChunked::decode("0\r\n\r\n", 0)); + $this->assertSame('', AwsChunked::decode("0\r\n", null)); + } + + public function testDecodeIgnoresUnverifiedTrailerAlgorithms(): void + { + $payload = "3\r\nabc\r\n0\r\nx-amz-checksum-sha256:not-checked\r\n\r\n"; + + $this->assertSame('abc', AwsChunked::decode($payload, 3)); + } + + public function testRejectsDeclaredLengthMismatch(): void + { + $this->expectException(AppwriteException::class); + $this->expectExceptionMessage('x-amz-decoded-content-length'); + + AwsChunked::decode("3\r\nabc\r\n0\r\n\r\n", 4); + } + + public function testRejectsCrc32TrailerMismatch(): void + { + $this->expectException(AppwriteException::class); + $this->expectExceptionMessage('CRC32'); + + AwsChunked::decode("3\r\nabc\r\n0\r\n" . 'x-amz-checksum-crc32:' . \base64_encode(\hash('crc32b', 'abd', true)) . "\r\n\r\n", 3); + } + + public function testRejectsTruncatedChunkData(): void + { + $this->expectException(AppwriteException::class); + $this->expectExceptionMessage('truncated'); + + AwsChunked::decode("6\r\nhel", null); + } + + public function testRejectsMissingFinalChunk(): void + { + $this->expectException(AppwriteException::class); + $this->expectExceptionMessage('zero-size chunk'); + + AwsChunked::decode("3\r\nabc\r\n", null); + } + + public function testRejectsNonHexadecimalChunkSize(): void + { + $this->expectException(AppwriteException::class); + $this->expectExceptionMessage('hexadecimal'); + + AwsChunked::decode("zz\r\nabc\r\n0\r\n\r\n", null); + } + + public function testRejectsChunkDataWithoutCrlfTerminator(): void + { + $this->expectException(AppwriteException::class); + $this->expectExceptionMessage('CRLF'); + + AwsChunked::decode("3\r\nabcXX0\r\n\r\n", null); + } +} diff --git a/tests/unit/Platform/Modules/S3/Responses/S3XmlTest.php b/tests/unit/Platform/Modules/S3/Responses/S3XmlTest.php new file mode 100644 index 0000000000..1d66dca3df --- /dev/null +++ b/tests/unit/Platform/Modules/S3/Responses/S3XmlTest.php @@ -0,0 +1,166 @@ + 'bucket-a', + '$createdAt' => '2026-01-01T00:00:00.000+00:00', + ]), + ]); + + $this->assertStringStartsWith('', $xml); + $this->assertStringContainsString('', $xml); + $this->assertStringContainsString('bucket-a', $xml); + } + + public function testListObjectsEscapesKeys(): void + { + $xml = S3Xml::listObjects('bucket-a', [ + new Document([ + '$id' => 'filea', + '$createdAt' => '2026-01-01T00:00:00.000+00:00', + 'name' => 'folder/file&one.txt', + 'signature' => 'etag-a', + 'sizeOriginal' => 12, + ]), + ]); + + $this->assertStringContainsString('folder/file&one.txt', $xml); + } + + public function testListObjectsUsesFolderAndName(): void + { + $xml = S3Xml::listObjects('bucket-a', [ + $this->file('filea', 'file&one.txt', 'folder/'), + ]); + + $this->assertStringContainsString('folder/file&one.txt', $xml); + } + + public function testListObjectsSupportsDelimiterAndMarkerPagination(): void + { + $firstPage = S3Xml::listObjects('bucket-a', [ + $this->file('filea', 'readme.txt', 'docs/'), + $this->file('fileb', 'a.txt', 'photos/2025/'), + $this->file('filec', 'b.txt', 'photos/2026/'), + ], delimiter: '/', maxKeys: 1); + + $this->assertStringContainsString('docs/', $firstPage); + $this->assertStringContainsString('true', $firstPage); + $this->assertStringContainsString('docs/', $firstPage); + + $secondPage = S3Xml::listObjects('bucket-a', [ + $this->file('filea', 'readme.txt', 'docs/'), + $this->file('fileb', 'a.txt', 'photos/2025/'), + $this->file('filec', 'b.txt', 'photos/2026/'), + ], delimiter: '/', maxKeys: 1, marker: 'docs/'); + + $this->assertStringNotContainsString('docs/', $secondPage); + $this->assertStringContainsString('photos/', $secondPage); + $this->assertStringContainsString('false', $secondPage); + } + + public function testListObjectsV2SupportsPrefixPaginationAndDelimiter(): void + { + $xml = S3Xml::listObjectsV2('bucket-a', [ + $this->file('filea', 'readme.txt', 'docs/'), + $this->file('fileb', 'c.txt', 'photos/2025/'), + $this->file('filec', 'a.txt', 'photos/2026/'), + $this->file('filed', 'b.txt', 'photos/2026/'), + ], prefix: 'photos/', delimiter: '/', maxKeys: 1); + + $this->assertStringContainsString('', $xml); + $this->assertStringContainsString('photos/', $xml); + $this->assertStringContainsString('1', $xml); + $this->assertStringContainsString('1', $xml); + $this->assertStringContainsString('true', $xml); + $this->assertStringContainsString('photos/2025/', $xml); + $this->assertStringContainsString('', $xml); + } + + public function testListObjectsV2ContinuationTokenStartsAfterLastKey(): void + { + $firstPage = S3Xml::listObjectsV2('bucket-a', [ + $this->file('filea', 'a.txt'), + $this->file('fileb', 'b.txt'), + ], maxKeys: 1); + + preg_match('/([^<]+)<\/NextContinuationToken>/', $firstPage, $matches); + $this->assertNotEmpty($matches[1] ?? ''); + + $secondPage = S3Xml::listObjectsV2('bucket-a', [ + $this->file('filea', 'a.txt'), + $this->file('fileb', 'b.txt'), + ], continuationToken: $matches[1]); + + $this->assertStringNotContainsString('a.txt', $secondPage); + $this->assertStringContainsString('b.txt', $secondPage); + $this->assertStringContainsString('false', $secondPage); + } + + public function testListObjectsV2OrdersAndPaginatesNumericKeysLexicographically(): void + { + $files = [ + $this->file('filea', '9'), + $this->file('fileb', '10'), + ]; + + $firstPage = S3Xml::listObjectsV2('bucket-a', $files, maxKeys: 1); + $this->assertStringContainsString('10', $firstPage); + $this->assertStringNotContainsString('9', $firstPage); + $this->assertStringContainsString('true', $firstPage); + + preg_match('/([^<]+)<\/NextContinuationToken>/', $firstPage, $matches); + $this->assertNotEmpty($matches[1] ?? ''); + + $secondPage = S3Xml::listObjectsV2('bucket-a', $files, maxKeys: 1, continuationToken: $matches[1]); + $this->assertStringContainsString('9', $secondPage); + $this->assertStringContainsString('false', $secondPage); + } + + public function testMultipartResponses(): void + { + $init = S3Xml::initiateMultipart('bucket-a', 'object.txt', 'upload-id'); + $parts = S3Xml::listParts('bucket-a', 'object.txt', 'upload-id', [ + ['partNumber' => 1, 'etag' => 'etag-1', 'size' => 10], + ]); + $complete = S3Xml::completeMultipart('bucket-a', 'object.txt', 'final-etag'); + + $this->assertStringContainsString('upload-id', $init); + $this->assertStringContainsString('1', $parts); + $this->assertStringContainsString('"final-etag"', $complete); + } + + public function testAccessControlPolicyRendersNestedGrantee(): void + { + $xml = S3Xml::accessControlPolicy(); + + $this->assertStringContainsString('appwriteappwrite', $xml); + $this->assertStringContainsString('appwriteappwrite', $xml); + $this->assertStringContainsString('FULL_CONTROL', $xml); + $this->assertStringNotContainsString('Array', $xml); + } + + private function file(string $id, string $name, string $folder = ''): Document + { + return new Document([ + '$id' => $id, + '$createdAt' => '2026-01-01T00:00:00.000+00:00', + 'name' => $name, + 'folder' => $folder, + 'signature' => 'etag-' . $id, + 'sizeOriginal' => 12, + ]); + } +}