feat: add S3-compatible storage service

This commit is contained in:
Torsten Dittmann committed 2026-09-07 19:01:50 +00:00
1 parent 45a0168a77
commit 9faf165613
17 files changed
+3425 -1

No files matched your search

+19 -1
View File
@@ -621,11 +621,29 @@ return function (Container $context): void {
$projectId = (string) $request->getQuery('project', '');
}
$route = $utopia->match($request)?->route;
// S3 uses the Appwrite project ID as its SigV4 access key. Header-signed
// requests carry the credential scope in Authorization; presigned URLs
// carry it in the X-Amz-Credential query parameter.
if ($projectId === '' && \in_array('s3', $route?->getGroups() ?? [], true)) {
$credential = '';
$authorizationHeader = $request->getHeaderLine('authorization', '');
if (\preg_match('/Credential=([^\s,]+)/', $authorizationHeader, $matches) === 1) {
$credential = $matches[1];
} else {
$credential = $request->getQuery('X-Amz-Credential', '');
}
if (\is_string($credential)) {
$projectId = \explode('/', $credential, 2)[0];
}
}
// Backwards compatibility for new services, originally project resources
// These endpoints moved from /v1/projects/:projectId/<resource> to /v1/<resource>
// When accessed via the old alias path, extract projectId from the URI
$deprecatedProjectPathPrefix = '/v1/projects/';
$route = $utopia->match($request)?->route;
if (!empty($route)) {
$isDeprecatedAlias = $projectIdFromPath !== '' &&
!\str_starts_with($route->getPath(), $deprecatedProjectPathPrefix);
+2
View File
@@ -17,6 +17,7 @@ use Appwrite\Platform\Modules\Presences;
use Appwrite\Platform\Modules\Project;
use Appwrite\Platform\Modules\Projects;
use Appwrite\Platform\Modules\Proxy;
use Appwrite\Platform\Modules\S3;
use Appwrite\Platform\Modules\Sites;
use Appwrite\Platform\Modules\Storage;
use Appwrite\Platform\Modules\Teams;
@@ -43,6 +44,7 @@ class Appwrite extends Platform
$this->addModule(new Sites\Module());
$this->addModule(new Console\Module());
$this->addModule(new Proxy\Module());
$this->addModule(new S3\Module());
$this->addModule(new Teams\Module());
$this->addModule(new Tokens\Module());
$this->addModule(new Users\Module());
@@ -0,0 +1,337 @@
<?php
namespace Appwrite\Platform\Modules\S3\Auth;
use Appwrite\Auth\Key;
use Appwrite\Extend\Exception;
use Appwrite\Utopia\Database\Documents\User;
use Utopia\Database\Document;
use Utopia\Http\Adapter\Swoole\Request;
class SignatureV4
{
private const MAX_CLOCK_SKEW = 900;
private const MAX_PRESIGNED_EXPIRY = 604800;
/**
* The AWS access key ID is the Appwrite project ID; the AWS secret is an Appwrite API key secret.
*/
public function verify(Request $request, Document $project, Document $team, User $user, array $requiredScopes): Key
{
$authorization = $request->getHeaderLine('authorization');
$queryAuthentication = $this->queryAuthentication($request);
if ($authorization !== '' && $queryAuthentication !== null) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Multiple AWS Signature V4 authentication methods were provided.');
}
if ($queryAuthentication !== null) {
$algorithm = $queryAuthentication['X-Amz-Algorithm'] ?? '';
if ($algorithm !== 'AWS4-HMAC-SHA256') {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Unsupported AWS Signature V4 algorithm.');
}
$credential = \explode('/', $queryAuthentication['X-Amz-Credential'] ?? '');
$signedHeaders = \explode(';', $queryAuthentication['X-Amz-SignedHeaders'] ?? '');
$signature = $queryAuthentication['X-Amz-Signature'] ?? '';
$date = $queryAuthentication['X-Amz-Date'] ?? '';
$expires = $queryAuthentication['X-Amz-Expires'] ?? '';
$presigned = true;
} else {
if (!\str_starts_with($authorization, 'AWS4-HMAC-SHA256 ')) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Missing AWS Signature V4 authentication.');
}
$parts = $this->parseAuthorization($authorization);
$credential = \explode('/', $parts['Credential'] ?? '');
$signedHeaders = \explode(';', $parts['SignedHeaders'] ?? '');
$signature = $parts['Signature'] ?? '';
$date = $request->getHeaderLine('x-amz-date') ?: $request->getHeaderLine('date');
$expires = null;
$presigned = false;
}
if (
\count($credential) !== 5
|| $credential[1] === ''
|| $credential[2] === ''
|| $credential[3] !== 's3'
|| $credential[4] !== 'aws4_request'
) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid AWS Signature V4 credential scope.');
}
$projectId = $credential[0];
if ($projectId === '') {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Missing AWS credential project ID.');
}
if ($projectId !== $project->getId()) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS credential project ID does not belong to the selected project.');
}
$normalizedSignedHeaders = \array_values(\array_filter(\array_map(
fn (string $header): string => \strtolower(\trim($header)),
$signedHeaders
)));
$sortedSignedHeaders = $normalizedSignedHeaders;
\sort($sortedSignedHeaders, SORT_STRING);
if (
$signature === ''
|| $date === ''
|| ($presigned && \preg_match('/^[a-f0-9]{64}$/D', $signature) !== 1)
|| ($presigned && !\in_array('host', $normalizedSignedHeaders, true))
|| ($presigned && $normalizedSignedHeaders !== $sortedSignedHeaders)
) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Incomplete AWS Signature V4 authentication.');
}
if ($presigned) {
$this->validatePresignedDate($date, $credential[1], (string) $expires);
} else {
$this->validateDate($date, $credential[1]);
}
$canonicalRequest = $this->canonicalRequest($request, $normalizedSignedHeaders, $presigned);
$scope = \implode('/', \array_slice($credential, 1, 4));
$stringToSign = "AWS4-HMAC-SHA256\n{$date}\n{$scope}\n" . \hash('sha256', $canonicalRequest);
foreach ($this->keySecrets($project) as $secret) {
$signingKey = $this->signingKey($secret, $credential[1], $credential[2], $credential[3]);
$expected = \hash_hmac('sha256', $stringToSign, $signingKey);
if (!\hash_equals($expected, $signature)) {
continue;
}
$apiKey = Key::decode($project, $team, $user, $secret);
if ($apiKey->getRole() !== User::ROLE_KEYS || $apiKey->isExpired()) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid or expired API key.');
}
if (empty(\array_intersect($requiredScopes, $apiKey->getScopes()))) {
throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, 'API key is missing required S3 storage scopes.');
}
return $apiKey;
}
throw new Exception(Exception::USER_UNAUTHORIZED, 'Signature does not match.');
}
private function validatePresignedDate(string $date, string $credentialDate, string $expires): void
{
if ($expires === '' || !\ctype_digit($expires)) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid AWS Signature V4 presigned URL expiry.');
}
$expires = (int) $expires;
if ($expires < 1 || $expires > self::MAX_PRESIGNED_EXPIRY) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 presigned URL expiry is outside the allowed range.');
}
$timestamp = $this->timestamp($date);
if ($timestamp->format('Ymd') !== $credentialDate) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 credential date does not match request timestamp.');
}
$now = \time();
if ($timestamp->getTimestamp() - $now > self::MAX_CLOCK_SKEW) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 timestamp is outside the allowed time window.');
}
if ($now > $timestamp->getTimestamp() + $expires) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 presigned URL has expired.');
}
}
private function validateDate(string $date, string $credentialDate): void
{
$timestamp = \DateTimeImmutable::createFromFormat('!Ymd\THis\Z', $date, new \DateTimeZone('UTC'));
if ($timestamp === false) {
$parsed = \strtotime($date);
if ($parsed === false) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid AWS Signature V4 timestamp.');
}
$timestamp = (new \DateTimeImmutable('@' . $parsed))->setTimezone(new \DateTimeZone('UTC'));
}
if ($timestamp->format('Ymd') !== $credentialDate) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 credential date does not match request timestamp.');
}
if (\abs(\time() - $timestamp->getTimestamp()) > self::MAX_CLOCK_SKEW) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 timestamp is outside the allowed time window.');
}
}
private function timestamp(string $date): \DateTimeImmutable
{
$timestamp = \DateTimeImmutable::createFromFormat('!Ymd\THis\Z', $date, new \DateTimeZone('UTC'));
$errors = \DateTimeImmutable::getLastErrors();
if (
$timestamp === false
|| (\is_array($errors) && ($errors['warning_count'] > 0 || $errors['error_count'] > 0))
|| $timestamp->format('Ymd\THis\Z') !== $date
) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid AWS Signature V4 timestamp.');
}
return $timestamp;
}
/**
* @return array<string>
*/
private function keySecrets(Document $project): array
{
$secrets = [];
foreach ($project->getAttribute('keys', []) as $key) {
$secret = $key instanceof Document
? $key->getAttribute('secret', '')
: (string) ($key['secret'] ?? '');
if ($secret !== '') {
$secrets[] = $secret;
}
}
return $secrets;
}
private function parseAuthorization(string $authorization): array
{
$authorization = \substr($authorization, \strlen('AWS4-HMAC-SHA256 '));
$result = [];
foreach (\explode(',', $authorization) as $part) {
[$key, $value] = \array_pad(\explode('=', \trim($part), 2), 2, '');
$result[$key] = $value;
}
return $result;
}
/**
* @return array<string, string>|null
*/
private function queryAuthentication(Request $request): ?array
{
$authenticationParameters = [
'X-Amz-Algorithm',
'X-Amz-Credential',
'X-Amz-Date',
'X-Amz-Expires',
'X-Amz-Signature',
'X-Amz-SignedHeaders',
];
$parameters = [];
$hasAuthenticationParameter = false;
foreach ($this->queryPairs($request) as [$name, $value]) {
if (!\in_array($name, $authenticationParameters, true)) {
continue;
}
$hasAuthenticationParameter = true;
if (\array_key_exists($name, $parameters)) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Duplicate AWS Signature V4 authentication parameter.');
}
$parameters[$name] = $value;
}
if (!$hasAuthenticationParameter) {
return null;
}
foreach ($authenticationParameters as $parameter) {
if (!\array_key_exists($parameter, $parameters)) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Incomplete AWS Signature V4 query authentication.');
}
}
return $parameters;
}
private function canonicalRequest(Request $request, array $signedHeaders, bool $presigned): string
{
$headers = [];
foreach ($signedHeaders as $header) {
$header = \strtolower(\trim($header));
if ($header === '') {
continue;
}
$headers[$header] = \preg_replace('/\s+/', ' ', \trim($request->getHeaderLine($header)));
}
\ksort($headers);
$canonicalHeaders = '';
foreach ($headers as $key => $value) {
$canonicalHeaders .= $key . ':' . $value . "\n";
}
$payloadHash = $presigned ? 'UNSIGNED-PAYLOAD' : $request->getHeaderLine('x-amz-content-sha256');
$payloadHash = $payloadHash === '' ? \hash('sha256', $request->getRawPayload()) : $payloadHash;
return \implode("\n", [
$request->getMethod(),
$this->canonicalUri($request),
$this->canonicalQuery($request, $presigned),
$canonicalHeaders,
\implode(';', \array_keys($headers)),
$payloadHash,
]);
}
private function canonicalUri(Request $request): string
{
$path = \parse_url($request->getURI(), PHP_URL_PATH);
return $path === null || $path === false || $path === '' ? '/' : $path;
}
private function canonicalQuery(Request $request, bool $presigned): string
{
$pairs = [];
foreach ($this->queryPairs($request) as [$key, $value]) {
if ($presigned && $key === 'X-Amz-Signature') {
continue;
}
$pairs[] = [\rawurlencode($key), \rawurlencode($value)];
}
\usort($pairs, function (array $left, array $right): int {
$keyOrder = \strcmp($left[0], $right[0]);
return $keyOrder === 0 ? \strcmp($left[1], $right[1]) : $keyOrder;
});
return \implode('&', \array_map(fn (array $pair): string => $pair[0] . '=' . $pair[1], $pairs));
}
/**
* @return array<int, array{0: string, 1: string}>
*/
private function queryPairs(Request $request): array
{
$query = $request->getServer('query_string', '') ?: (\parse_url($request->getURI(), PHP_URL_QUERY) ?: '');
if ($query === '') {
return [];
}
$pairs = [];
foreach (\explode('&', $query) as $part) {
[$name, $value] = \array_pad(\explode('=', $part, 2), 2, '');
$pairs[] = [\rawurldecode($name), \rawurldecode($value)];
}
return $pairs;
}
private function signingKey(string $secret, string $date, string $region, string $service): string
{
$kDate = \hash_hmac('sha256', $date, 'AWS4' . $secret, true);
$kRegion = \hash_hmac('sha256', $region, $kDate, true);
$kService = \hash_hmac('sha256', $service, $kRegion, true);
return \hash_hmac('sha256', 'aws4_request', $kService, true);
}
}
@@ -0,0 +1,109 @@
<?php
namespace Appwrite\Platform\Modules\S3\Http;
use Appwrite\Event\Event;
use Appwrite\Geo\Geo;
use Appwrite\Network\Cors;
use Appwrite\Usage\Context as UsageContext;
use Appwrite\Utopia\Database\Documents\User;
use Appwrite\Utopia\Request;
use Appwrite\Utopia\Response;
use Utopia\Database\Document;
use Utopia\Platform\Action;
use Utopia\System\System;
/**
* The S3 routes run in the `s3` group, not `api`: they authenticate with AWS
* Signature V4 instead of Appwrite headers, so none of the `api` hooks fire
* for them. The module's hooks carry what the gateway still owes the
* platform: the events queue identity here, event fan-out and network
* metering in the shutdown hooks. Abuse limits are deliberately absent — the
* S3 surface is API-key only.
*/
class Init extends Action
{
public static function getName(): string
{
return 'init';
}
public function __construct()
{
$this
->setType(Action::TYPE_INIT)
->groups(['s3'])
->desc('S3 gateway request setup')
->inject('request')
->inject('response')
->inject('cors')
->inject('project')
->inject('user')
->inject('queueForEvents')
->inject('usage')
->inject('geo')
->callback($this->action(...));
}
public function action(Request $request, Response $response, Cors $cors, Document $project, User $user, Event $queueForEvents, UsageContext $usage, Geo $geo): void
{
$corsHeaders = $cors->headers($request->getOrigin());
$exposedHeaders = \array_filter(\array_map('trim', \explode(',', (string) ($corsHeaders[Cors::HEADER_EXPOSE_HEADERS] ?? ''))));
$corsHeaders[Cors::HEADER_EXPOSE_HEADERS] = \implode(', ', \array_unique(\array_merge($exposedHeaders, [
'Accept-Ranges',
'Content-Length',
'Content-Range',
'ETag',
'Last-Modified',
'x-amz-server-side-encryption',
])));
foreach ($corsHeaders as $name => $value) {
$response->addHeader($name, $value);
}
// The S3 actions set the event name and params themselves; project and
// user identify the tenant for event fan-out and for the cache-delete
// messages the module builds from this queue.
$queueForEvents
->setProject($project)
->setUser($user);
// Mirrors the `api` group's "Enrich usage context with request
// metadata" hook in app/controllers/general.php — the gateway meters
// network usage through the same context.
$country = '';
$geoRecord = $geo->get($request->getIP());
if (!$geoRecord->isEmpty()) {
$country = \strtolower($geoRecord->getCountryCode());
}
$uri = $request->getURI();
$parts = explode('/', trim($uri, '/'));
$service = count($parts) >= 2 ? $parts[1] : $parts[0];
$usage
->setPath($uri)
->setMethod($request->getMethod())
->setUserAgent($request->getUserAgent(''))
->setHostname($request->getOrigin('') ?: $request->getHostname())
->setCountry($country)
->setIp($request->getIP())
->setSdk(\strtolower($request->getHeaderLine('x-sdk-name', '')))
->setSdkVersion($request->getHeaderLine('x-sdk-version', ''))
->setRegion(System::getEnv('_APP_REGION', 'default'))
->setService($service);
if (!$project->isEmpty()) {
$usage
->setTeamId((string) $project->getAttribute('teamId', ''))
->setTeamInternalId((string) $project->getAttribute('teamInternalId', ''));
}
// Always reset — clears values carried over from a previous request
// when this Context instance is reused across requests in the same worker.
$usage->setResourceType('');
$usage->setResourceId('');
$usage->setResourceInternalId('');
$usage->setResourcePath('');
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,225 @@
<?php
namespace Appwrite\Platform\Modules\S3\Http\S3;
use Appwrite\Event\Event;
use Appwrite\Event\Publisher\Audit as AuditPublisher;
use Appwrite\Event\Publisher\Delete as DeletePublisher;
use Appwrite\Event\Publisher\Func as FunctionPublisher;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Functions\EventProcessor;
use Appwrite\Platform\Modules\S3\Responses\S3Xml;
use Appwrite\Utopia\Database\Documents\User;
use Appwrite\Utopia\Response;
use Utopia\Cache\Cache;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Http\Adapter\Swoole\Request;
use Utopia\Platform\Action;
use Utopia\Storage\Device;
class Create extends Base
{
public static function getName(): string
{
return 's3Create';
}
public function __construct()
{
parent::__construct(Action::HTTP_REQUEST_METHOD_PUT);
$this->setHttpMethods([Action::HTTP_REQUEST_METHOD_PUT, Action::HTTP_REQUEST_METHOD_POST]);
}
protected function route(Request $request, Response $response, Database $dbForProject, Document $project, Document $team, User $user, Device $deviceForFiles, callable $locks, Cache $cache, Event $queueForEvents, DeletePublisher $publisherForDeletes, AuditPublisher $publisherForAudits, Event $queueForRealtime, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, EventProcessor $eventProcessor): void
{
[$bucketId, $key] = $this->parts($request);
if ($bucketId === '') {
throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Bucket is required.');
}
if ($this->unsupportedSubresource($request)) {
$this->authorize($request, $project, $team, $user, $key === '' ? ['buckets.write'] : ['files.write']);
$this->sendXml($response, S3Xml::error('NotImplemented', 'This S3 feature is not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED);
return;
}
if ($this->hasQuery($request, 'acl')) {
$this->authorize($request, $project, $team, $user, $key === '' ? ['buckets.write'] : ['files.write']);
$bucket = $this->bucket($dbForProject, $bucketId);
if ($key !== '') {
$this->getObject($dbForProject, $bucket, $key);
}
$response->setStatusCode(Response::STATUS_CODE_OK)->send('');
return;
}
if ($request->getMethod() === Action::HTTP_REQUEST_METHOD_POST && $key === '' && $this->hasQuery($request, 'delete')) {
$this->authorize($request, $project, $team, $user, ['files.write']);
$bucket = $this->bucket($dbForProject, $bucketId);
$payload = $this->requestBody($request);
$deleted = [];
foreach ($this->deleteObjectKeys($payload) as $objectKey) {
[$file, $removed] = $locks($this->objectLockKey($project, $bucket, $objectKey), 600, function () use ($dbForProject, $deviceForFiles, $bucket, $objectKey): array {
$file = $this->findObject($dbForProject, $bucket, $objectKey);
if ($file === null) {
return [null, true];
}
$path = $file->getAttribute('path', '');
if ($path !== '' && !$deviceForFiles->delete($path)) {
// Storage delete failed; keep the DB document so the
// object remains reachable and report no deletion.
return [$file, false];
}
$dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $file->getId()));
return [$file, true];
}, timeout: 120.0);
if (!$removed) {
continue;
}
if ($file !== null) {
$this->enqueueFileCacheDelete($publisherForDeletes, $queueForEvents, $bucket, $file);
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.delete', $file, 'file');
$this->triggerFileEvent($queueForEvents, $queueForRealtime, $publisherForFunctions, $queueForWebhooks, $eventProcessor, $dbForProject, $response, $bucket, $file, 'delete');
}
$deleted[] = $objectKey;
}
$this->sendXml($response, S3Xml::deleteObjects($deleted, $this->deleteQuiet($payload)));
return;
}
if ($request->getMethod() === Action::HTTP_REQUEST_METHOD_POST && $key !== '' && $this->hasQuery($request, 'uploads')) {
$this->authorize($request, $project, $team, $user, ['files.write']);
if ($this->isFolderMarker($key)) {
$this->sendXml($response, S3Xml::error('NotImplemented', 'S3 folders are not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED);
return;
}
$bucket = $this->bucket($dbForProject, $bucketId);
$uploadId = \bin2hex(\random_bytes(16));
$this->createMultipartUpload($dbForProject, $cache, $project, $deviceForFiles, $bucket, $key, $uploadId, $request->getHeaderLine('content-type', 'application/octet-stream'), $this->requestObjectMetadata($request));
$this->sendXml($response, S3Xml::initiateMultipart($bucketId, $key, $uploadId));
return;
}
if ($request->getMethod() === Action::HTTP_REQUEST_METHOD_PUT && $key !== '' && $this->query($request, 'uploadId') !== '' && $this->query($request, 'partNumber') !== '') {
$this->authorize($request, $project, $team, $user, ['files.write']);
$bucket = $this->bucket($dbForProject, $bucketId);
$partNumber = (int) $this->query($request, 'partNumber');
$uploadId = $this->query($request, 'uploadId');
$body = $this->requestBody($request);
$copySource = $request->getHeaderLine('x-amz-copy-source');
if ($copySource !== '') {
// Copy reads the source bytes, so the key must also hold files.read.
$this->authorize($request, $project, $team, $user, ['files.read']);
$copySource = \rawurldecode(\ltrim($copySource, '/'));
[$sourceBucketId, $sourceKey] = \array_pad(\explode('/', $copySource, 2), 2, '');
$sourceBucket = $this->bucket($dbForProject, $sourceBucketId);
$sourceFile = $this->getObject($dbForProject, $sourceBucket, $sourceKey);
$this->validateCopySize($bucket, $sourceFile);
$body = (string) $deviceForFiles->read($sourceFile->getAttribute('path', ''));
if (\preg_match('/^bytes=(\d+)-(\d+)$/', $request->getHeaderLine('x-amz-copy-source-range'), $matches) === 1) {
$start = (int) $matches[1];
$end = (int) $matches[2];
$body = \substr($body, $start, $end - $start + 1);
}
}
$etag = \md5($body);
// SDKs upload parts of one upload concurrently; the cached parts map
// and device chunk bookkeeping are read-modify-write, so load and
// save must share the upload's critical section.
$locks($this->multipartLockKey($project, $uploadId), 600, function () use ($cache, $project, $deviceForFiles, $uploadId, $bucketId, $key, $partNumber, $body, $etag): void {
$upload = $this->multipartUploadForBucket($cache, $project, $uploadId, $bucketId, $key);
$this->uploadMultipartPart($cache, $project, $deviceForFiles, $upload, $partNumber, $body, $etag);
}, timeout: 120.0);
if ($copySource !== '') {
$this->sendXml($response, S3Xml::copyPart($etag));
return;
}
$response
->setStatusCode(Response::STATUS_CODE_OK)
->addHeader('ETag', '"' . $etag . '"')
->send('');
return;
}
if ($request->getMethod() === Action::HTTP_REQUEST_METHOD_POST && $key !== '' && $this->query($request, 'uploadId') !== '') {
$this->authorize($request, $project, $team, $user, ['files.write']);
$bucket = $this->bucket($dbForProject, $bucketId);
// No multipart lock here: clients complete only after every part
// returned 200, and each part's state save commits inside its lock
// before that response, so the snapshot below always holds all
// acknowledged parts. Nesting the multipart lock around the object
// lock inside completeMultipartUpload would also need two
// connections from the fail-fast lock pool at once.
$selected = $this->completedPartNumbers($this->requestBody($request));
$upload = $this->multipartUploadForBucket($cache, $project, $this->query($request, 'uploadId'), $bucketId, $key);
[$file, $event] = $this->completeMultipartUpload($locks, $dbForProject, $cache, $project, $deviceForFiles, $bucket, $upload, $selected);
if ($event !== null) {
$this->queueFileEvent($queueForEvents, $response, $bucket, $file, $event);
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.' . $event, $file, 'file');
}
$this->sendXml($response, S3Xml::completeMultipart($bucketId, $key, $file->getAttribute('signature', '')));
return;
}
if ($key === '') {
$this->authorize($request, $project, $team, $user, ['buckets.write']);
$this->createBucket($dbForProject, $bucketId);
$bucket = $this->bucket($dbForProject, $bucketId);
$this->queueBucketEvent($queueForEvents, $response, $bucket, 'create');
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'bucket.create', $bucket, 'bucket');
$response->setStatusCode(Response::STATUS_CODE_OK)->send('');
return;
}
$this->authorize($request, $project, $team, $user, ['files.write']);
$bucket = $this->bucket($dbForProject, $bucketId);
if ($this->isFolderMarker($key)) {
if ($request->getHeaderLine('x-amz-copy-source') !== '') {
$this->sendXml($response, S3Xml::error('NotImplemented', 'Copying to an S3 folder marker is not supported.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED);
return;
}
$this->objectPath($key . '.folder');
$response->setStatusCode(Response::STATUS_CODE_OK)->send('');
return;
}
$copySource = $request->getHeaderLine('x-amz-copy-source');
if ($copySource !== '') {
// Copy reads the source bytes, so the key must also hold files.read.
$this->authorize($request, $project, $team, $user, ['files.read']);
$copySource = \rawurldecode(\ltrim($copySource, '/'));
[$sourceBucketId, $sourceKey] = \array_pad(\explode('/', $copySource, 2), 2, '');
$sourceBucket = $this->bucket($dbForProject, $sourceBucketId);
$sourceFile = $this->getObject($dbForProject, $sourceBucket, $sourceKey);
$this->validateCopySize($bucket, $sourceFile);
$metadata = $sourceFile->getAttribute('metadata', []);
if (\strtoupper($request->getHeaderLine('x-amz-metadata-directive')) === 'REPLACE') {
$metadata = $this->requestObjectMetadata($request);
}
[$file, $event] = $this->putObjectLocked($locks, $dbForProject, $project, $deviceForFiles, $bucket, $key, (string) $deviceForFiles->read($sourceFile->getAttribute('path', '')), $sourceFile->getAttribute('mimeType', 'application/octet-stream'), $metadata);
$this->queueFileEvent($queueForEvents, $response, $bucket, $file, $event);
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.' . $event, $file, 'file');
$this->sendXml($response, S3Xml::copyObject($file->getAttribute('signature', ''), $file->getUpdatedAt() ?: $file->getCreatedAt()));
return;
}
[$file, $event] = $this->putObjectLocked($locks, $dbForProject, $project, $deviceForFiles, $bucket, $key, $this->requestBody($request), $request->getHeaderLine('content-type', 'application/octet-stream'), $this->requestObjectMetadata($request));
$this->queueFileEvent($queueForEvents, $response, $bucket, $file, $event);
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.' . $event, $file, 'file');
$response
->setStatusCode(Response::STATUS_CODE_OK)
->addHeader('ETag', '"' . $file->getAttribute('signature', '') . '"');
$sse = (string) ($file->getAttribute('metadata', [])['serverSideEncryption'] ?? '');
if ($sse !== '') {
$response->addHeader('x-amz-server-side-encryption', $sse);
}
$response->send('');
}
}
@@ -0,0 +1,104 @@
<?php
namespace Appwrite\Platform\Modules\S3\Http\S3;
use Appwrite\Event\Event;
use Appwrite\Event\Publisher\Audit as AuditPublisher;
use Appwrite\Event\Publisher\Delete as DeletePublisher;
use Appwrite\Event\Publisher\Func as FunctionPublisher;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Functions\EventProcessor;
use Appwrite\Platform\Modules\S3\Responses\S3Xml;
use Appwrite\Utopia\Database\Documents\User;
use Appwrite\Utopia\Response;
use Utopia\Cache\Cache;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Http\Adapter\Swoole\Request;
use Utopia\Platform\Action;
use Utopia\Storage\Device;
class Delete extends Base
{
public static function getName(): string
{
return 's3Delete';
}
public function __construct()
{
parent::__construct(Action::HTTP_REQUEST_METHOD_DELETE);
}
protected function route(Request $request, Response $response, Database $dbForProject, Document $project, Document $team, User $user, Device $deviceForFiles, callable $locks, Cache $cache, Event $queueForEvents, DeletePublisher $publisherForDeletes, AuditPublisher $publisherForAudits, Event $queueForRealtime, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, EventProcessor $eventProcessor): void
{
[$bucketId, $key] = $this->parts($request);
if ($bucketId === '') {
throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Bucket is required.');
}
if ($this->unsupportedSubresource($request)) {
$this->authorize($request, $project, $team, $user, $key === '' ? ['buckets.write'] : ['files.write']);
$this->sendXml($response, S3Xml::error('NotImplemented', 'This S3 feature is not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED);
return;
}
if ($key === '') {
$this->authorize($request, $project, $team, $user, ['buckets.write']);
$bucket = $this->bucket($dbForProject, $bucketId);
$files = $dbForProject->getAuthorization()->skip(fn () => $dbForProject->find('bucket_' . $bucket->getSequence(), [Query::limit(1)]));
if ($files !== []) {
$this->sendXml($response, S3Xml::error('BucketNotEmpty', 'The bucket you tried to delete is not empty.', $request->getURI()), Response::STATUS_CODE_CONFLICT);
return;
}
$dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteCollection('bucket_' . $bucket->getSequence()));
$dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteDocument('buckets', $bucketId));
$this->queueBucketEvent($queueForEvents, $response, $bucket, 'delete');
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'bucket.delete', $bucket, 'bucket');
$response->noContent();
return;
}
$this->authorize($request, $project, $team, $user, ['files.write']);
$bucket = $this->bucket($dbForProject, $bucketId);
if ($this->isFolderMarker($key)) {
$response->noContent();
return;
}
if ($this->query($request, 'uploadId') !== '') {
$uploadId = $this->query($request, 'uploadId');
// Abort races in-flight part uploads for the same upload; take the
// upload's critical section so a late part cannot resurrect state
// that was just purged or upload into an aborted device staging.
$locks($this->multipartLockKey($project, $uploadId), 600, function () use ($cache, $project, $deviceForFiles, $bucket, $uploadId, $bucketId, $key): void {
$this->deleteMultipartUpload($cache, $project, $deviceForFiles, $bucket, $this->multipartUploadForBucket($cache, $project, $uploadId, $bucketId, $key));
}, timeout: 120.0);
$response->noContent();
return;
}
$file = $locks($this->objectLockKey($project, $bucket, $key), 600, function () use ($dbForProject, $deviceForFiles, $bucket, $key): ?Document {
$file = $this->findObject($dbForProject, $bucket, $key);
if ($file === null) {
return null;
}
if (!$deviceForFiles->delete($file->getAttribute('path', ''))) {
throw new AppwriteException(AppwriteException::GENERAL_SERVER_ERROR, 'Failed to delete object.');
}
$dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $file->getId()));
return $file;
}, timeout: 120.0);
if ($file !== null) {
$this->enqueueFileCacheDelete($publisherForDeletes, $queueForEvents, $bucket, $file);
$this->queueFileEvent($queueForEvents, $response, $bucket, $file, 'delete');
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.delete', $file, 'file');
}
$response->noContent();
}
}
@@ -0,0 +1,311 @@
<?php
namespace Appwrite\Platform\Modules\S3\Http\S3;
use Appwrite\Event\Event;
use Appwrite\Event\Publisher\Audit as AuditPublisher;
use Appwrite\Event\Publisher\Delete as DeletePublisher;
use Appwrite\Event\Publisher\Func as FunctionPublisher;
use Appwrite\Functions\EventProcessor;
use Appwrite\Platform\Modules\S3\Responses\S3Xml;
use Appwrite\Utopia\Database\Documents\User;
use Appwrite\Utopia\Response;
use Utopia\Cache\Cache;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Http\Adapter\Swoole\Request;
use Utopia\Platform\Action;
use Utopia\Storage\Device;
class Get extends Base
{
public static function getName(): string
{
return 's3Get';
}
public function __construct()
{
parent::__construct(Action::HTTP_REQUEST_METHOD_GET);
}
protected function route(Request $request, Response $response, Database $dbForProject, Document $project, Document $team, User $user, Device $deviceForFiles, callable $locks, Cache $cache, Event $queueForEvents, DeletePublisher $publisherForDeletes, AuditPublisher $publisherForAudits, Event $queueForRealtime, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, EventProcessor $eventProcessor): void
{
$isHead = $request->getMethod() === Action::HTTP_REQUEST_METHOD_HEAD;
[$bucketId, $key] = $this->parts($request);
if ($bucketId === '') {
$this->authorize($request, $project, $team, $user, ['buckets.read']);
if ($isHead) {
$response->send('');
return;
}
$buckets = $dbForProject->getAuthorization()->skip(fn () => $dbForProject->find('buckets', [Query::limit(1000)]));
$this->sendXml($response, S3Xml::listBuckets($buckets));
return;
}
if ($key === '') {
$this->authorize($request, $project, $team, $user, ['files.read', 'buckets.read']);
$bucket = $this->bucket($dbForProject, $bucketId);
if ($isHead) {
$response->send('');
return;
}
if ($this->hasQuery($request, 'acl')) {
$this->sendXml($response, S3Xml::accessControlPolicy());
return;
}
if ($this->hasQuery($request, 'location')) {
$this->sendXml($response, S3Xml::bucketLocation());
return;
}
if ($this->unsupportedSubresource($request)) {
$this->sendXml($response, S3Xml::error('NotImplemented', 'This S3 feature is not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED);
return;
}
if ($this->hasQuery($request, 'uploads')) {
$this->sendXml($response, S3Xml::listMultipartUploads($bucketId, $this->getMultipartUploads($cache, $project, $bucket)));
return;
}
// iterate() treats the limit as its page size and follows cursors until every file is loaded.
$files = $dbForProject->getAuthorization()->skip(fn () => \iterator_to_array($dbForProject->iterate('bucket_' . $bucket->getSequence(), [
Query::orderAsc('$sequence'),
Query::limit(1000),
]), false));
$this->assertUniqueObjectKeys($files);
if ($this->query($request, 'list-type') === '2') {
$this->sendXml($response, S3Xml::listObjectsV2(
$bucketId,
$files,
$this->query($request, 'prefix'),
$this->query($request, 'delimiter'),
(int) $this->query($request, 'max-keys', '1000'),
$this->query($request, 'continuation-token'),
$this->query($request, 'start-after')
));
return;
}
$this->sendXml($response, S3Xml::listObjects(
$bucketId,
$files,
$this->query($request, 'prefix'),
$this->query($request, 'delimiter'),
(int) $this->query($request, 'max-keys', '1000'),
$this->query($request, 'marker')
));
return;
}
$this->authorize($request, $project, $team, $user, ['files.read']);
$bucket = $this->bucket($dbForProject, $bucketId);
if ($this->query($request, 'uploadId') !== '') {
$parts = $this->getMultipartParts($this->multipartUploadForBucket($cache, $project, $this->query($request, 'uploadId'), $bucketId, $key));
$this->sendXml($response, S3Xml::listParts($bucketId, $key, $this->query($request, 'uploadId'), $parts));
return;
}
$isAcl = $this->hasQuery($request, 'acl');
$isUnsupported = $this->unsupportedSubresource($request);
$ifNoneMatch = $request->getHeaderLine('if-none-match');
$ifMatch = $request->getHeaderLine('if-match');
// Ordinary PUT overwrites its Device path in place, so metadata lookup and
// body materialization must not overlap that key's write critical section.
[$file, $body, $changedRepeatedly] = $locks($this->objectLockKey($project, $bucket, $key), 600, function () use ($dbForProject, $deviceForFiles, $bucket, $key, $isHead, $isAcl, $isUnsupported, $ifNoneMatch, $ifMatch, $request): array {
$file = $this->getObject($dbForProject, $bucket, $key);
$body = null;
$retried = false;
$changedRepeatedly = false;
while (true) {
$path = $file->getAttribute('path', '');
$size = (int) $file->getAttribute('sizeOriginal', 0);
$range = $this->range($request, $size);
$etag = '"' . $file->getAttribute('signature', '') . '"';
if (
$isHead
|| $isAcl
|| $isUnsupported
|| $ifNoneMatch === $etag
|| ($ifMatch !== '' && $ifMatch !== $etag)
|| $range === false
) {
break;
}
try {
$body = (string) $deviceForFiles->read($path);
break;
} catch (\Throwable $error) {
$latest = $this->getObject($dbForProject, $bucket, $key);
if ($latest->getAttribute('path', '') === $path) {
throw $error;
}
// One moved-path retry closes races with writes outside the S3 gateway.
if ($retried) {
$changedRepeatedly = true;
break;
}
$retried = true;
$file = $latest;
}
}
return [$file, $body, $changedRepeatedly];
}, timeout: 120.0);
if ($changedRepeatedly) {
$this->sendXml($response, S3Xml::error('SlowDown', 'The object changed repeatedly while it was being read. Please retry.', $request->getURI()), Response::STATUS_CODE_SERVICE_UNAVAILABLE);
return;
}
if ($isAcl) {
$this->sendXml($response, S3Xml::accessControlPolicy());
return;
}
if ($isUnsupported) {
$this->sendXml($response, S3Xml::error('NotImplemented', 'This S3 feature is not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED);
return;
}
$size = (int) $file->getAttribute('sizeOriginal', 0);
$range = $this->range($request, $size);
$etag = '"' . $file->getAttribute('signature', '') . '"';
if ($ifNoneMatch === $etag) {
$response->setStatusCode(Response::STATUS_CODE_NOT_MODIFIED)->send('');
return;
}
if ($ifMatch !== '' && $ifMatch !== $etag) {
$this->sendXml($response, S3Xml::error('PreconditionFailed', 'At least one of the pre-conditions you specified did not hold.', $request->getURI()), Response::STATUS_CODE_PRECONDITION_FAILED);
return;
}
if ($range === false) {
$response->addHeader('Content-Range', 'bytes */' . $size);
$this->sendXml($response, S3Xml::error('InvalidRange', 'The requested range is not satisfiable.', $request->getURI()), Response::STATUS_CODE_REQUESTED_RANGE_NOT_SATISFIABLE);
return;
}
$response
->setContentType($file->getAttribute('mimeType', 'application/octet-stream'))
->addHeader('ETag', $etag)
->addHeader('Accept-Ranges', 'bytes');
$lastModified = \strtotime($file->getUpdatedAt() ?: $file->getCreatedAt());
if ($lastModified !== false) {
$response->addHeader('Last-Modified', \gmdate('D, d M Y H:i:s', $lastModified) . ' GMT');
}
$metadata = $file->getAttribute('metadata', []);
foreach (($metadata['userMetadata'] ?? []) as $name => $value) {
$response->addHeader('x-amz-meta-' . $name, (string) $value);
}
$sse = (string) ($metadata['serverSideEncryption'] ?? '');
if ($sse !== '') {
$response->addHeader('x-amz-server-side-encryption', $sse);
}
foreach ([
'cacheControl' => 'Cache-Control',
'contentDisposition' => 'Content-Disposition',
'contentEncoding' => 'Content-Encoding',
'contentLanguage' => 'Content-Language',
'expires' => 'Expires',
] as $attribute => $header) {
$value = (string) ($metadata[$attribute] ?? '');
if ($value !== '') {
$response->addHeader($header, $value);
}
}
foreach ([
'response-cache-control' => 'Cache-Control',
'response-content-disposition' => 'Content-Disposition',
'response-content-encoding' => 'Content-Encoding',
'response-content-language' => 'Content-Language',
'response-expires' => 'Expires',
] as $parameter => $header) {
$value = $this->query($request, $parameter);
if ($value !== '') {
$response->setHeader($header, $value);
}
}
$responseContentType = $this->query($request, 'response-content-type');
if ($responseContentType !== '') {
$response->setContentType($responseContentType);
}
if ($range !== null) {
[$start, $end] = $range;
$response
->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT)
->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size)
->addHeader('Content-Length', (string) ($end - $start + 1));
} else {
$response->addHeader('Content-Length', (string) $size);
}
if ($isHead) {
$response->send('');
return;
}
if ($range !== null) {
[$start, $end] = $range;
$body = \substr((string) $body, $start, $end - $start + 1);
}
$response->send((string) $body);
}
/**
* @return array{0: int, 1: int}|false|null
*/
private function range(Request $request, int $size): array|false|null
{
$header = \trim($request->getHeaderLine('range'));
if ($header === '') {
return null;
}
if (!\preg_match('/^bytes=(\d*)-(\d*)$/', $header, $matches)) {
return false;
}
if ($size <= 0) {
return false;
}
$start = $matches[1];
$end = $matches[2];
if ($start === '' && $end === '') {
return false;
}
if ($start === '') {
$suffix = (int) $end;
if ($suffix <= 0) {
return false;
}
return [\max(0, $size - $suffix), \max(0, $size - 1)];
}
$start = (int) $start;
$end = $end === '' ? $size - 1 : (int) $end;
if ($start >= $size || $start > $end) {
return false;
}
return [$start, \min($end, $size - 1)];
}
}
@@ -0,0 +1,103 @@
<?php
namespace Appwrite\Platform\Modules\S3\Http\Shutdown;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Func as FunctionMessage;
use Appwrite\Event\Publisher\Func as FunctionPublisher;
use Appwrite\Event\Realtime;
use Appwrite\Functions\EventProcessor;
use Appwrite\Utopia\Response;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Http\Route;
use Utopia\Platform\Action;
/**
* Fans queued S3 events out to realtime, event-triggered functions and
* webhooks. Mirrors the `api` group's events shutdown hook in
* app/controllers/shared/api.php, which does not run for the `s3` group.
*/
class Events extends Action
{
public static function getName(): string
{
return 'shutdownEvents';
}
public function __construct()
{
$this
->setType(Action::TYPE_SHUTDOWN)
->groups(['s3'])
->desc('S3 gateway event fan-out')
->inject('route')
->inject('response')
->inject('project')
->inject('queueForEvents')
->inject('publisherForFunctions')
->inject('queueForWebhooks')
->inject('queueForRealtime')
->inject('dbForProject')
->inject('eventProcessor')
->callback($this->action(...));
}
public function action(Route $route, Response $response, Document $project, Event $queueForEvents, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, EventProcessor $eventProcessor): void
{
if (empty($queueForEvents->getEvent())) {
return;
}
if (empty($queueForEvents->getPayload())) {
$queueForEvents->setPayload($response->getPayload());
}
// Get project and function/webhook events (cached)
$functionsEvents = $eventProcessor->getFunctionsEvents($project, $dbForProject);
$webhooksEvents = $eventProcessor->getWebhooksEvents($project);
// Generate events for this operation
$generatedEvents = Event::generateEvents(
$queueForEvents->getEvent(),
$queueForEvents->getParams()
);
$allowedOnConsole = !empty(\array_intersect($route->getGroups(), Realtime::CONSOLE_ALLOWLIST));
if ($project->getId() !== 'console' || $allowedOnConsole) {
$queueForRealtime
->from($queueForEvents)
->trigger();
}
// Only trigger functions if there are matching function events
if (!empty($functionsEvents)) {
foreach ($generatedEvents as $event) {
if (isset($functionsEvents[$event])) {
$publisherForFunctions->enqueue(FunctionMessage::fromEvent(
event: $queueForEvents->getEvent(),
params: $queueForEvents->getParams(),
project: $queueForEvents->getProject(),
user: $queueForEvents->getUser(),
userId: $queueForEvents->getUserId(),
payload: $queueForEvents->getPayload(),
platform: $queueForEvents->getPlatform(),
));
break;
}
}
}
// Only trigger webhooks if there are matching webhook events
if (!empty($webhooksEvents)) {
foreach ($generatedEvents as $event) {
if (isset($webhooksEvents[$event])) {
$queueForWebhooks
->from($queueForEvents)
->trigger();
break;
}
}
}
}
}
@@ -0,0 +1,77 @@
<?php
namespace Appwrite\Platform\Modules\S3\Http\Shutdown;
use Appwrite\Bus\Events\RequestCompleted;
use Appwrite\Event\Message\Usage as UsageMessage;
use Appwrite\Event\Publisher\Usage as UsagePublisher;
use Appwrite\Usage\Context as UsageContext;
use Appwrite\Utopia\Database\Documents\User;
use Appwrite\Utopia\Request;
use Appwrite\Utopia\Response;
use Utopia\Bus\Bus;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Action;
/**
* Meters S3 gateway traffic. Mirrors the `api` group's usage shutdown hook in
* app/controllers/shared/api.php, which does not run for the `s3`
* group.
*/
class Usage extends Action
{
public static function getName(): string
{
return 'shutdownUsage';
}
public function __construct()
{
$this
->setType(Action::TYPE_SHUTDOWN)
->groups(['s3'])
->desc('S3 gateway usage metering')
->inject('request')
->inject('response')
->inject('project')
->inject('user')
->inject('usage')
->inject('publisherForUsage')
->inject('authorization')
->inject('bus')
->callback($this->action(...));
}
public function action(Request $request, Response $response, Document $project, User $user, UsageContext $usage, UsagePublisher $publisherForUsage, Authorization $authorization, Bus $bus): void
{
if ($project->isEmpty() || $project->getId() === 'console') {
return;
}
$usage->setStatus($response->getStatusCode());
if ($usage->getResourcePath() === '') {
$usage->fillMissingResource('project', $project->getId(), (string) $project->getSequence());
}
// The usage bus listener adds METRIC_NETWORK_REQUESTS/INBOUND/OUTBOUND,
// exactly as on the `api` group. The SigV4 key never becomes the
// `apiKey` resource, so per-key disabled-metrics filtering cannot
// apply here.
if (!$user->isPrivileged($authorization->getRoles())) {
$bus->dispatch(new RequestCompleted(
project: $project->getArrayCopy(),
request: $request,
response: $response,
));
}
if (!$usage->isEmpty()) {
$publisherForUsage->enqueue(new UsageMessage(
project: $project,
metrics: $usage->getMetrics(),
reduce: $usage->getReduce(),
));
}
}
}
@@ -0,0 +1,14 @@
<?php
namespace Appwrite\Platform\Modules\S3;
use Appwrite\Platform\Modules\S3\Services\Http;
use Utopia\Platform;
class Module extends Platform\Module
{
public function __construct()
{
$this->addService('http', new Http());
}
}
@@ -0,0 +1,128 @@
<?php
namespace Appwrite\Platform\Modules\S3\Requests;
use Appwrite\Extend\Exception as AppwriteException;
/**
* Decoder for aws-chunked request bodies (AWS SigV4 streaming payloads).
*
* Modern AWS SDKs upload objects as `Content-Encoding: aws-chunked` with
* `x-amz-content-sha256: STREAMING-*`: the body arrives framed as
* `<hex-size>[;chunk-signature=…]\r\n<bytes>\r\n` chunks, terminated by a
* zero-size chunk and optional trailing headers. Storing the raw payload
* would persist the framing bytes as object content.
*
* Per-chunk signatures are stripped without verification — the request is
* already authenticated by the seed signature over the canonical request,
* which pins the STREAMING-* payload mode. Integrity is checked through the
* declared decoded length and, when present, the CRC32 trailer checksum.
*/
class AwsChunked
{
public static function applies(string $contentSha256, string $contentEncoding): bool
{
if (\str_starts_with(\strtoupper(\trim($contentSha256)), 'STREAMING-')) {
return true;
}
foreach (\explode(',', $contentEncoding) as $encoding) {
if (\strtolower(\trim($encoding)) === 'aws-chunked') {
return true;
}
}
return false;
}
public static function decode(string $payload, ?int $declaredLength = null): string
{
$decoded = '';
$offset = 0;
$length = \strlen($payload);
while (true) {
if ($offset >= $length) {
throw self::malformed('Missing final zero-size chunk.');
}
$lineEnd = \strpos($payload, "\r\n", $offset);
if ($lineEnd === false) {
throw self::malformed('Chunk header is not CRLF-terminated.');
}
$sizeHex = \explode(';', \substr($payload, $offset, $lineEnd - $offset), 2)[0];
if ($sizeHex === '' || \strlen($sizeHex) > 8 || \preg_match('/^[0-9a-fA-F]+$/', $sizeHex) !== 1) {
throw self::malformed('Chunk size is not hexadecimal.');
}
$size = (int) \hexdec($sizeHex);
$offset = $lineEnd + 2;
if ($size === 0) {
self::verifyTrailers(self::trailers($payload, $offset), $decoded);
break;
}
if ($offset + $size > $length) {
throw self::malformed('Chunk data is truncated.');
}
$decoded .= \substr($payload, $offset, $size);
$offset += $size;
if (\substr($payload, $offset, 2) !== "\r\n") {
throw self::malformed('Chunk data is not CRLF-terminated.');
}
$offset += 2;
}
if ($declaredLength !== null && \strlen($decoded) !== $declaredLength) {
throw self::malformed('Decoded length does not match x-amz-decoded-content-length.');
}
return $decoded;
}
/**
* @return array<string, string>
*/
private static function trailers(string $payload, int $offset): array
{
$trailers = [];
$length = \strlen($payload);
while ($offset < $length) {
$lineEnd = \strpos($payload, "\r\n", $offset);
$line = $lineEnd === false ? \substr($payload, $offset) : \substr($payload, $offset, $lineEnd - $offset);
$offset = $lineEnd === false ? $length : $lineEnd + 2;
if (\trim($line) === '') {
continue;
}
[$name, $value] = \array_pad(\explode(':', $line, 2), 2, '');
$trailers[\strtolower(\trim($name))] = \trim($value);
}
return $trailers;
}
/**
* @param array<string, string> $trailers
*/
private static function verifyTrailers(array $trailers, string $decoded): void
{
// CRC32 is the SDK default checksum algorithm; other algorithms
// (crc32c, sha1, sha256, crc64nvme) are accepted but not verified.
$crc32 = $trailers['x-amz-checksum-crc32'] ?? '';
if ($crc32 !== '' && \base64_encode(\hash('crc32b', $decoded, true)) !== $crc32) {
throw self::malformed('CRC32 trailer checksum does not match the decoded payload.');
}
}
private static function malformed(string $reason): AppwriteException
{
return new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Malformed aws-chunked payload. ' . $reason);
}
}
@@ -0,0 +1,360 @@
<?php
namespace Appwrite\Platform\Modules\S3\Responses;
use Utopia\Database\Document;
class S3Xml
{
public static function error(string $code, string $message, string $resource = '', string $requestId = ''): string
{
return self::document('Error', [
'Code' => $code,
'Message' => $message,
'Resource' => $resource,
'RequestId' => $requestId ?: \bin2hex(\random_bytes(8)),
]);
}
/**
* @param array<Document> $buckets
*/
public static function listBuckets(array $buckets): string
{
$items = '';
foreach ($buckets as $bucket) {
$items .= self::element('Bucket', [
'Name' => $bucket->getId(),
'CreationDate' => self::date($bucket->getCreatedAt()),
]);
}
return self::rawDocument('ListAllMyBucketsResult', '<Buckets>' . $items . '</Buckets>');
}
public static function accessControlPolicy(): string
{
return self::rawDocument('AccessControlPolicy', self::element('Owner', [
'ID' => 'appwrite',
'DisplayName' => 'appwrite',
]) . '<AccessControlList>' . self::element('Grant', [
'Grantee' => [
'ID' => 'appwrite',
'DisplayName' => 'appwrite',
],
'Permission' => 'FULL_CONTROL',
]) . '</AccessControlList>');
}
public static function bucketLocation(string $region = 'us-east-1'): string
{
return self::rawDocument('LocationConstraint', $region === 'us-east-1' ? '' : self::escape($region));
}
public static function copyObject(string $etag, ?string $lastModified = null): string
{
return self::document('CopyObjectResult', [
'LastModified' => self::date($lastModified),
'ETag' => '"' . $etag . '"',
]);
}
public static function copyPart(string $etag, ?string $lastModified = null): string
{
return self::document('CopyPartResult', [
'LastModified' => self::date($lastModified),
'ETag' => '"' . $etag . '"',
]);
}
/**
* @param array<int, string> $keys
*/
public static function deleteObjects(array $keys, bool $quiet = false): string
{
$items = '';
if (!$quiet) {
foreach ($keys as $key) {
$items .= self::element('Deleted', ['Key' => $key]);
}
}
return self::rawDocument('DeleteResult', $items);
}
/**
* @param array<int, array{key: string, uploadId: string, initiated: string}> $uploads
*/
public static function listMultipartUploads(string $bucketId, array $uploads): string
{
$items = '';
foreach ($uploads as $upload) {
$items .= self::element('Upload', [
'Key' => $upload['key'],
'UploadId' => $upload['uploadId'],
'Initiated' => self::date($upload['initiated']),
]);
}
return self::rawDocument('ListMultipartUploadsResult', self::element('Bucket', $bucketId) . self::element('IsTruncated', 'false') . $items);
}
/**
* @param array<Document> $files
*/
public static function listObjects(string $bucketId, array $files, string $prefix = '', string $delimiter = '', int $maxKeys = 1000, string $marker = ''): string
{
$maxKeys = \max(0, \min(1000, $maxKeys));
$entries = [];
$commonPrefixes = [];
foreach ($files as $file) {
$key = self::fileKey($file);
if (($prefix !== '' && !\str_starts_with($key, $prefix)) || ($marker !== '' && \strcmp($key, $marker) <= 0)) {
continue;
}
if ($delimiter !== '') {
$rest = \substr($key, \strlen($prefix));
$position = \strpos($rest, $delimiter);
if ($position !== false) {
$commonPrefixes[$prefix . \substr($rest, 0, $position + \strlen($delimiter))] = true;
continue;
}
}
$entries[$key] = ['type' => 'file', 'file' => $file];
}
foreach (\array_keys($commonPrefixes) as $commonPrefix) {
if ($marker !== '' && \strcmp($commonPrefix, $marker) <= 0) {
continue;
}
$entries[$commonPrefix] = ['type' => 'prefix', 'prefix' => $commonPrefix];
}
\ksort($entries, SORT_STRING);
$selected = \array_slice($entries, 0, $maxKeys, true);
$isTruncated = \count($entries) > $maxKeys;
$lastKey = $selected === [] ? '' : (string) \array_key_last($selected);
$items = '';
foreach ($selected as $key => $entry) {
if ($entry['type'] === 'prefix') {
$items .= self::element('CommonPrefixes', ['Prefix' => (string) $entry['prefix']]);
continue;
}
$file = $entry['file'];
$items .= self::element('Contents', [
'Key' => $key,
'LastModified' => self::date($file->getUpdatedAt() ?: $file->getCreatedAt()),
'ETag' => '"' . $file->getAttribute('signature', '') . '"',
'Size' => (string) $file->getAttribute('sizeOriginal', 0),
'StorageClass' => 'STANDARD',
]);
}
$body = self::elements([
'Name' => $bucketId,
'Prefix' => $prefix,
'Marker' => $marker,
'MaxKeys' => (string) $maxKeys,
'IsTruncated' => $isTruncated ? 'true' : 'false',
]);
if ($delimiter !== '') {
$body .= self::element('Delimiter', $delimiter);
}
if ($isTruncated && $lastKey !== '') {
$body .= self::element('NextMarker', $lastKey);
}
return self::rawDocument('ListBucketResult', $body . $items);
}
/**
* @param array<Document> $files
*/
public static function listObjectsV2(
string $bucketId,
array $files,
string $prefix = '',
string $delimiter = '',
int $maxKeys = 1000,
string $continuationToken = '',
string $startAfter = ''
): string {
$maxKeys = \max(0, \min(1000, $maxKeys));
$after = $continuationToken !== '' ? self::decodeContinuationToken($continuationToken) : $startAfter;
$entries = [];
$commonPrefixes = [];
foreach ($files as $file) {
$key = self::fileKey($file);
if ($prefix !== '' && !\str_starts_with($key, $prefix)) {
continue;
}
if ($after !== '' && \strcmp($key, $after) <= 0) {
continue;
}
if ($delimiter !== '') {
$rest = \substr($key, \strlen($prefix));
$position = \strpos($rest, $delimiter);
if ($position !== false) {
$commonPrefixes[$prefix . \substr($rest, 0, $position + \strlen($delimiter))] = true;
continue;
}
}
$entries[$key] = ['type' => 'file', 'file' => $file];
}
foreach (\array_keys($commonPrefixes) as $commonPrefix) {
if ($after !== '' && \strcmp($commonPrefix, $after) <= 0) {
continue;
}
$entries[$commonPrefix] = ['type' => 'prefix', 'prefix' => $commonPrefix];
}
// PHP casts numeric keys to int; without SORT_STRING they sort numerically,
// which breaks the strcmp-based continuation filter and skips objects.
\ksort($entries, SORT_STRING);
$selected = \array_slice($entries, 0, $maxKeys, true);
$isTruncated = \count($entries) > $maxKeys;
$lastKey = $selected === [] ? '' : (string) \array_key_last($selected);
$items = '';
foreach ($selected as $entry) {
if ($entry['type'] === 'prefix') {
$items .= self::element('CommonPrefixes', [
'Prefix' => (string) $entry['prefix'],
]);
continue;
}
$file = $entry['file'];
$items .= self::element('Contents', [
'Key' => self::fileKey($file),
'LastModified' => self::date($file->getUpdatedAt() ?: $file->getCreatedAt()),
'ETag' => '"' . $file->getAttribute('signature', '') . '"',
'Size' => (string) $file->getAttribute('sizeOriginal', 0),
'StorageClass' => 'STANDARD',
]);
}
$body = self::elements([
'Name' => $bucketId,
'Prefix' => $prefix,
'KeyCount' => (string) \count($selected),
'MaxKeys' => (string) $maxKeys,
'IsTruncated' => $isTruncated ? 'true' : 'false',
]);
if ($delimiter !== '') {
$body .= self::element('Delimiter', $delimiter);
}
if ($continuationToken !== '') {
$body .= self::element('ContinuationToken', $continuationToken);
}
if ($startAfter !== '') {
$body .= self::element('StartAfter', $startAfter);
}
if ($isTruncated && $lastKey !== '') {
$body .= self::element('NextContinuationToken', self::encodeContinuationToken($lastKey));
}
return self::rawDocument('ListBucketResult', $body . $items);
}
public static function initiateMultipart(string $bucketId, string $key, string $uploadId): string
{
return self::document('InitiateMultipartUploadResult', [
'Bucket' => $bucketId,
'Key' => $key,
'UploadId' => $uploadId,
]);
}
public static function completeMultipart(string $bucketId, string $key, string $etag): string
{
return self::document('CompleteMultipartUploadResult', [
'Location' => '/' . $bucketId . '/' . $key,
'Bucket' => $bucketId,
'Key' => $key,
'ETag' => '"' . $etag . '"',
]);
}
/**
* @param array<int, array{partNumber: int, etag: string, size: int}> $parts
*/
public static function listParts(string $bucketId, string $key, string $uploadId, array $parts): string
{
$items = '';
foreach ($parts as $part) {
$items .= self::element('Part', [
'PartNumber' => (string) $part['partNumber'],
'ETag' => '"' . $part['etag'] . '"',
'Size' => (string) $part['size'],
]);
}
return self::rawDocument('ListPartsResult', self::elements([
'Bucket' => $bucketId,
'Key' => $key,
'UploadId' => $uploadId,
]) . $items);
}
private static function document(string $root, array $values): string
{
return self::rawDocument($root, self::elements($values));
}
private static function rawDocument(string $root, string $body): string
{
return '<?xml version="1.0" encoding="UTF-8"?>' . '<' . $root . '>' . $body . '</' . $root . '>';
}
private static function elements(array $values): string
{
$xml = '';
foreach ($values as $key => $value) {
$xml .= self::element((string) $key, \is_array($value) ? $value : (string) $value);
}
return $xml;
}
private static function element(string $name, string|array $value): string
{
$content = \is_array($value) ? self::elements($value) : self::escape($value);
return '<' . $name . '>' . $content . '</' . $name . '>';
}
private static function fileKey(Document $file): string
{
return $file->getAttribute('folder', '') . $file->getAttribute('name', $file->getId());
}
private static function encodeContinuationToken(string $key): string
{
return \rtrim(\strtr(\base64_encode($key), '+/', '-_'), '=');
}
private static function decodeContinuationToken(string $token): string
{
$decoded = \base64_decode(\strtr($token, '-_', '+/'), true);
return \is_string($decoded) ? $decoded : '';
}
private static function escape(string $value): string
{
return \htmlspecialchars($value, ENT_XML1 | ENT_COMPAT, 'UTF-8');
}
private static function date(?string $date): string
{
$time = \strtotime($date ?? '');
return \gmdate('Y-m-d\TH:i:s.000\Z', $time === false ? \time() : $time);
}
}
@@ -0,0 +1,26 @@
<?php
namespace Appwrite\Platform\Modules\S3\Services;
use Appwrite\Platform\Modules\S3\Http\Init;
use Appwrite\Platform\Modules\S3\Http\S3\Create;
use Appwrite\Platform\Modules\S3\Http\S3\Delete;
use Appwrite\Platform\Modules\S3\Http\S3\Get;
use Appwrite\Platform\Modules\S3\Http\Shutdown\Events as ShutdownEvents;
use Appwrite\Platform\Modules\S3\Http\Shutdown\Usage as ShutdownUsage;
use Utopia\Platform\Service;
class Http extends Service
{
public function __construct()
{
$this->type = Service::TYPE_HTTP;
$this->addAction(Init::getName(), new Init());
$this->addAction(Get::getName(), new Get());
$this->addAction(Create::getName(), new Create());
$this->addAction(Delete::getName(), new Delete());
$this->addAction(ShutdownEvents::getName(), new ShutdownEvents());
$this->addAction(ShutdownUsage::getName(), new ShutdownUsage());
}
}
@@ -0,0 +1,324 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Platform\Modules\S3\Auth;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Platform\Modules\S3\Auth\SignatureV4;
use Appwrite\Utopia\Database\Documents\User;
use PHPUnit\Framework\TestCase;
use Swoole\Http\Request as SwooleRequest;
use Utopia\Database\Document;
use Utopia\Http\Adapter\Swoole\Request;
final class SignatureV4Test extends TestCase
{
public function testVerifiesValidSignatureForExistingProjectApiKey(): void
{
$secret = 'test-api-key-secret';
$request = $this->signedRequest('GET', '/v1/s3/bucket/object.txt', '', $secret);
$key = (new SignatureV4())->verify(
$request,
$this->project($secret, ['files.read']),
new Document(),
new User(),
['files.read']
);
$this->assertSame('project-test', $key->getProjectId());
$this->assertContains('files.read', $key->getScopes());
}
public function testVerifiesUnsignedPayloadSignature(): void
{
$secret = 'test-api-key-secret';
$request = $this->signedRequest('GET', '/v1/s3/bucket', '', $secret, 'UNSIGNED-PAYLOAD', 'delimiter=%2F&list-type=2&max-keys=100');
$key = (new SignatureV4())->verify(
$request,
$this->project($secret, ['files.read']),
new Document(),
new User(),
['files.read']
);
$this->assertSame('project-test', $key->getProjectId());
}
public function testRejectsExpiredPresignedRequest(): void
{
$secret = 'test-api-key-secret';
$request = $this->presignedRequestAt(
'GET',
'/v1/s3/bucket/object.txt',
$secret,
\gmdate('Ymd\THis\Z', \time() - 120),
60,
);
$this->expectException(AppwriteException::class);
$this->expectExceptionMessage('presigned URL has expired');
(new SignatureV4())->verify(
$request,
$this->project($secret, ['files.read']),
new Document(),
new User(),
['files.read']
);
}
public function testVerifiesOpenDalRangedGetSignature(): void
{
$secret = 'test-api-key-secret';
$request = $this->signedRequest(
'GET',
'/v1/s3/test/ai_history_fundamentals_quiz_quizizz.xlsx',
'',
$secret,
'UNSIGNED-PAYLOAD',
additionalHeaders: ['range' => 'bytes=0-8218'],
);
$key = (new SignatureV4())->verify(
$request,
$this->project($secret, ['files.read']),
new Document(),
new User(),
['files.read']
);
$this->assertSame('project-test', $key->getProjectId());
}
public function testVerifiesOpenDalDeleteSignature(): void
{
$secret = 'test-api-key-secret';
$request = $this->signedRequest(
'DELETE',
'/v1/s3/test/ai_history_fundamentals_quiz_quizizz.xlsx',
'',
$secret,
'UNSIGNED-PAYLOAD',
);
$key = (new SignatureV4())->verify(
$request,
$this->project($secret, ['files.write']),
new Document(),
new User(),
['files.write']
);
$this->assertSame('project-test', $key->getProjectId());
}
public function testRejectsValidSignatureWhenApiKeyMissingRequiredScope(): void
{
$secret = 'test-api-key-secret';
$request = $this->signedRequest('PUT', '/v1/s3/bucket/object.txt', 'body', $secret);
$this->expectException(AppwriteException::class);
$this->expectExceptionMessage('API key is missing required S3 storage scopes.');
(new SignatureV4())->verify(
$request,
$this->project($secret, ['files.read']),
new Document(),
new User(),
['files.write']
);
}
public function testRejectsInvalidSignature(): void
{
$secret = 'test-api-key-secret';
$request = $this->signedRequest('GET', '/v1/s3/bucket/object.txt', '', $secret);
$request->headerMap['authorization'] = \preg_replace('/Signature=[0-9a-f]+/', 'Signature=deadbeef', $request->headerMap['authorization']);
$this->expectException(AppwriteException::class);
$this->expectExceptionMessage('Signature does not match.');
(new SignatureV4())->verify(
$request,
$this->project($secret, ['files.read']),
new Document(),
new User(),
['files.read']
);
}
public function testRejectsExpiredSignatureTimestamp(): void
{
$secret = 'test-api-key-secret';
$date = \gmdate('Ymd\THis\Z', \time() - 901);
$request = $this->signedRequest('GET', '/v1/s3/bucket/object.txt', '', $secret, date: $date);
$this->expectException(AppwriteException::class);
$this->expectExceptionMessage('AWS Signature V4 timestamp is outside the allowed time window.');
(new SignatureV4())->verify(
$request,
$this->project($secret, ['files.read']),
new Document(),
new User(),
['files.read']
);
}
public function testRejectsFutureSignatureTimestamp(): void
{
$secret = 'test-api-key-secret';
$date = \gmdate('Ymd\THis\Z', \time() + 901);
$request = $this->signedRequest('GET', '/v1/s3/bucket/object.txt', '', $secret, date: $date);
$this->expectException(AppwriteException::class);
$this->expectExceptionMessage('AWS Signature V4 timestamp is outside the allowed time window.');
(new SignatureV4())->verify(
$request,
$this->project($secret, ['files.read']),
new Document(),
new User(),
['files.read']
);
}
private function project(string $secret, array $scopes): Document
{
return new Document([
'$id' => 'project-test',
'keys' => [
[
'$id' => 'key-test',
'name' => 'S3 Test Key',
'secret' => $secret,
'scopes' => $scopes,
'expire' => null,
],
],
]);
}
private function presignedRequestAt(string $method, string $uri, string $secret, string $date, int $expires): TestRequest
{
$shortDate = \substr($date, 0, 8);
$scope = "{$shortDate}/us-east-1/s3/aws4_request";
$parameters = [
'X-Amz-Algorithm' => 'AWS4-HMAC-SHA256',
'X-Amz-Credential' => 'project-test/' . $scope,
'X-Amz-Date' => $date,
'X-Amz-Expires' => (string) $expires,
'X-Amz-SignedHeaders' => 'host',
];
$encoded = [];
foreach ($parameters as $name => $value) {
$encoded[] = \rawurlencode($name) . '=' . \rawurlencode($value);
}
\sort($encoded, SORT_STRING);
$canonicalQuery = \implode('&', $encoded);
$canonicalRequest = \implode("\n", [
$method,
$uri,
$canonicalQuery,
"host:cloud.appwrite.test\n",
'host',
'UNSIGNED-PAYLOAD',
]);
$stringToSign = "AWS4-HMAC-SHA256\n{$date}\n{$scope}\n" . \hash('sha256', $canonicalRequest);
$signature = \hash_hmac('sha256', $stringToSign, $this->signingKey($secret, $shortDate, 'us-east-1', 's3'));
$query = $canonicalQuery . '&X-Amz-Signature=' . $signature;
return new TestRequest($method, $uri . '?' . $query, ['host' => 'cloud.appwrite.test'], '', $query);
}
private function signedRequest(string $method, string $uri, string $body, string $secret, ?string $payloadHash = null, string $query = '', ?string $date = null, array $additionalHeaders = []): TestRequest
{
$date ??= \gmdate('Ymd\THis\Z');
$shortDate = \substr($date, 0, 8);
$region = 'us-east-1';
$service = 's3';
$headers = \array_merge([
'host' => 'cloud.appwrite.test',
'x-amz-content-sha256' => $payloadHash ?? \hash('sha256', $body),
'x-amz-date' => $date,
], $additionalHeaders);
\ksort($headers);
$signedHeaders = \implode(';', \array_keys($headers));
$canonicalHeaders = '';
foreach ($headers as $name => $value) {
$canonicalHeaders .= $name . ':' . \preg_replace('/\s+/', ' ', \trim($value)) . "\n";
}
$canonicalRequest = \implode("\n", [
$method,
$uri,
$query,
$canonicalHeaders,
$signedHeaders,
$headers['x-amz-content-sha256'],
]);
$scope = "{$shortDate}/{$region}/{$service}/aws4_request";
$stringToSign = "AWS4-HMAC-SHA256\n{$date}\n{$scope}\n" . \hash('sha256', $canonicalRequest);
$signature = \hash_hmac('sha256', $stringToSign, $this->signingKey($secret, $shortDate, $region, $service));
$headers['authorization'] = "AWS4-HMAC-SHA256 Credential=project-test/{$scope}, SignedHeaders={$signedHeaders}, Signature={$signature}";
return new TestRequest($method, $query === '' ? $uri : $uri . '?' . $query, $headers, $body, $query);
}
private function signingKey(string $secret, string $date, string $region, string $service): string
{
$kDate = \hash_hmac('sha256', $date, 'AWS4' . $secret, true);
$kRegion = \hash_hmac('sha256', $region, $kDate, true);
$kService = \hash_hmac('sha256', $service, $kRegion, true);
return \hash_hmac('sha256', 'aws4_request', $kService, true);
}
}
final class TestRequest extends Request
{
/**
* @param array<string, string|null> $headerMap
*/
public function __construct(
private readonly string $method,
public readonly string $uri,
public array $headerMap,
private readonly string $body,
public readonly string $query = '',
) {
}
public function getMethod(): string
{
return $this->method;
}
public function getURI(): string
{
return $this->uri;
}
public function getHeaderLine(string $key, string $default = ''): string
{
return $this->headerMap[\strtolower($key)] ?? $default;
}
public function getRawPayload(): string
{
return $this->body;
}
public function getServer(string $key, ?string $default = null): ?string
{
return $key === 'query_string' ? $this->query : $default;
}
public function getSwooleRequest(): SwooleRequest
{
throw new \LogicException('Not used by this test.');
}
}
@@ -0,0 +1,105 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Platform\Modules\S3\Requests;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Platform\Modules\S3\Requests\AwsChunked;
use PHPUnit\Framework\TestCase;
final class AwsChunkedTest extends TestCase
{
public function testAppliesDetectsStreamingPayloadsAndChunkedEncoding(): void
{
$this->assertTrue(AwsChunked::applies('STREAMING-UNSIGNED-PAYLOAD-TRAILER', ''));
$this->assertTrue(AwsChunked::applies('streaming-aws4-hmac-sha256-payload', ''));
$this->assertTrue(AwsChunked::applies('', 'aws-chunked'));
$this->assertTrue(AwsChunked::applies('', 'gzip, aws-chunked'));
$this->assertFalse(AwsChunked::applies(\hash('sha256', 'body'), ''));
$this->assertFalse(AwsChunked::applies('UNSIGNED-PAYLOAD', ''));
$this->assertFalse(AwsChunked::applies('', 'gzip'));
}
public function testDecodesUnsignedTrailerFraming(): void
{
$payload = "6\r\nhello \r\n"
. "5\r\nworld\r\n"
. "0\r\n"
. 'x-amz-checksum-crc32:' . \base64_encode(\hash('crc32b', 'hello world', true)) . "\r\n"
. "\r\n";
$this->assertSame('hello world', AwsChunked::decode($payload, 11));
}
public function testDecodesSignedChunkFraming(): void
{
$payload = "6;chunk-signature=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\nhello \r\n"
. "5;chunk-signature=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\r\nworld\r\n"
. "0;chunk-signature=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\r\n\r\n";
$this->assertSame('hello world', AwsChunked::decode($payload, 11));
}
public function testDecodesEmptyBody(): void
{
$this->assertSame('', AwsChunked::decode("0\r\n\r\n", 0));
$this->assertSame('', AwsChunked::decode("0\r\n", null));
}
public function testDecodeIgnoresUnverifiedTrailerAlgorithms(): void
{
$payload = "3\r\nabc\r\n0\r\nx-amz-checksum-sha256:not-checked\r\n\r\n";
$this->assertSame('abc', AwsChunked::decode($payload, 3));
}
public function testRejectsDeclaredLengthMismatch(): void
{
$this->expectException(AppwriteException::class);
$this->expectExceptionMessage('x-amz-decoded-content-length');
AwsChunked::decode("3\r\nabc\r\n0\r\n\r\n", 4);
}
public function testRejectsCrc32TrailerMismatch(): void
{
$this->expectException(AppwriteException::class);
$this->expectExceptionMessage('CRC32');
AwsChunked::decode("3\r\nabc\r\n0\r\n" . 'x-amz-checksum-crc32:' . \base64_encode(\hash('crc32b', 'abd', true)) . "\r\n\r\n", 3);
}
public function testRejectsTruncatedChunkData(): void
{
$this->expectException(AppwriteException::class);
$this->expectExceptionMessage('truncated');
AwsChunked::decode("6\r\nhel", null);
}
public function testRejectsMissingFinalChunk(): void
{
$this->expectException(AppwriteException::class);
$this->expectExceptionMessage('zero-size chunk');
AwsChunked::decode("3\r\nabc\r\n", null);
}
public function testRejectsNonHexadecimalChunkSize(): void
{
$this->expectException(AppwriteException::class);
$this->expectExceptionMessage('hexadecimal');
AwsChunked::decode("zz\r\nabc\r\n0\r\n\r\n", null);
}
public function testRejectsChunkDataWithoutCrlfTerminator(): void
{
$this->expectException(AppwriteException::class);
$this->expectExceptionMessage('CRLF');
AwsChunked::decode("3\r\nabcXX0\r\n\r\n", null);
}
}
@@ -0,0 +1,166 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Platform\Modules\S3\Responses;
use Appwrite\Platform\Modules\S3\Responses\S3Xml;
use PHPUnit\Framework\TestCase;
use Utopia\Database\Document;
final class S3XmlTest extends TestCase
{
public function testListBucketsReturnsS3Xml(): void
{
$xml = S3Xml::listBuckets([
new Document([
'$id' => 'bucket-a',
'$createdAt' => '2026-01-01T00:00:00.000+00:00',
]),
]);
$this->assertStringStartsWith('<?xml version="1.0" encoding="UTF-8"?>', $xml);
$this->assertStringContainsString('<ListAllMyBucketsResult>', $xml);
$this->assertStringContainsString('<Name>bucket-a</Name>', $xml);
}
public function testListObjectsEscapesKeys(): void
{
$xml = S3Xml::listObjects('bucket-a', [
new Document([
'$id' => 'filea',
'$createdAt' => '2026-01-01T00:00:00.000+00:00',
'name' => 'folder/file&one.txt',
'signature' => 'etag-a',
'sizeOriginal' => 12,
]),
]);
$this->assertStringContainsString('<Key>folder/file&amp;one.txt</Key>', $xml);
}
public function testListObjectsUsesFolderAndName(): void
{
$xml = S3Xml::listObjects('bucket-a', [
$this->file('filea', 'file&one.txt', 'folder/'),
]);
$this->assertStringContainsString('<Key>folder/file&amp;one.txt</Key>', $xml);
}
public function testListObjectsSupportsDelimiterAndMarkerPagination(): void
{
$firstPage = S3Xml::listObjects('bucket-a', [
$this->file('filea', 'readme.txt', 'docs/'),
$this->file('fileb', 'a.txt', 'photos/2025/'),
$this->file('filec', 'b.txt', 'photos/2026/'),
], delimiter: '/', maxKeys: 1);
$this->assertStringContainsString('<CommonPrefixes><Prefix>docs/</Prefix></CommonPrefixes>', $firstPage);
$this->assertStringContainsString('<IsTruncated>true</IsTruncated>', $firstPage);
$this->assertStringContainsString('<NextMarker>docs/</NextMarker>', $firstPage);
$secondPage = S3Xml::listObjects('bucket-a', [
$this->file('filea', 'readme.txt', 'docs/'),
$this->file('fileb', 'a.txt', 'photos/2025/'),
$this->file('filec', 'b.txt', 'photos/2026/'),
], delimiter: '/', maxKeys: 1, marker: 'docs/');
$this->assertStringNotContainsString('<Prefix>docs/</Prefix>', $secondPage);
$this->assertStringContainsString('<CommonPrefixes><Prefix>photos/</Prefix></CommonPrefixes>', $secondPage);
$this->assertStringContainsString('<IsTruncated>false</IsTruncated>', $secondPage);
}
public function testListObjectsV2SupportsPrefixPaginationAndDelimiter(): void
{
$xml = S3Xml::listObjectsV2('bucket-a', [
$this->file('filea', 'readme.txt', 'docs/'),
$this->file('fileb', 'c.txt', 'photos/2025/'),
$this->file('filec', 'a.txt', 'photos/2026/'),
$this->file('filed', 'b.txt', 'photos/2026/'),
], prefix: 'photos/', delimiter: '/', maxKeys: 1);
$this->assertStringContainsString('<ListBucketResult>', $xml);
$this->assertStringContainsString('<Prefix>photos/</Prefix>', $xml);
$this->assertStringContainsString('<KeyCount>1</KeyCount>', $xml);
$this->assertStringContainsString('<MaxKeys>1</MaxKeys>', $xml);
$this->assertStringContainsString('<IsTruncated>true</IsTruncated>', $xml);
$this->assertStringContainsString('<CommonPrefixes><Prefix>photos/2025/</Prefix></CommonPrefixes>', $xml);
$this->assertStringContainsString('<NextContinuationToken>', $xml);
}
public function testListObjectsV2ContinuationTokenStartsAfterLastKey(): void
{
$firstPage = S3Xml::listObjectsV2('bucket-a', [
$this->file('filea', 'a.txt'),
$this->file('fileb', 'b.txt'),
], maxKeys: 1);
preg_match('/<NextContinuationToken>([^<]+)<\/NextContinuationToken>/', $firstPage, $matches);
$this->assertNotEmpty($matches[1] ?? '');
$secondPage = S3Xml::listObjectsV2('bucket-a', [
$this->file('filea', 'a.txt'),
$this->file('fileb', 'b.txt'),
], continuationToken: $matches[1]);
$this->assertStringNotContainsString('<Key>a.txt</Key>', $secondPage);
$this->assertStringContainsString('<Key>b.txt</Key>', $secondPage);
$this->assertStringContainsString('<IsTruncated>false</IsTruncated>', $secondPage);
}
public function testListObjectsV2OrdersAndPaginatesNumericKeysLexicographically(): void
{
$files = [
$this->file('filea', '9'),
$this->file('fileb', '10'),
];
$firstPage = S3Xml::listObjectsV2('bucket-a', $files, maxKeys: 1);
$this->assertStringContainsString('<Key>10</Key>', $firstPage);
$this->assertStringNotContainsString('<Key>9</Key>', $firstPage);
$this->assertStringContainsString('<IsTruncated>true</IsTruncated>', $firstPage);
preg_match('/<NextContinuationToken>([^<]+)<\/NextContinuationToken>/', $firstPage, $matches);
$this->assertNotEmpty($matches[1] ?? '');
$secondPage = S3Xml::listObjectsV2('bucket-a', $files, maxKeys: 1, continuationToken: $matches[1]);
$this->assertStringContainsString('<Key>9</Key>', $secondPage);
$this->assertStringContainsString('<IsTruncated>false</IsTruncated>', $secondPage);
}
public function testMultipartResponses(): void
{
$init = S3Xml::initiateMultipart('bucket-a', 'object.txt', 'upload-id');
$parts = S3Xml::listParts('bucket-a', 'object.txt', 'upload-id', [
['partNumber' => 1, 'etag' => 'etag-1', 'size' => 10],
]);
$complete = S3Xml::completeMultipart('bucket-a', 'object.txt', 'final-etag');
$this->assertStringContainsString('<UploadId>upload-id</UploadId>', $init);
$this->assertStringContainsString('<PartNumber>1</PartNumber>', $parts);
$this->assertStringContainsString('<ETag>&quot;final-etag&quot;</ETag>', $complete);
}
public function testAccessControlPolicyRendersNestedGrantee(): void
{
$xml = S3Xml::accessControlPolicy();
$this->assertStringContainsString('<Owner><ID>appwrite</ID><DisplayName>appwrite</DisplayName></Owner>', $xml);
$this->assertStringContainsString('<Grantee><ID>appwrite</ID><DisplayName>appwrite</DisplayName></Grantee>', $xml);
$this->assertStringContainsString('<Permission>FULL_CONTROL</Permission>', $xml);
$this->assertStringNotContainsString('<Grantee>Array</Grantee>', $xml);
}
private function file(string $id, string $name, string $folder = ''): Document
{
return new Document([
'$id' => $id,
'$createdAt' => '2026-01-01T00:00:00.000+00:00',
'name' => $name,
'folder' => $folder,
'signature' => 'etag-' . $id,
'sizeOriginal' => 12,
]);
}
}